From c7c59cdc5a70bfd6e5f66f3b032ea3f8f6b4d12a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 20 Apr 2026 07:27:35 +0200 Subject: builds cmd refactor (#975) - Bugfix: `builds download` partial-block fetch decisions now account for build storage host latency - Bugfix: Transfer rate displays in `builds` commands now smooth correctly - Split `buildstorageoperations.cpp` (8.5k lines) into per-operation TUs: buildinspect, buildprimecache, buildstorageresolve, buildupdatefolder, builduploadfolder, buildvalidatebuildpart; stats moved to buildstoragestats.h. - FilteredRate extracted to zenutil. - BuildsCommand shared state consolidated into a BuildsConfiguration struct; subcommands inherit from BuildsSubCmdBase holding a `const BuildsConfiguration&` instead of a `BuildsCommand&`. - `ProgressBar` renamed to `ConsoleProgressBar`; mode enum (`ConsoleProgressMode`) lifted to namespace scope; `PushLogOperation`/`PopLogOperation`/`ForceLinebreak` promoted to virtuals on `ProgressBase`. - Free-function wrappers (`UploadFolder`, `DownloadFolder`, `ValidateBuildPart`) added around the existing operation classes so callers stop reimplementing setup + stats logging. --- src/zen/authutils.cpp | 1 + src/zen/authutils.h | 1 - src/zen/cmds/builds_cmd.cpp | 3588 ++------ src/zen/cmds/builds_cmd.h | 378 +- src/zen/cmds/exec_cmd.cpp | 45 +- src/zen/cmds/projectstore_cmd.cpp | 69 +- src/zen/cmds/wipe_cmd.cpp | 49 +- src/zen/consoleprogress.cpp | 582 ++ src/zen/consoleprogress.h | 19 + src/zen/progressbar.cpp | 567 -- src/zen/progressbar.h | 56 - src/zen/zen.cpp | 2 +- src/zenremotestore/builds/buildinspect.cpp | 463 ++ src/zenremotestore/builds/buildprimecache.cpp | 350 + .../builds/buildstorageoperations.cpp | 8560 -------------------- src/zenremotestore/builds/buildstorageresolve.cpp | 249 + src/zenremotestore/builds/buildstorageutil.cpp | 1658 +++- src/zenremotestore/builds/buildupdatefolder.cpp | 4947 +++++++++++ src/zenremotestore/builds/builduploadfolder.cpp | 2634 ++++++ .../builds/buildvalidatebuildpart.cpp | 371 + src/zenremotestore/builds/jupiterbuildstorage.cpp | 1 + .../include/zenremotestore/builds/buildinspect.h | 60 + .../zenremotestore/builds/buildprimecache.h | 96 + .../include/zenremotestore/builds/buildstorage.h | 2 +- .../zenremotestore/builds/buildstoragecache.h | 4 +- .../zenremotestore/builds/buildstorageoperations.h | 1123 --- .../zenremotestore/builds/buildstorageresolve.h | 46 + .../zenremotestore/builds/buildstoragestats.h | 182 + .../zenremotestore/builds/buildstorageutil.h | 109 +- .../zenremotestore/builds/buildupdatefolder.h | 529 ++ .../zenremotestore/builds/builduploadfolder.h | 393 + .../zenremotestore/builds/buildvalidatebuildpart.h | 121 + .../zenremotestore/chunking/chunkedcontent.h | 1 - .../include/zenremotestore/transferthreadworkers.h | 1 - src/zenremotestore/zenremotestore.cpp | 4 +- .../storage/projectstore/httpprojectstore.cpp | 6 +- src/zenutil/filteredrate.cpp | 92 + src/zenutil/include/zenutil/filteredrate.h | 37 + src/zenutil/include/zenutil/progress.h | 3 + src/zenutil/progress.cpp | 3 + 40 files changed, 13891 insertions(+), 13511 deletions(-) create mode 100644 src/zen/consoleprogress.cpp create mode 100644 src/zen/consoleprogress.h delete mode 100644 src/zen/progressbar.cpp delete mode 100644 src/zen/progressbar.h create mode 100644 src/zenremotestore/builds/buildinspect.cpp create mode 100644 src/zenremotestore/builds/buildprimecache.cpp delete mode 100644 src/zenremotestore/builds/buildstorageoperations.cpp create mode 100644 src/zenremotestore/builds/buildstorageresolve.cpp create mode 100644 src/zenremotestore/builds/buildupdatefolder.cpp create mode 100644 src/zenremotestore/builds/builduploadfolder.cpp create mode 100644 src/zenremotestore/builds/buildvalidatebuildpart.cpp create mode 100644 src/zenremotestore/include/zenremotestore/builds/buildinspect.h create mode 100644 src/zenremotestore/include/zenremotestore/builds/buildprimecache.h delete mode 100644 src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h create mode 100644 src/zenremotestore/include/zenremotestore/builds/buildstorageresolve.h create mode 100644 src/zenremotestore/include/zenremotestore/builds/buildstoragestats.h create mode 100644 src/zenremotestore/include/zenremotestore/builds/buildupdatefolder.h create mode 100644 src/zenremotestore/include/zenremotestore/builds/builduploadfolder.h create mode 100644 src/zenremotestore/include/zenremotestore/builds/buildvalidatebuildpart.h create mode 100644 src/zenutil/filteredrate.cpp create mode 100644 src/zenutil/include/zenutil/filteredrate.h (limited to 'src') diff --git a/src/zen/authutils.cpp b/src/zen/authutils.cpp index 922007ac8..2696cdcc3 100644 --- a/src/zen/authutils.cpp +++ b/src/zen/authutils.cpp @@ -9,6 +9,7 @@ #include #include #include +#include ZEN_THIRD_PARTY_INCLUDES_START #include diff --git a/src/zen/authutils.h b/src/zen/authutils.h index fa9670b3f..bd31d2daa 100644 --- a/src/zen/authutils.h +++ b/src/zen/authutils.h @@ -3,7 +3,6 @@ #pragma once #include "zen.h" -#include "zenutil/authutils.h" namespace zen { diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 3373506f2..e9b443dcb 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -4,68 +4,30 @@ #include #include -#include -#include -#include -#include -#include #include -#include -#include #include +#include #include #include -#include #include -#include -#include +#include #include -#include -#include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include #include #include -#include -#include #include #include -#include #include -#include - -#include "../progressbar.h" #include #include -#include - -ZEN_THIRD_PARTY_INCLUDES_START -#include -#include -#include -ZEN_THIRD_PARTY_INCLUDES_END - -#if ZEN_PLATFORM_WINDOWS -# include -#else -# include -# include -# include -# include -#endif - -static const bool DoExtraContentVerify = false; namespace zen { @@ -115,14 +77,6 @@ namespace builds_impl { } }; - struct MemMap - { - void* Handle = nullptr; - void* Data = nullptr; - size_t Size = 0; - std::string Name; - }; - class ZenState { public: @@ -236,1840 +190,110 @@ namespace builds_impl { } } - const std::string ZenFolderName = ".zen"; - std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.cbo"; } - // std::filesystem::path ZenStateFileJsonPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.json"; - // } - std::filesystem::path UploadTempDirectory(const std::filesystem::path& Path) { const std::u8string LocalPathString = Path.generic_u8string(); IoHash PathHash = IoHash::HashBuffer(LocalPathString.data(), LocalPathString.length()); return std::filesystem::temp_directory_path() / fmt::format("zen_{}", PathHash); - } - - const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; - - const std::string UnsyncFolderName = ".unsync"; - - const std::string UGSFolderName = ".ugs"; - const std::string LegacyZenTempFolderName = ".zen-tmp"; - - const std::vector DefaultExcludeFolders({UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName}); - const std::vector DefaultExcludeExtensions({}); - - const double DefaultLatency = 0; // .0010; - const double DefaultDelayPerKBSec = 0; // 0.00005; - - const bool SingleThreaded = false; - bool UseSparseFiles = false; - - static bool IsVerbose = false; - static bool IsQuiet = false; - static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; - -#undef ZEN_CONSOLE_VERBOSE -#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ - if (IsVerbose) \ - { \ - ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \ - } - - const std::string DefaultAccessTokenEnvVariableName( -#if ZEN_PLATFORM_WINDOWS - "UE-CloudDataCacheAccessToken"sv -#endif -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - "UE_CloudDataCacheAccessToken"sv -#endif - - ); - - static uint64_t GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory) - { - return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u; - } - - class FilteredRate - { - public: - FilteredRate() {} - - void Start() - { - if (StartTimeUS == (uint64_t)-1) - { - uint64_t Expected = (uint64_t)-1; - if (StartTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs())) - { - LastTimeUS = StartTimeUS.load(); - } - } - } - void Stop() - { - if (EndTimeUS == (uint64_t)-1) - { - uint64_t Expected = (uint64_t)-1; - EndTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs()); - } - } - - void Update(uint64_t Count) - { - if (LastTimeUS == (uint64_t)-1) - { - return; - } - uint64_t TimeUS = Timer.GetElapsedTimeUs(); - uint64_t TimeDeltaUS = TimeUS - LastTimeUS; - if (TimeDeltaUS >= 2000000) - { - uint64_t Delta = Count - LastCount; - uint64_t PerSecond = (Delta * 1000000) / TimeDeltaUS; - - LastPerSecond = PerSecond; - - LastCount = Count; - - FilteredPerSecond = (PerSecond + (LastPerSecond * 7)) / 8; - - LastTimeUS = TimeUS; - } - } - - uint64_t GetCurrent() const // If Stopped - return total count / total time - { - if (LastTimeUS == (uint64_t)-1) - { - return 0; - } - return FilteredPerSecond; - } - - uint64_t GetElapsedTimeUS() const - { - if (StartTimeUS == (uint64_t)-1) - { - return 0; - } - if (EndTimeUS == (uint64_t)-1) - { - return 0; - } - uint64_t TimeDeltaUS = EndTimeUS - StartTimeUS; - return TimeDeltaUS; - } - - bool IsActive() const { return (StartTimeUS != (uint64_t)-1) && (EndTimeUS == (uint64_t)-1); } - - private: - Stopwatch Timer; - std::atomic StartTimeUS = (uint64_t)-1; - std::atomic EndTimeUS = (uint64_t)-1; - std::atomic LastTimeUS = (uint64_t)-1; - uint64_t LastCount = 0; - uint64_t LastPerSecond = 0; - uint64_t FilteredPerSecond = 0; - }; - - uint64_t GetBytesPerSecond(uint64_t ElapsedWallTimeUS, uint64_t Count) - { - if (ElapsedWallTimeUS == 0) - { - return 0; - } - return Count * 1000000 / ElapsedWallTimeUS; - } - - bool CleanAndRemoveDirectory(WorkerThreadPool& WorkerPool, const std::filesystem::path& Directory) - { - return CleanAndRemoveDirectory(WorkerPool, AbortFlag, PauseFlag, Directory); - } - - void ValidateBuildPart(LoggerRef Log, - ProgressBase& Progress, - TransferThreadWorkers& Workers, - BuildStorageBase& Storage, - const Oid& BuildId, - Oid BuildPartId, - const std::string_view BuildPartName) - { - ZEN_TRACE_CPU("ValidateBuildPart"); - - ProgressBar::SetLogOperationName(ProgressMode, "Validate Part"); - - BuildsOperationValidateBuildPart ValidateOp(Log, - Progress, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - BuildPartId, - BuildPartName, - BuildsOperationValidateBuildPart::Options{.IsQuiet = IsQuiet, .IsVerbose = IsVerbose}); - - ValidateOp.Execute(); - - const uint64_t DownloadedCount = ValidateOp.m_DownloadStats.DownloadedChunkCount + ValidateOp.m_DownloadStats.DownloadedBlockCount; - const uint64_t DownloadedByteCount = - ValidateOp.m_DownloadStats.DownloadedChunkByteCount + ValidateOp.m_DownloadStats.DownloadedBlockByteCount; - ZEN_CONSOLE("Verified: {:>8} ({}), {}B/sec, {}", - DownloadedCount, - NiceBytes(DownloadedByteCount), - NiceNum(GetBytesPerSecond(ValidateOp.m_ValidateStats.ElapsedWallTimeUS, DownloadedByteCount)), - NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000)); - } - - struct UploadFolderOptions - { - std::filesystem::path TempDir; - uint64_t FindBlockMaxCount; - uint8_t BlockReuseMinPercentLimit; - bool AllowMultiparts; - bool CreateBuild; - bool IgnoreExistingBlocks; - bool UploadToZenCache; - const std::vector& ExcludeFolders = DefaultExcludeFolders; - const std::vector& ExcludeExtensions = DefaultExcludeExtensions; - }; - - std::vector> UploadFolder(LoggerRef Log, - ProgressBase& Progress, - TransferThreadWorkers& Workers, - StorageInstance& Storage, - const Oid& BuildId, - const Oid& BuildPartId, - const std::string_view BuildPartName, - const std::filesystem::path& Path, - const std::filesystem::path& ManifestPath, - const CbObject& MetaData, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - const UploadFolderOptions& Options) - { - ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder"); - - Stopwatch UploadTimer; - - BuildsOperationUploadFolder UploadOp( - Log, - Progress, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - Path, - Options.CreateBuild, - std::move(MetaData), - BuildsOperationUploadFolder::Options{.IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .DoExtraContentValidation = DoExtraContentVerify, - .FindBlockMaxCount = Options.FindBlockMaxCount, - .BlockReuseMinPercentLimit = Options.BlockReuseMinPercentLimit, - .AllowMultiparts = Options.AllowMultiparts, - .IgnoreExistingBlocks = Options.IgnoreExistingBlocks, - .TempDir = Options.TempDir, - .ExcludeFolders = Options.ExcludeFolders, - .ExcludeExtensions = Options.ExcludeExtensions, - .ZenExcludeManifestName = ZenExcludeManifestName, - .NonCompressableExtensions = DefaultSplitOnlyExtensions, - .PopulateCache = Options.UploadToZenCache}); - - std::vector> UploadedParts = - UploadOp.Execute(BuildPartId, BuildPartName, ManifestPath, ChunkController, ChunkCache); - if (AbortFlag) - { - return {}; - } - - ZEN_CONSOLE_VERBOSE( - "Folder scanning stats:" - "\n FoundFileCount: {}" - "\n FoundFileByteCount: {}" - "\n AcceptedFileCount: {}" - "\n AcceptedFileByteCount: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), - UploadOp.m_LocalFolderScanStats.AcceptedFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.AcceptedFileByteCount.load()), - NiceLatencyNs(UploadOp.m_LocalFolderScanStats.ElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Chunking stats:" - "\n FilesProcessed: {}" - "\n FilesChunked: {}" - "\n BytesHashed: {}" - "\n UniqueChunksFound: {}" - "\n UniqueSequencesFound: {}" - "\n UniqueBytesFound: {}" - "\n FilesFoundInCache: {}" - "\n ChunksFoundInCache: {}" - "\n FilesStoredInCache: {}" - "\n ChunksStoredInCache: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_ChunkingStats.FilesProcessed.load(), - UploadOp.m_ChunkingStats.FilesChunked.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesHashed.load()), - UploadOp.m_ChunkingStats.UniqueChunksFound.load(), - UploadOp.m_ChunkingStats.UniqueSequencesFound.load(), - NiceBytes(UploadOp.m_ChunkingStats.UniqueBytesFound.load()), - UploadOp.m_ChunkingStats.FilesFoundInCache.load(), - UploadOp.m_ChunkingStats.ChunksFoundInCache.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesFoundInCache.load()), - UploadOp.m_ChunkingStats.FilesStoredInCache.load(), - UploadOp.m_ChunkingStats.ChunksStoredInCache.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesStoredInCache.load()), - NiceLatencyNs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Find block stats:" - "\n FindBlockTimeMS: {}" - "\n PotentialChunkCount: {}" - "\n PotentialChunkByteCount: {}" - "\n FoundBlockCount: {}" - "\n FoundBlockChunkCount: {}" - "\n FoundBlockByteCount: {}" - "\n AcceptedBlockCount: {}" - "\n NewBlocksCount: {}" - "\n NewBlocksChunkCount: {}" - "\n NewBlocksChunkByteCount: {}", - NiceTimeSpanMs(UploadOp.m_FindBlocksStats.FindBlockTimeMS), - UploadOp.m_FindBlocksStats.PotentialChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.PotentialChunkByteCount), - UploadOp.m_FindBlocksStats.FoundBlockCount, - UploadOp.m_FindBlocksStats.FoundBlockChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.FoundBlockByteCount), - UploadOp.m_FindBlocksStats.AcceptedBlockCount, - UploadOp.m_FindBlocksStats.NewBlocksCount, - UploadOp.m_FindBlocksStats.NewBlocksChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount)); - - ZEN_CONSOLE_VERBOSE( - "Reuse block stats:" - "\n AcceptedChunkCount: {}" - "\n AcceptedByteCount: {}" - "\n AcceptedRawByteCount: {}" - "\n RejectedBlockCount: {}" - "\n RejectedChunkCount: {}" - "\n RejectedByteCount: {}" - "\n AcceptedReduntantChunkCount: {}" - "\n AcceptedReduntantByteCount: {}", - UploadOp.m_ReuseBlocksStats.AcceptedChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedByteCount), - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedRawByteCount), - UploadOp.m_ReuseBlocksStats.RejectedBlockCount, - UploadOp.m_ReuseBlocksStats.RejectedChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.RejectedByteCount), - UploadOp.m_ReuseBlocksStats.AcceptedReduntantChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedReduntantByteCount)); - - ZEN_CONSOLE_VERBOSE( - "Generate blocks stats:" - "\n GeneratedBlockByteCount: {}" - "\n GeneratedBlockCount: {}" - "\n GenerateBlocksElapsedWallTimeUS: {}", - NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), - UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), - NiceLatencyNs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Generate blocks stats:" - "\n ChunkCount: {}" - "\n ChunkByteCount: {}" - "\n CompressedChunkCount: {}" - "\n CompressChunksElapsedWallTimeUS: {}", - UploadOp.m_LooseChunksStats.ChunkCount, - NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount), - UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), - NiceLatencyNs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Disk stats:" - "\n OpenReadCount: {}" - "\n OpenWriteCount: {}" - "\n ReadCount: {}" - "\n ReadByteCount: {}" - "\n WriteCount: {} ({} cloned)" - "\n WriteByteCount: {} ({} cloned)" - "\n CurrentOpenFileCount: {}", - UploadOp.m_DiskStats.OpenReadCount.load(), - UploadOp.m_DiskStats.OpenWriteCount.load(), - UploadOp.m_DiskStats.ReadCount.load(), - NiceBytes(UploadOp.m_DiskStats.ReadByteCount.load()), - UploadOp.m_DiskStats.WriteCount.load(), - UploadOp.m_DiskStats.CloneCount.load(), - NiceBytes(UploadOp.m_DiskStats.WriteByteCount.load()), - NiceBytes(UploadOp.m_DiskStats.CloneByteCount.load()), - UploadOp.m_DiskStats.CurrentOpenFileCount.load()); - - ZEN_CONSOLE_VERBOSE( - "Upload stats:" - "\n BlockCount: {}" - "\n BlocksBytes: {}" - "\n ChunkCount: {}" - "\n ChunksBytes: {}" - "\n ReadFromDiskBytes: {}" - "\n MultipartAttachmentCount: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_UploadStats.BlockCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), - UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), - NiceBytes(UploadOp.m_UploadStats.ReadFromDiskBytes.load()), - UploadOp.m_UploadStats.MultipartAttachmentCount.load(), - NiceLatencyNs(UploadOp.m_UploadStats.ElapsedWallTimeUS * 1000)); - - const double DeltaByteCountPercent = - UploadOp.m_ChunkingStats.BytesHashed > 0 - ? (100.0 * (UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes)) / - (UploadOp.m_ChunkingStats.BytesHashed) - : 0.0; - - const std::string MultipartAttachmentStats = - Options.AllowMultiparts ? fmt::format(" ({} as multipart)", UploadOp.m_UploadStats.MultipartAttachmentCount.load()) : ""; - - if (!IsQuiet) - { - ZEN_CONSOLE( - "Uploaded part {} ('{}') to build {}, {}\n" - " Scanned files: {:>8} ({}), {}B/sec, {}\n" - " New data: {:>8} ({}) {:.1f}%\n" - " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n" - " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n" - " Uploaded: {:>8} ({}), {}bits/sec, {}\n" - " Blocks: {:>8} ({})\n" - " Chunks: {:>8} ({}){}", - BuildPartId, - BuildPartName, - BuildId, - NiceTimeSpanMs(UploadTimer.GetElapsedTimeMs()), - - UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_ChunkingStats.ElapsedWallTimeUS, UploadOp.m_ChunkingStats.BytesHashed)), - NiceTimeSpanMs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS / 1000), - - UploadOp.m_FindBlocksStats.NewBlocksChunkCount + UploadOp.m_LooseChunksStats.CompressedChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes), - DeltaByteCountPercent, - - UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount), - NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, - UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount)), - NiceTimeSpanMs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000), - - UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkRawBytes), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS, - UploadOp.m_LooseChunksStats.CompressedChunkRawBytes)), - NiceTimeSpanMs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000), - - UploadOp.m_UploadStats.BlockCount.load() + UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes + UploadOp.m_UploadStats.ChunksBytes), - NiceNum(GetBytesPerSecond(UploadOp.m_UploadStats.ElapsedWallTimeUS, - (UploadOp.m_UploadStats.ChunksBytes + UploadOp.m_UploadStats.BlocksBytes) * 8)), - NiceTimeSpanMs(UploadOp.m_UploadStats.ElapsedWallTimeUS / 1000), - - UploadOp.m_UploadStats.BlockCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), - - UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), - MultipartAttachmentStats); - } - return UploadedParts; - } - - struct VerifyFolderStatistics - { - std::atomic FilesVerified = 0; - std::atomic FilesFailed = 0; - std::atomic ReadBytes = 0; - uint64_t VerifyElapsedWallTimeUs = 0; - }; - - void VerifyFolder(TransferThreadWorkers& Workers, - const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const std::filesystem::path& Path, - const std::vector& ExcludeFolders, - bool VerifyFileHash, - VerifyFolderStatistics& VerifyFolderStats) - { - ZEN_TRACE_CPU("VerifyFolder"); - - Stopwatch Timer; - - ProgressBar ProgressBar(ProgressMode, "Verify Files"); - - WorkerThreadPool& VerifyPool = Workers.GetIOWorkerPool(); - - ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - - const uint32_t PathCount = gsl::narrow(Content.Paths.size()); - - RwLock ErrorLock; - std::vector Errors; - - auto IsAcceptedFolder = [ExcludeFolders = ExcludeFolders](const std::string_view& RelativePath) -> bool { - for (const std::string& ExcludeFolder : ExcludeFolders) - { - if (RelativePath.starts_with(ExcludeFolder)) - { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } - } - } - return true; - }; - - for (uint32_t PathIndex = 0; PathIndex < PathCount; PathIndex++) - { - if (Work.IsAborted()) - { - break; - } - - Work.ScheduleWork( - VerifyPool, - [&Path, &Content, &Lookup, &ErrorLock, &Errors, &VerifyFolderStats, VerifyFileHash, &IsAcceptedFolder, PathIndex]( - std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("VerifyFile_work"); - - // TODO: Convert ScheduleWork body to function - - const std::filesystem::path TargetPath = (Path / Content.Paths[PathIndex]).make_preferred(); - if (IsAcceptedFolder(TargetPath.parent_path().generic_string())) - { - const uint64_t ExpectedSize = Content.RawSizes[PathIndex]; - if (!IsFile(TargetPath)) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize)); - }); - VerifyFolderStats.FilesFailed++; - } - else - { - std::error_code Ec; - uint64_t SizeOnDisk = gsl::narrow(FileSizeFromPath(TargetPath, Ec)); - if (Ec) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back( - fmt::format("Failed to get size of file {}: {} ({})", TargetPath, Ec.message(), Ec.value())); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk < ExpectedSize) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Size of file {} is smaller than expected. Expected: {}, Found: {}", - TargetPath, - ExpectedSize, - SizeOnDisk)); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk > ExpectedSize) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Size of file {} is bigger than expected. Expected: {}, Found: {}", - TargetPath, - ExpectedSize, - SizeOnDisk)); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk > 0 && VerifyFileHash) - { - const IoHash& ExpectedRawHash = Content.RawHashes[PathIndex]; - IoBuffer Buffer = IoBufferBuilder::MakeFromFile(TargetPath); - IoHash RawHash = IoHash::HashBuffer(Buffer); - if (RawHash != ExpectedRawHash) - { - uint64_t FileOffset = 0; - const uint32_t SequenceIndex = Lookup.RawHashToSequenceIndex.at(ExpectedRawHash); - const uint32_t OrderOffset = Lookup.SequenceIndexChunkOrderOffset[SequenceIndex]; - for (uint32_t OrderIndex = OrderOffset; - OrderIndex < OrderOffset + Content.ChunkedContent.ChunkCounts[SequenceIndex]; - OrderIndex++) - { - uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex]; - uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; - IoHash ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; - IoBuffer FileChunk = IoBuffer(Buffer, FileOffset, ChunkSize); - if (IoHash::HashBuffer(FileChunk) != ChunkHash) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format( - "WARNING: Hash of file {} does not match expected hash. Expected: {}, Found: {}. " - "Mismatch at chunk {}", - TargetPath, - ExpectedRawHash, - RawHash, - OrderIndex - OrderOffset)); - }); - break; - } - FileOffset += ChunkSize; - } - VerifyFolderStats.FilesFailed++; - } - VerifyFolderStats.ReadBytes += SizeOnDisk; - } - } - } - VerifyFolderStats.FilesVerified++; - } - }, - [&, PathIndex](std::exception_ptr Ex, std::atomic&) { - std::string Description; - try - { - std::rethrow_exception(Ex); - } - catch (const std::exception& Ex) - { - Description = Ex.what(); - } - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Failed verifying file '{}'. Reason: {}", - (Path / Content.Paths[PathIndex]).make_preferred(), - Description)); - }); - VerifyFolderStats.FilesFailed++; - }); - } - - Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(PendingWork); - std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}", - VerifyFolderStats.FilesVerified.load(), - PathCount, - NiceBytes(VerifyFolderStats.ReadBytes.load()), - VerifyFolderStats.FilesFailed.load()); - ProgressBar.UpdateState({.Task = "Verifying files ", - .Details = Details, - .TotalCount = gsl::narrow(PathCount), - .RemainingCount = gsl::narrow(PathCount - VerifyFolderStats.FilesVerified.load()), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }); - VerifyFolderStats.VerifyElapsedWallTimeUs = Timer.GetElapsedTimeUs(); - - ProgressBar.Finish(); - if (AbortFlag) - { - return; - } - - for (const std::string& Error : Errors) - { - ZEN_CONSOLE_ERROR("{}", Error); - } - if (!Errors.empty()) - { - throw std::runtime_error(fmt::format("Verify failed with {} errors", Errors.size())); - } - } - - CbObject GetBuild(BuildStorageBase& Storage, const Oid& BuildId) - { - Stopwatch GetBuildTimer; - CbObject BuildObject = Storage.GetBuild(BuildId); - if (!IsQuiet) - { - ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", - NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), - BuildObject["name"sv].AsString(), - NiceBytes(BuildObject.GetSize())); - - ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv)); - } - return BuildObject; - } - - std::vector GetNewPaths(const std::span KnownPaths, - const std::span Paths) - { - tsl::robin_set KnownPathsSet; - KnownPathsSet.reserve(KnownPaths.size()); - for (const std::filesystem::path& LocalPath : KnownPaths) - { - KnownPathsSet.insert(LocalPath.generic_string()); - } - - std::vector NewPaths; - for (const std::filesystem::path& UntrackedPath : Paths) - { - if (!KnownPathsSet.contains(UntrackedPath.generic_string())) - { - NewPaths.push_back(UntrackedPath); - } - } - return NewPaths; - } - - BuildSaveState GetLocalStateFromPaths(TransferThreadWorkers& Workers, - GetFolderContentStatistics& LocalFolderScanStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - std::span PathsToCheck) - { - FolderContent FolderState; - ChunkedFolderContent ChunkedContent; - { - ProgressBar ProgressBar(ProgressMode, "Check Files"); - FolderState = GetValidFolderContent( - Workers.GetIOWorkerPool(), - LocalFolderScanStats, - Path, - PathsToCheck, - [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) { - std::string Details = - fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load()); - ProgressBar.UpdateState({.Task = "Checking files ", - .Details = Details, - .TotalCount = PathCount, - .RemainingCount = PathCount - CompletedPathCount, - .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, - false); - }, - GetUpdateDelayMS(ProgressMode), - AbortFlag, - PauseFlag); - ProgressBar.Finish(); - } - - if (FolderState.Paths.size() > 0) - { - uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : FolderState.RawSizes) - { - ByteCountToScan += RawSize; - } - ProgressBar ProgressBar(ProgressMode, "Scan Files"); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkingStatistics LocalChunkingStats; - ChunkedContent = ChunkFolderContent( - LocalChunkingStats, - Workers.GetIOWorkerPool(), - Path, - FolderState, - ChunkController, - ChunkCache, - GetUpdateDelayMS(ProgressMode), - [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { - FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - LocalChunkingStats.FilesProcessed.load(), - FolderState.Paths.size(), - NiceBytes(LocalChunkingStats.BytesHashed.load()), - NiceBytes(ByteCountToScan), - NiceNum(FilteredBytesHashed.GetCurrent()), - LocalChunkingStats.UniqueChunksFound.load(), - NiceBytes(LocalChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }, - AbortFlag, - PauseFlag); - ChunkingStats += LocalChunkingStats; - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - } - - return BuildSaveState{.State = BuildState{.ChunkedContent = std::move(ChunkedContent)}, - .FolderState = FolderState, - .LocalPath = Path}; - } - - BuildSaveState GetLocalContent(TransferThreadWorkers& Workers, - GetFolderContentStatistics& LocalFolderScanStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - const std::filesystem::path& StateFilePath, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache) - { - Stopwatch ReadStateTimer; - bool FileExists = IsFile(StateFilePath); - if (!FileExists) - { - ZEN_CONSOLE("No known local state file in {}, falling back to scanning", Path); - return {}; - } - - BuildSaveState SavedLocalState; - try - { - SavedLocalState = ReadBuildSaveStateFile(StateFilePath); - if (!IsQuiet) - { - ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); - } - } - catch (const std::exception& Ex) - { - ZEN_CONSOLE_WARN("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what()); - return {}; - } - - FolderContent CurrentLocalFolderState; - { - ProgressBar ProgressBar(ProgressMode, "Check Known Files"); - CurrentLocalFolderState = GetValidFolderContent( - Workers.GetIOWorkerPool(), - LocalFolderScanStats, - Path, - SavedLocalState.FolderState.Paths, - [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) { - std::string Details = - fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load()); - ProgressBar.UpdateState({.Task = "Checking files ", - .Details = Details, - .TotalCount = PathCount, - .RemainingCount = PathCount - CompletedPathCount, - .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, - false); - }, - GetUpdateDelayMS(ProgressMode), - AbortFlag, - PauseFlag); - ProgressBar.Finish(); - } - if (AbortFlag) - { - return {}; - } - - if (!SavedLocalState.FolderState.AreKnownFilesEqual(CurrentLocalFolderState)) - { - const size_t LocalStatePathCount = SavedLocalState.FolderState.Paths.size(); - std::vector DeletedPaths; - FolderContent UpdatedContent = GetUpdatedContent(SavedLocalState.FolderState, CurrentLocalFolderState, DeletedPaths); - if (!DeletedPaths.empty()) - { - SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths); - } - - if (!IsQuiet) - { - ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", - DeletedPaths.size(), - UpdatedContent.Paths.size(), - LocalStatePathCount); - } - if (UpdatedContent.Paths.size() > 0) - { - uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : UpdatedContent.RawSizes) - { - ByteCountToScan += RawSize; - } - ProgressBar ProgressBar(ProgressMode, "Scan Known Files"); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkingStatistics LocalChunkingStats; - ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( - LocalChunkingStats, - Workers.GetIOWorkerPool(), - Path, - UpdatedContent, - ChunkController, - ChunkCache, - GetUpdateDelayMS(ProgressMode), - [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { - FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - LocalChunkingStats.FilesProcessed.load(), - UpdatedContent.Paths.size(), - NiceBytes(LocalChunkingStats.BytesHashed.load()), - NiceBytes(ByteCountToScan), - NiceNum(FilteredBytesHashed.GetCurrent()), - LocalChunkingStats.UniqueChunksFound.load(), - NiceBytes(LocalChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }, - AbortFlag, - PauseFlag); - - ChunkingStats += LocalChunkingStats; - - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - if (AbortFlag) - { - return {}; - } - SavedLocalState.State.ChunkedContent = - MergeChunkedFolderContents(SavedLocalState.State.ChunkedContent, {{UpdatedLocalContent}}); - } - } - else - { - // Remove files from LocalContent no longer in LocalFolderState - tsl::robin_set LocalFolderPaths; - LocalFolderPaths.reserve(SavedLocalState.FolderState.Paths.size()); - for (const std::filesystem::path& LocalFolderPath : SavedLocalState.FolderState.Paths) - { - LocalFolderPaths.insert(LocalFolderPath.generic_string()); - } - std::vector DeletedPaths; - for (const std::filesystem::path& LocalContentPath : SavedLocalState.State.ChunkedContent.Paths) - { - if (!LocalFolderPaths.contains(LocalContentPath.generic_string())) - { - DeletedPaths.push_back(LocalContentPath); - } - } - if (!DeletedPaths.empty()) - { - SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths); - } - } - - SavedLocalState.FolderState = CurrentLocalFolderState; - - return SavedLocalState; - } - - ChunkedFolderContent ScanAndChunkFolder( - TransferThreadWorkers& Workers, - GetFolderContentStatistics& GetFolderContentStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - std::function&& IsAcceptedFolder, - std::function&& IsAcceptedFile, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache) - { - Stopwatch Timer; - - ZEN_TRACE_CPU("ScanAndChunkFolder"); - - FolderContent Content = GetFolderContent( - GetFolderContentStats, - Path, - std::move(IsAcceptedFolder), - std::move(IsAcceptedFile), - Workers.GetIOWorkerPool(), - GetUpdateDelayMS(ProgressMode), - [](bool, std::ptrdiff_t) {}, - AbortFlag); - if (AbortFlag) - { - return {}; - } - - BuildState LocalContent = GetLocalContent(Workers, - GetFolderContentStats, - ChunkingStats, - Path, - ZenStateFilePath(Path / ZenFolderName), - ChunkController, - ChunkCache) - .State; - - std::vector UntrackedPaths = GetNewPaths(LocalContent.ChunkedContent.Paths, Content.Paths); - - BuildState UntrackedLocalContent = - GetLocalStateFromPaths(Workers, GetFolderContentStats, ChunkingStats, Path, ChunkController, ChunkCache, UntrackedPaths).State; - - ChunkedFolderContent Result = MergeChunkedFolderContents(LocalContent.ChunkedContent, - std::vector{UntrackedLocalContent.ChunkedContent}); - - const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0)); - const uint64_t ChunkedRawSize = - std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0)); - - if (!IsQuiet) - { - ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", - Result.Paths.size(), - NiceBytes(TotalRawSize), - Result.ChunkedContent.ChunkHashes.size(), - NiceBytes(ChunkedRawSize), - Path, - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); - } - return Result; - }; - - struct DownloadOptions - { - std::filesystem::path SystemRootDir; - std::filesystem::path ZenFolderPath; - bool AllowMultiparts = true; - EPartialBlockRequestMode PartialBlockRequestMode = EPartialBlockRequestMode::Mixed; - bool CleanTargetFolder = false; - bool PostDownloadVerify = false; - bool EnableOtherDownloadsScavenging = true; - bool EnableTargetFolderScavenging = true; - bool AllowFileClone = true; - std::vector IncludeWildcards; - std::vector ExcludeWildcards; - uint64_t MaximumInMemoryPayloadSize = 512u * 1024u; - bool PopulateCache = true; - bool AppendNewContent = false; - std::vector ExcludeFolders = DefaultExcludeFolders; - }; - - void DownloadFolder(LoggerRef InLog, - ProgressBase& Progress, - TransferThreadWorkers& Workers, - StorageInstance& Storage, - const BuildStorageCache::Statistics& StorageCacheStats, - const Oid& BuildId, - const std::vector& BuildPartIds, - std::span BuildPartNames, - const std::filesystem::path& DownloadSpecPath, - const std::filesystem::path& Path, - const DownloadOptions& Options) - { - ZEN_TRACE_CPU("DownloadFolder"); - ZEN_SCOPED_LOG(InLog); - - ProgressBar::SetLogOperationName(ProgressMode, "Download Folder"); - - enum TaskSteps : uint32_t - { - CheckState, - CompareState, - Download, - Verify, - Cleanup, - StepCount - }; - - auto EndProgress = - MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); - - Stopwatch DownloadTimer; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount); - - const std::filesystem::path ZenTempFolder = ZenTempFolderPath(Options.ZenFolderPath); - CreateDirectories(ZenTempFolder); - - std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); - - std::vector> AllBuildParts = - ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); - - BuildManifest Manifest; - if (!DownloadSpecPath.empty()) - { - const std::filesystem::path AbsoluteDownloadSpecPath = - DownloadSpecPath.is_relative() ? MakeSafeAbsolutePath(Path / DownloadSpecPath) : MakeSafeAbsolutePath(DownloadSpecPath); - Manifest = ParseBuildManifest(DownloadSpecPath); - } - - std::vector PartContents; - - std::unique_ptr ChunkController; - - std::vector BlockDescriptions; - std::vector LooseChunkHashes; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount); - - ChunkedFolderContent RemoteContent = GetRemoteContent(InLog, - Storage, - BuildId, - AllBuildParts, - Manifest, - Options.IncludeWildcards, - Options.ExcludeWildcards, - ChunkController, - PartContents, - BlockDescriptions, - LooseChunkHashes, - IsQuiet, - IsVerbose, - DoExtraContentVerify); - - const std::uint64_t LargeAttachmentSize = Options.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; - GetFolderContentStatistics LocalFolderScanStats; - ChunkingStatistics ChunkingStats; - - BuildSaveState LocalState; - - if (IsDir(Path)) - { - if (!ChunkController && !IsQuiet) - { - ZEN_CONSOLE_INFO("Unspecified chunking algorithm, using default"); - ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - } - std::unique_ptr ChunkCache(CreateNullChunkingCache()); - - LocalState = GetLocalContent(Workers, - LocalFolderScanStats, - ChunkingStats, - Path, - ZenStateFilePath(Path / ZenFolderName), - *ChunkController, - *ChunkCache); - - std::vector UntrackedPaths = GetNewPaths(LocalState.State.ChunkedContent.Paths, RemoteContent.Paths); - - BuildSaveState UntrackedLocalContent = - GetLocalStateFromPaths(Workers, LocalFolderScanStats, ChunkingStats, Path, *ChunkController, *ChunkCache, UntrackedPaths); - - if (!UntrackedLocalContent.State.ChunkedContent.Paths.empty()) - { - LocalState.State.ChunkedContent = - MergeChunkedFolderContents(LocalState.State.ChunkedContent, - std::vector{UntrackedLocalContent.State.ChunkedContent}); - - // TODO: Helper - LocalState.FolderState.Paths.insert(LocalState.FolderState.Paths.begin(), - UntrackedLocalContent.FolderState.Paths.begin(), - UntrackedLocalContent.FolderState.Paths.end()); - LocalState.FolderState.RawSizes.insert(LocalState.FolderState.RawSizes.begin(), - UntrackedLocalContent.FolderState.RawSizes.begin(), - UntrackedLocalContent.FolderState.RawSizes.end()); - LocalState.FolderState.Attributes.insert(LocalState.FolderState.Attributes.begin(), - UntrackedLocalContent.FolderState.Attributes.begin(), - UntrackedLocalContent.FolderState.Attributes.end()); - LocalState.FolderState.ModificationTicks.insert(LocalState.FolderState.ModificationTicks.begin(), - UntrackedLocalContent.FolderState.ModificationTicks.begin(), - UntrackedLocalContent.FolderState.ModificationTicks.end()); - } - - if (Options.AppendNewContent) - { - RemoteContent = ApplyChunkedContentOverlay(LocalState.State.ChunkedContent, - RemoteContent, - Options.IncludeWildcards, - Options.ExcludeWildcards); - } -#if ZEN_BUILD_DEBUG - ValidateChunkedFolderContent(RemoteContent, - BlockDescriptions, - LooseChunkHashes, - Options.IncludeWildcards, - Options.ExcludeWildcards); -#endif // ZEN_BUILD_DEBUG - } - else - { - CreateDirectories(Path); - } - if (AbortFlag) - { - return; - } - - LocalState.LocalPath = Path; - - { - BuildsSelection::Build RemoteBuildState = {.Id = BuildId, - .IncludeWildcards = Options.IncludeWildcards, - .ExcludeWildcards = Options.ExcludeWildcards}; - RemoteBuildState.Parts.reserve(BuildPartIds.size()); - for (size_t PartIndex = 0; PartIndex < BuildPartIds.size(); PartIndex++) - { - RemoteBuildState.Parts.push_back( - {BuildsSelection::BuildPart{.Id = BuildPartIds[PartIndex], - .Name = PartIndex < BuildPartNames.size() ? BuildPartNames[PartIndex] : ""}}); - } - - if (Options.AppendNewContent) - { - LocalState.State.Selection.Builds.emplace_back(std::move(RemoteBuildState)); - } - else - { - LocalState.State.Selection.Builds = std::vector{std::move(RemoteBuildState)}; - } - } - - if ((Options.EnableTargetFolderScavenging || Options.AppendNewContent) && !Options.CleanTargetFolder && - CompareChunkedContent(RemoteContent, LocalState.State.ChunkedContent)) - { - if (!IsQuiet) - { - ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", - NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); - } - - Stopwatch WriteStateTimer; - - CbObject StateObject = CreateBuildSaveStateObject(LocalState); - CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path()); - TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView()); - if (!IsQuiet) - { - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); - } - - AddDownloadedPath(Options.SystemRootDir, - BuildsDownloadInfo{.Selection = LocalState.State.Selection, - .LocalPath = Path, - .StateFilePath = ZenStateFilePath(Options.ZenFolderPath), - .Iso8601Date = DateTime::Now().ToIso8601()}); - } - else - { - ExtendableStringBuilder<128> BuildPartString; - for (const std::pair& BuildPart : AllBuildParts) - { - BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first)); - } - - uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0)); - - if (!IsQuiet) - { - ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize)); - } - - Stopwatch IndexTimer; - - const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalState.State.ChunkedContent); - const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); - - if (!IsQuiet) - { - ZEN_INFO("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount); - - BuildsOperationUpdateFolder Updater( - InLog, - Progress, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - Path, - LocalState.State.ChunkedContent, - LocalLookup, - RemoteContent, - RemoteLookup, - BlockDescriptions, - LooseChunkHashes, - BuildsOperationUpdateFolder::Options{ - .IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .AllowFileClone = Options.AllowFileClone, - .UseSparseFiles = UseSparseFiles, - .SystemRootDir = Options.SystemRootDir, - .ZenFolderPath = Options.ZenFolderPath, - .LargeAttachmentSize = LargeAttachmentSize, - .PreferredMultipartChunkSize = PreferredMultipartChunkSize, - .PartialBlockRequestMode = Options.PartialBlockRequestMode, - .WipeTargetFolder = Options.CleanTargetFolder, - .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging, - .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging || Options.AppendNewContent, - .ValidateCompletedSequences = Options.PostDownloadVerify, - .ExcludeFolders = Options.ExcludeFolders, - .MaximumInMemoryPayloadSize = Options.MaximumInMemoryPayloadSize, - .PopulateCache = Options.PopulateCache}); - { - ProgressBar::PushLogOperation(ProgressMode, "Download"); - auto _ = MakeGuard([]() { ProgressBar::PopLogOperation(ProgressMode); }); - FolderContent UpdatedLocalFolderState; - Updater.Execute(UpdatedLocalFolderState); - - LocalState.State.ChunkedContent = RemoteContent; - LocalState.FolderState = std::move(UpdatedLocalFolderState); - } - - VerifyFolderStatistics VerifyFolderStats; - if (!AbortFlag) - { - AddDownloadedPath(Options.SystemRootDir, - BuildsDownloadInfo{.Selection = LocalState.State.Selection, - .LocalPath = Path, - .StateFilePath = ZenStateFilePath(Options.ZenFolderPath), - .Iso8601Date = DateTime::Now().ToIso8601()}); - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount); - - VerifyFolder(Workers, - RemoteContent, - RemoteLookup, - Path, - Options.ExcludeFolders, - Options.PostDownloadVerify, - VerifyFolderStats); - - Stopwatch WriteStateTimer; - CbObject StateObject = CreateBuildSaveStateObject(LocalState); - - CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path()); - TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView()); - if (!IsQuiet) - { - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); - } - -#if 0 - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(StateObject, SB); - WriteFile(ZenStateFileJsonPath(Options.ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); -#endif // 0 - const uint64_t DownloadCount = Updater.m_DownloadStats.DownloadedChunkCount.load() + - Updater.m_DownloadStats.DownloadedBlockCount.load() + - Updater.m_DownloadStats.DownloadedPartialBlockCount.load(); - const uint64_t DownloadByteCount = Updater.m_DownloadStats.DownloadedChunkByteCount.load() + - Updater.m_DownloadStats.DownloadedBlockByteCount.load() + - Updater.m_DownloadStats.DownloadedPartialBlockByteCount.load(); - const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs(); - - if (!IsQuiet) - { - std::string CloneInfo; - if (Updater.m_DiskStats.CloneByteCount > 0) - { - CloneInfo = fmt::format(" ({} cloned)", NiceBytes(Updater.m_DiskStats.CloneByteCount.load())); - } - - std::string DownloadDetails; - { - ExtendableStringBuilder<128> SB; - BuildStorageBase::ExtendedStatistics ExtendedDownloadStats; - if (Storage.BuildStorage->GetExtendedStatistics(ExtendedDownloadStats)) - { - if (!ExtendedDownloadStats.ReceivedBytesPerSource.empty()) - { - for (auto& It : ExtendedDownloadStats.ReceivedBytesPerSource) - { - if (SB.Size() > 0) - { - SB.Append(", "sv); - } - SB.Append(It.first); - SB.Append(": "sv); - SB.Append(NiceBytes(It.second)); - } - } - } - if (Storage.CacheStorage) - { - if (SB.Size() > 0) - { - SB.Append(", "sv); - } - SB.Append("Cache: "); - SB.Append(NiceBytes(StorageCacheStats.TotalBytesRead.load())); - } - if (SB.Size() > 0) - { - DownloadDetails = fmt::format(" ({})", SB.ToView()); - } - } - - ZEN_CONSOLE( - "Downloaded build {}, parts:{} in {}\n" - " Scavenge: {} (Target: {}, Cache: {}, Others: {})\n" - " Download: {} ({}) {}bits/s{}\n" - " Write: {} ({}) {}B/s{}\n" - " Clean: {}\n" - " Finalize: {}\n" - " Verify: {}", - BuildId, - BuildPartString.ToView(), - NiceTimeSpanMs(DownloadTimeMs), - - NiceTimeSpanMs((Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs + - Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs + - Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs) / - 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000), - - DownloadCount, - NiceBytes(DownloadByteCount), - NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)), - DownloadDetails, - - Updater.m_DiskStats.WriteCount.load(), - NiceBytes(Updater.m_WrittenChunkByteCount.load()), - NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.WriteTimeUs, Updater.m_DiskStats.WriteByteCount.load())), - CloneInfo, - - NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000), - - NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000), - - NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); - } - } - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); - - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), ZenTempFolder); - } - - void ListBuild(StorageInstance& Storage, - const Oid& BuildId, - const std::vector& BuildPartIds, - std::span BuildPartNames, - std::span IncludeWildcards, - std::span ExcludeWildcards, - CbObjectWriter* OptionalStructuredOutput) - { - std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->AddObjectId("buildId"sv, BuildId); - OptionalStructuredOutput->AddObject("build"sv, BuildObject); - } - - std::vector> AllBuildParts = - ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); - - if (!AllBuildParts.empty()) - { - Stopwatch GetBuildPartTimer; - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginArray("parts"sv); - } - - for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++) - { - const Oid BuildPartId = AllBuildParts[BuildPartIndex].first; - const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second; - CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginObject(); - OptionalStructuredOutput->AddObjectId("id"sv, BuildPartId); - OptionalStructuredOutput->AddString("partName"sv, BuildPartName); - } - { - if (OptionalStructuredOutput != nullptr) - { - } - else if (!IsQuiet) - { - ZEN_CONSOLE("{}Part: {} ('{}'):\n", - BuildPartIndex > 0 ? "\n" : "", - BuildPartId, - BuildPartName, - NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), - NiceBytes(BuildPartManifest.GetSize())); - } - - std::vector Paths; - std::vector RawHashes; - std::vector RawSizes; - std::vector Attributes; - - SourcePlatform Platform; - std::vector SequenceRawHashes; - std::vector ChunkCounts; - std::vector AbsoluteChunkOrders; - std::vector LooseChunkHashes; - std::vector LooseChunkRawSizes; - std::vector BlockRawHashes; - - ReadBuildContentFromCompactBinary(BuildPartManifest, - Platform, - Paths, - RawHashes, - RawSizes, - Attributes, - SequenceRawHashes, - ChunkCounts, - AbsoluteChunkOrders, - LooseChunkHashes, - LooseChunkRawSizes, - BlockRawHashes); - - std::vector Order(Paths.size()); - std::iota(Order.begin(), Order.end(), 0); - - std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) { - const std::filesystem::path& LhsPath = Paths[Lhs]; - const std::filesystem::path& RhsPath = Paths[Rhs]; - return LhsPath < RhsPath; - }); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginArray("files"sv); - } - { - for (size_t Index : Order) - { - const std::filesystem::path& Path = Paths[Index]; - if (IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(Path.generic_string()), /*CaseSensitive*/ true)) - { - const IoHash& RawHash = RawHashes[Index]; - const uint64_t RawSize = RawSizes[Index]; - const uint32_t Attribute = Attributes[Index]; - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginObject(); - { - OptionalStructuredOutput->AddString("path"sv, fmt::format("{}", Path)); - OptionalStructuredOutput->AddInteger("rawSize"sv, RawSize); - OptionalStructuredOutput->AddHash("rawHash"sv, RawHash); - switch (Platform) - { - case SourcePlatform::Windows: - OptionalStructuredOutput->AddInteger("attributes"sv, Attribute); - break; - case SourcePlatform::MacOS: - case SourcePlatform::Linux: - OptionalStructuredOutput->AddString("chmod"sv, fmt::format("{:#04o}", Attribute)); - break; - default: - throw std::runtime_error(fmt::format("Unsupported platform: {}", (int)Platform)); - } - } - OptionalStructuredOutput->EndObject(); - } - else - { - ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash); - } - } - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndArray(); // "files" - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndObject(); - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndArray(); // parts - } - } - } - - void DiffFolders(TransferThreadWorkers& Workers, - const std::filesystem::path& BasePath, - const std::filesystem::path& ComparePath, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - const std::vector& ExcludeFolders, - const std::vector& ExcludeExtensions) - { - ZEN_TRACE_CPU("DiffFolders"); - - ProgressBar::SetLogOperationName(ProgressMode, "Diff Folders"); - - enum TaskSteps : uint32_t - { - CheckBase, - CheckCompare, - Diff, - Cleanup, - StepCount - }; - - auto EndProgress = - MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); - - ChunkedFolderContent BaseFolderContent; - ChunkedFolderContent CompareFolderContent; - - { - auto IsAcceptedFolder = [ExcludeFolders](const std::string_view& RelativePath) -> bool { - for (const std::string& ExcludeFolder : ExcludeFolders) - { - if (RelativePath.starts_with(ExcludeFolder)) - { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } - } - } - return true; - }; - - auto IsAcceptedFile = [ExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool { - for (const std::string& ExcludeExtension : ExcludeExtensions) - { - if (RelativePath.ends_with(ExcludeExtension)) - { - return false; - } - } - return true; - }; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckBase, TaskSteps::StepCount); - - GetFolderContentStatistics BaseGetFolderContentStats; - ChunkingStatistics BaseChunkingStats; - BaseFolderContent = ScanAndChunkFolder(Workers, - BaseGetFolderContentStats, - BaseChunkingStats, - BasePath, - IsAcceptedFolder, - IsAcceptedFile, - ChunkController, - ChunkCache); - if (AbortFlag) - { - return; - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckCompare, TaskSteps::StepCount); - - GetFolderContentStatistics CompareGetFolderContentStats; - ChunkingStatistics CompareChunkingStats; - CompareFolderContent = ScanAndChunkFolder(Workers, - CompareGetFolderContentStats, - CompareChunkingStats, - ComparePath, - IsAcceptedFolder, - IsAcceptedFile, - ChunkController, - ChunkCache); - - if (AbortFlag) - { - return; - } - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Diff, TaskSteps::StepCount); - - std::vector AddedHashes; - std::vector RemovedHashes; - uint64_t RemovedSize = 0; - uint64_t AddedSize = 0; - - tsl::robin_map BaseRawHashLookup; - for (size_t PathIndex = 0; PathIndex < BaseFolderContent.RawHashes.size(); PathIndex++) - { - const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex]; - BaseRawHashLookup.insert_or_assign(RawHash, PathIndex); - } - tsl::robin_map CompareRawHashLookup; - for (size_t PathIndex = 0; PathIndex < CompareFolderContent.RawHashes.size(); PathIndex++) - { - const IoHash& RawHash = CompareFolderContent.RawHashes[PathIndex]; - if (!BaseRawHashLookup.contains(RawHash)) - { - AddedHashes.push_back(RawHash); - AddedSize += CompareFolderContent.RawSizes[PathIndex]; - } - CompareRawHashLookup.insert_or_assign(RawHash, PathIndex); - } - for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++) - { - const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex]; - if (!CompareRawHashLookup.contains(RawHash)) - { - RemovedHashes.push_back(RawHash); - RemovedSize += BaseFolderContent.RawSizes[PathIndex]; - } - } + } + + // Debugging knobs for file build storage - always 0 in shipped builds + const double DefaultLatency = 0; // .0010; + const double DefaultDelayPerKBSec = 0; // 0.00005; - uint64_t BaseTotalRawSize = 0; - for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++) + void WriteResultObject(const std::filesystem::path& Path, const CbObject& Response) + { + const MemoryView ResponseView = Response.GetView(); + if (ToLower(Path.extension().string()) == ".cbo") { - BaseTotalRawSize += BaseFolderContent.RawSizes[PathIndex]; + WriteFile(Path, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); } - - double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0; - - ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept", - RemovedHashes.size(), - NiceBytes(RemovedSize), - AddedHashes.size(), - NiceBytes(AddedSize), - BaseFolderContent.Paths.size() - RemovedHashes.size(), - NiceBytes(BaseTotalRawSize - RemovedSize), - KeptPercent); - - uint64_t CompareTotalRawSize = 0; - - uint64_t FoundChunkCount = 0; - uint64_t FoundChunkSize = 0; - uint64_t NewChunkCount = 0; - uint64_t NewChunkSize = 0; - const ChunkedContentLookup BaseFolderLookup = BuildChunkedContentLookup(BaseFolderContent); - for (uint32_t ChunkIndex = 0; ChunkIndex < CompareFolderContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++) + else { - const IoHash& ChunkHash = CompareFolderContent.ChunkedContent.ChunkHashes[ChunkIndex]; - if (BaseFolderLookup.ChunkHashToChunkIndex.contains(ChunkHash)) - { - FoundChunkCount++; - FoundChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - } - else - { - NewChunkCount++; - NewChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - } - CompareTotalRawSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(ResponseView, SB); + WriteFile(Path, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); } - - double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0; - double NewPercent = CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0; - - ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.", - FoundChunkCount, - NiceBytes(FoundChunkSize), - FoundPercent, - CompareFolderContent.ChunkedContent.ChunkHashes.size(), - NiceBytes(CompareTotalRawSize), - BaseFolderContent.ChunkedContent.ChunkHashes.size(), - NiceBytes(BaseTotalRawSize), - NewChunkCount, - NiceBytes(NewChunkSize), - NewPercent); - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); } } // namespace builds_impl -////////////////////////////////////////////////////////////////////////////////////////////////////// -// BuildsCommand - Option-adding helpers -// +////////////////////////////////////////////////////////////////////////// void -BuildsCommand::AddSystemOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddSystemOptions(cxxopts::Options& Ops) { - Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), ""); + Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(SystemRootDir), ""); Ops.add_option("", "", "use-sparse-files", "Enable use of sparse files when writing large files. Defaults to true.", - cxxopts::value(m_UseSparseFiles), + cxxopts::value(UseSparseFiles), ""); } void -BuildsCommand::AddCloudOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddCloudOptions(cxxopts::Options& Ops) { - m_AuthOptions.AddOptions(Ops); + AuthOptions.AddOptions(Ops); - Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), ""); - Ops.add_option("cloud build", - "", - "url", - "Cloud Builds host url (legacy - use --override-host)", - cxxopts::value(m_OverrideHost), - ""); - Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), ""); - Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), ""); + Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(OverrideHost), ""); + Ops.add_option("cloud build", "", "url", "Cloud Builds host url (legacy - use --override-host)", cxxopts::value(OverrideHost), ""); + Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(Url), ""); + Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(Host), ""); Ops.add_option("cloud build", "", "assume-http2", "Assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake", - cxxopts::value(m_AssumeHttp2), + cxxopts::value(AssumeHttp2), ""); Ops.add_option("cloud build", "", "verbose-http", "Enable verbose option for http client", - cxxopts::value(m_VerboseHttp), + cxxopts::value(VerboseHttp), ""); - Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), ""); - Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), ""); - Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(m_AllowRedirect), ""); + Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(Namespace), ""); + Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(Bucket), ""); + Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(AllowRedirect), ""); } void -BuildsCommand::AddFileOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddFileOptions(cxxopts::Options& Ops) { - Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(m_StoragePath), ""); + Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(StoragePath), ""); Ops.add_option("filestorage", "", "json-metadata", "Write build, part and block metadata as .json files in addition to .cb files", - cxxopts::value(m_WriteMetadataAsJson), + cxxopts::value(WriteMetadataAsJson), ""); } void -BuildsCommand::AddCacheOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddCacheOptions(cxxopts::Options& Ops) { - Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), ""); + Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(ZenCacheHost), ""); } void -BuildsCommand::AddOutputOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddOutputOptions(cxxopts::Options& Ops) { - Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), ""); - Ops.add_option("output", - "", - "log-progress", - "Write @progress style progress to output", - cxxopts::value(m_LogProgress), - ""); - Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), ""); - Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), ""); + Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(PlainProgress), ""); + Ops.add_option("output", "", "log-progress", "Write @progress style progress to output", cxxopts::value(LogProgress), ""); + Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(Verbose), ""); + Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(Quiet), ""); } void -BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddWorkerOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "boost-worker-count", "Increase the number of worker threads - may cause computer to be less responsive", - cxxopts::value(m_BoostWorkerCount), + cxxopts::value(BoostWorkerCount), ""); Ops.add_option("", @@ -2077,47 +301,47 @@ BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops) "boost-worker-memory", "Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to " "be less responsive due to high memory usage", - cxxopts::value(m_BoostWorkerMemory), + cxxopts::value(BoostWorkerMemory), ""); Ops.add_option("", "", "boost-workers", "Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive", - cxxopts::value(m_BoostWorkers), + cxxopts::value(BoostWorkers), ""); } void -BuildsCommand::AddZenFolderOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddZenFolderOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "zen-folder-path", - fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", builds_impl::ZenFolderName), - cxxopts::value(m_ZenFolderPath), + fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", ZenFolderName), + cxxopts::value(ZenFolderPath), ""); } void -BuildsCommand::AddChunkingCacheOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddChunkingCacheOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "chunking-cache-path", "Path to cache for chunking information of scanned files. Default is empty resulting in no caching", - cxxopts::value(m_ChunkingCachePath), + cxxopts::value(ChunkingCachePath), ""); } void -BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddWildcardOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "wildcard", "Windows style wildcard(s) (using * and ?) to match file paths to include, separated by ;", - cxxopts::value(m_IncludeWildcard), + cxxopts::value(IncludeWildcard), ""); Ops.add_option("", @@ -2125,46 +349,46 @@ BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops) "exclude-wildcard", "Windows style wildcard(s) (using * and ?) to match file paths to exclude, separated by ;. Applied after --wildcard " "include filter", - cxxopts::value(m_ExcludeWildcard), + cxxopts::value(ExcludeWildcard), ""); } void -BuildsCommand::AddExcludeFolderOption(cxxopts::Options& Ops) +BuildsConfiguration::AddExcludeFolderOption(cxxopts::Options& Ops) { Ops.add_option("", "", "exclude-folders", "Names of folders to exclude, separated by ;", - cxxopts::value(m_ExcludeFolders), + cxxopts::value(ExcludeFolders), ""); } void -BuildsCommand::AddExcludeExtensionsOption(cxxopts::Options& Ops) +BuildsConfiguration::AddExcludeExtensionsOption(cxxopts::Options& Ops) { Ops.add_option("", "", "exclude-extensions", "Extensions to exclude, separated by ;" "include filter", - cxxopts::value(m_ExcludeExtensions), + cxxopts::value(ExcludeExtensions), ""); } void -BuildsCommand::AddMultipartOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddMultipartOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "allow-multipart", "Allow large attachments to be transfered using multipart protocol. Defaults to true.", - cxxopts::value(m_AllowMultiparts), + cxxopts::value(AllowMultiparts), ""); } void -BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddPartialBlockRequestOptions(cxxopts::Options& Ops) { Ops.add_option("", "", @@ -2177,12 +401,12 @@ BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops) "allowed to host\n" " true = multiple partial block ranges requests per block allowed to zen cache and host\n" "Defaults to 'mixed'.", - cxxopts::value(m_AllowPartialBlockRequests), + cxxopts::value(AllowPartialBlockRequests), ""); } void -BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddAppendNewContentOptions(cxxopts::Options& Ops) { Ops.add_option("", "", @@ -2191,26 +415,26 @@ BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops) " false = the local content will be replaced by the remote content\n" " true = the remote data will be overlayed on top of local data\n" "Defaults to false.", - cxxopts::value(m_AppendNewContent), + cxxopts::value(AppendNewContent), ""); } BuildsCommand::BuildsCommand() -: m_ListNamespacesSubCmd(*this) -, m_ListSubCmd(*this) -, m_ListBlocksSubCmd(*this) -, m_UploadSubCmd(*this) -, m_DownloadSubCmd(*this) -, m_LsSubCmd(*this) -, m_DiffSubCmd(*this) -, m_FetchBlobSubCmd(*this) -, m_PrimeCacheSubCmd(*this) -, m_PauseSubCmd(*this) -, m_ResumeSubCmd(*this) -, m_AbortSubCmd(*this) -, m_ValidatePartSubCmd(*this) -, m_TestSubCmd(*this) -, m_MultiTestDownloadSubCmd(*this) +: m_ListNamespacesSubCmd(m_Configuration) +, m_ListSubCmd(m_Configuration) +, m_ListBlocksSubCmd(m_Configuration) +, m_UploadSubCmd(m_Configuration) +, m_DownloadSubCmd(m_Configuration) +, m_LsSubCmd(m_Configuration) +, m_DiffSubCmd(m_Configuration) +, m_FetchBlobSubCmd(m_Configuration) +, m_PrimeCacheSubCmd(m_Configuration) +, m_PauseSubCmd(m_Configuration) +, m_ResumeSubCmd(m_Configuration) +, m_AbortSubCmd(m_Configuration) +, m_ValidatePartSubCmd(m_Configuration) +, m_TestSubCmd(m_Configuration) +, m_MultiTestDownloadSubCmd(m_Configuration) { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("__hidden__", "", "subcommand", "", cxxopts::value(m_SubCommand)->default_value(""), ""); @@ -2246,158 +470,222 @@ BuildsCommand::OnParentOptionsParsed(const ZenCliOptions& /*GlobalOptions*/) #endif // ZEN_PLATFORM_WINDOWS // Validate output options - if (m_Verbose && m_Quiet) + if (m_Configuration.Verbose && m_Configuration.Quiet) { throw OptionParseException("'--verbose' conflicts with '--quiet'", {}); } - if (m_LogProgress && m_PlainProgress) + if (m_Configuration.LogProgress && m_Configuration.PlainProgress) { throw OptionParseException("'--plain-progress' conflicts with '--log-progress'", {}); } - if (m_LogProgress && m_Quiet) + if (m_Configuration.LogProgress && m_Configuration.Quiet) { throw OptionParseException("'--quiet' conflicts with '--log-progress'", {}); } - if (m_PlainProgress && m_Quiet) + if (m_Configuration.PlainProgress && m_Configuration.Quiet) { throw OptionParseException("'--quiet' conflicts with '--plain-progress'", {}); } - IsVerbose = m_Verbose; - IsQuiet = m_Quiet; - if (m_LogProgress) + if (m_Configuration.LogProgress) { - ProgressMode = ProgressBar::Mode::Log; + m_Configuration.ProgressMode = ConsoleProgressMode::Log; } - else if (m_PlainProgress) + else if (m_Configuration.PlainProgress) { - ProgressMode = ProgressBar::Mode::Plain; + m_Configuration.ProgressMode = ConsoleProgressMode::Plain; } - else if (IsQuiet) + else if (m_Configuration.Quiet) { - ProgressMode = ProgressBar::Mode::Quiet; + m_Configuration.ProgressMode = ConsoleProgressMode::Quiet; } else { - ProgressMode = ProgressBar::Mode::Pretty; + m_Configuration.ProgressMode = ConsoleProgressMode::Pretty; } - if (m_BoostWorkers) + if (m_Configuration.BoostWorkers) { - m_BoostWorkerCount = true; - m_BoostWorkerMemory = true; + m_Configuration.BoostWorkerCount = true; + m_Configuration.BoostWorkerMemory = true; } // Parse system options - if (m_SystemRootDir.empty()) + if (m_Configuration.SystemRootDir.empty()) { - m_SystemRootDir = PickDefaultSystemRootDirectory(); + m_Configuration.SystemRootDir = PickDefaultSystemRootDirectory(); } - MakeSafeAbsolutePathInPlace(m_SystemRootDir); - - UseSparseFiles = m_UseSparseFiles; + MakeSafeAbsolutePathInPlace(m_Configuration.SystemRootDir); + MakeSafeAbsolutePathInPlace(m_Configuration.ChunkingCachePath); return true; } +std::atomic& +BuildsSubCmdBase::AbortFlag() const +{ + return builds_impl::AbortFlag; +} + +std::atomic& +BuildsSubCmdBase::PauseFlag() const +{ + return builds_impl::PauseFlag; +} + +std::unique_ptr +BuildsSubCmdBase::CreateProgress() const +{ + return std::unique_ptr(CreateConsoleProgress(m_Config.ProgressMode)); +} + +////////////////////////////////////////////////////////////////////////// + +void +BuildsSubCmdBase::LogBanner() +{ + if (!m_Config.Quiet) + { + ZenCmdBase::LogExecutableVersionAndPid(); + } +} + +void +BuildsSubCmdBase::LogWorkersInfo(const TransferThreadWorkers& Workers) +{ + if (!m_Config.Quiet) + { + ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); + } +} + void -BuildsCommand::ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts) +BuildsSubCmdBase::CleanZenFolder() +{ + CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), AbortFlag(), PauseFlag(), GetZenFolderPath()); +} + +////////////////////////////////////////////////////////////////////////// + +BuildsSubCmdBase::ResolvedStorage +BuildsSubCmdBase::ParseStorageOptions(std::string& BuildId, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + const std::filesystem::path& SystemRootDirOverride, + const std::filesystem::path& StoragePathOverride) { - if (!m_Url.empty()) + ResolvedStorage Resolved{.SystemRootDir = SystemRootDirOverride.empty() ? m_Config.SystemRootDir : SystemRootDirOverride, + .Host = m_Config.Host, + .Namespace = m_Config.Namespace, + .Bucket = m_Config.Bucket, + .StoragePath = StoragePathOverride.empty() ? m_Config.StoragePath : StoragePathOverride}; + + if (!m_Config.Url.empty()) { - if (!m_Host.empty()) + if (!Resolved.Host.empty()) { - throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", m_Host, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", Resolved.Host, m_Config.Url), + SubOpts.help()); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", m_Bucket, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", Resolved.Bucket, m_Config.Url), + SubOpts.help()); } if (!BuildId.empty()) { - throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Config.Url), + SubOpts.help()); } - if (!ParseBuildStorageUrl(m_Url, m_Host, m_Namespace, m_Bucket, BuildId)) + if (!ParseBuildStorageUrl(m_Config.Url, Resolved.Host, Resolved.Namespace, Resolved.Bucket, BuildId)) { throw OptionParseException("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", SubOpts.help()); } } - if (!m_OverrideHost.empty() || !m_Host.empty()) + if (!m_Config.OverrideHost.empty() || !Resolved.Host.empty()) { - if (!m_StoragePath.empty()) + if (!Resolved.StoragePath.empty()) { throw OptionParseException( - fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", m_StoragePath), + fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", Resolved.StoragePath), SubOpts.help()); } - if (RequireNamespace && m_Namespace.empty()) + if (Options.RequireNamespace && Resolved.Namespace.empty()) { throw OptionParseException("'--namespace' is required", SubOpts.help()); } - if (RequireBucket && m_Bucket.empty()) + if (Options.RequireBucket && Resolved.Bucket.empty()) { throw OptionParseException("'--bucket' is required", SubOpts.help()); } } - else if (m_StoragePath.empty()) + else if (Resolved.StoragePath.empty()) { throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help()); } - MakeSafeAbsolutePathInPlace(m_StoragePath); + MakeSafeAbsolutePathInPlace(Resolved.StoragePath); + return Resolved; } StorageInstance -BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, - BuildStorageCache::Statistics& StorageCacheStats, - const std::filesystem::path& TempPath, - std::string& BuildId, - bool RequireNamespace, - bool RequireBucket, - bool BoostCacheBackgroundWorkerPool, - std::unique_ptr& Auth, - cxxopts::Options& SubOpts) +BuildsSubCmdBase::CreateBuildStorage(const std::filesystem::path& ZenFolderDefault, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + BuildStorageBase::Statistics& StorageStats, + BuildStorageCache::Statistics& StorageCacheStats, + std::unique_ptr& Auth, + const ResolvedStorage& Resolved) { using namespace builds_impl; - ParseStorageOptions(BuildId, RequireNamespace, RequireBucket, SubOpts); + ResolveZenFolderPath(ZenFolderDefault); + CreateDirectories(GetZenFolderPath()); + const std::filesystem::path TempPath = ZenTempFolderPath(GetZenFolderPath()); - HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", - .AssumeHttp2 = m_AssumeHttp2, - .AllowResume = true, - .RetryCount = 2, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}; + HttpClientSettings ClientSettings{ + .LogCategory = "httpbuildsclient", + .AssumeHttp2 = m_Config.AssumeHttp2, + .AllowResume = true, + .RetryCount = 2, + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)}; std::string StorageDescription; std::string CacheDescription; StorageInstance Result; - if (!m_Host.empty() || !m_OverrideHost.empty()) - { - m_AuthOptions.ParseOptions(SubOpts, - m_SystemRootDir, - ClientSettings, - m_Host.empty() ? m_OverrideHost : m_Host, - Auth, - IsQuiet, - /*Hidden*/ false, - m_Verbose); - - BuildStorageResolveResult ResolveRes = - ResolveBuildStorage(ConsoleLog(), ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All, m_Verbose); + if (!Resolved.Host.empty() || !m_Config.OverrideHost.empty()) + { + AuthCommandLineOptions AuthOpts = m_Config.AuthOptions; + AuthOpts.ParseOptions(SubOpts, + Resolved.SystemRootDir, + ClientSettings, + Resolved.Host.empty() ? m_Config.OverrideHost : Resolved.Host, + Auth, + m_Config.Quiet, + /*Hidden*/ false, + m_Config.Verbose); + + BuildStorageResolveResult ResolveRes = ResolveBuildStorage(ConsoleLog(), + ClientSettings, + Resolved.Host, + m_Config.OverrideHost, + m_Config.ZenCacheHost, + ZenCacheResolveMode::All, + m_Config.Verbose); if (!ResolveRes.Cloud.Address.empty()) { ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2; Result.BuildStorageHttp = - std::make_unique(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); }); + std::make_unique(ResolveRes.Cloud.Address, ClientSettings, [this]() { return AbortFlag().load(); }); Result.BuildStorage = CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, - m_Namespace, - m_Bucket, - m_AllowRedirect, + Resolved.Namespace, + Resolved.Bucket, + m_Config.AllowRedirect, TempPath / "storage"); Result.BuildStorageHost = ResolveRes.Cloud; @@ -2408,8 +696,8 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, ResolveRes.Cloud.Name, (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address), Result.BuildStorageHttp->GetSessionId(), - m_Namespace, - m_Bucket, + Resolved.Namespace, + Resolved.Bucket, NiceLatencyNs(HostLatencyNs)); if (!ResolveRes.Cache.Address.empty()) @@ -2423,17 +711,17 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2, .AllowResume = true, .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)}, + [this]() { return AbortFlag().load(); }); Result.CacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, - m_Namespace, - m_Bucket, + Resolved.Namespace, + Resolved.Bucket, TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); + Options.BoostCacheBackgroundWorkers ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); Result.CacheHost = ResolveRes.Cache; uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0; @@ -2445,69 +733,69 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, Result.CacheHttp->GetSessionId(), NiceLatencyNs(CacheLatencyNs)); - if (!m_Namespace.empty()) + if (!Resolved.Namespace.empty()) { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); + CacheDescription += fmt::format(". Namespace '{}'", Resolved.Namespace); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + CacheDescription += fmt::format(" Bucket '{}'", Resolved.Bucket); } } } } - else if (!m_StoragePath.empty()) + else if (!Resolved.StoragePath.empty()) { - StorageDescription = fmt::format("folder {}", m_StoragePath); - Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + StorageDescription = fmt::format("folder {}", Resolved.StoragePath); + Result.BuildStorage = CreateFileBuildStorage(Resolved.StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(), + Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = Resolved.StoragePath.generic_string(), .Name = "Disk", .LatencySec = 1.0 / 100000, // 1 us .Caps = {.MaxRangeCountPerRequest = 2048u}}; - if (!m_ZenCacheHost.empty()) + if (!m_Config.ZenCacheHost.empty()) { - ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp); + ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_Config.ZenCacheHost, m_Config.AssumeHttp2, m_Config.VerboseHttp); if (TestResult.Success) { Result.CacheHttp = std::make_unique( - m_ZenCacheHost, + m_Config.ZenCacheHost, HttpClientSettings{ .LogCategory = "httpcacheclient", .ConnectTimeout = std::chrono::milliseconds{3000}, .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = m_AssumeHttp2, + .AssumeHttp2 = m_Config.AssumeHttp2, .AllowResume = true, .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)}, + [this]() { return AbortFlag().load(); }); Result.CacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, - m_Namespace, - m_Bucket, + Resolved.Namespace, + Resolved.Bucket, TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); - Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_ZenCacheHost, - .Name = m_ZenCacheHost, - .AssumeHttp2 = m_AssumeHttp2, + Options.BoostCacheBackgroundWorkers ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); + Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_Config.ZenCacheHost, + .Name = m_Config.ZenCacheHost, + .AssumeHttp2 = m_Config.AssumeHttp2, .LatencySec = TestResult.LatencySeconds, .Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}}; CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId()); - if (!m_Namespace.empty()) + if (!Resolved.Namespace.empty()) { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); + CacheDescription += fmt::format(". Namespace '{}'", Resolved.Namespace); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + CacheDescription += fmt::format(" Bucket '{}'", Resolved.Bucket); } } } @@ -2517,7 +805,7 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help()); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Remote: {}", StorageDescription); if (!Result.CacheHost.Name.empty()) @@ -2529,7 +817,7 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, } Oid -BuildsCommand::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts) { if (BuildIdStr.length() != Oid::StringLength) { @@ -2548,7 +836,7 @@ BuildsCommand::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& Sub } Oid -BuildsCommand::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts) { if (BuildPartIdStr.length() != Oid::StringLength) { @@ -2567,7 +855,7 @@ BuildsCommand::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Opti } std::vector -BuildsCommand::ParseBuildPartIds(const std::vector& BuildPartIdStrs, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartIds(const std::vector& BuildPartIdStrs, cxxopts::Options& SubOpts) { std::vector BuildPartIds; for (const std::string& BuildPartId : BuildPartIdStrs) @@ -2582,7 +870,7 @@ BuildsCommand::ParseBuildPartIds(const std::vector& BuildPartIdStrs } std::vector -BuildsCommand::ParseBuildPartNames(const std::vector& BuildPartNameStrs, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartNames(const std::vector& BuildPartNameStrs, cxxopts::Options& SubOpts) { std::vector BuildPartNames; for (const std::string& BuildPartName : BuildPartNameStrs) @@ -2597,10 +885,10 @@ BuildsCommand::ParseBuildPartNames(const std::vector& BuildPartName } CbObject -BuildsCommand::ParseBuildMetadata(bool CreateBuild, - std::filesystem::path& BuildMetadataPath, - const std::string& BuildMetadata, - cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildMetadata(bool CreateBuild, + std::filesystem::path& BuildMetadataPath, + const std::string& BuildMetadata, + cxxopts::Options& SubOpts) { if (CreateBuild) { @@ -2659,7 +947,7 @@ BuildsCommand::ParseBuildMetadata(bool CreateBuild, } void -BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) { if (Path.empty()) { @@ -2669,7 +957,7 @@ BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) } IoHash -BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts) { if (BlobHashStr.empty()) { @@ -2693,7 +981,7 @@ BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& S } void -BuildsCommand::ParseFileFilters(std::vector& OutIncludeWildcards, std::vector& OutExcludeWildcards) +BuildsSubCmdBase::ParseFileFilters(std::vector& OutIncludeWildcards, std::vector& OutExcludeWildcards) { auto SplitAndAppendWildcard = [](const std::string_view Wildcard, std::vector& Output) { ForEachStrTok(Wildcard, ';', [&Output](std::string_view Wildcard) { @@ -2719,12 +1007,13 @@ BuildsCommand::ParseFileFilters(std::vector& OutIncludeWildcards, s }); }; - SplitAndAppendWildcard(m_IncludeWildcard, OutIncludeWildcards); - SplitAndAppendWildcard(m_ExcludeWildcard, OutExcludeWildcards); + SplitAndAppendWildcard(m_Config.IncludeWildcard, OutIncludeWildcards); + SplitAndAppendWildcard(m_Config.ExcludeWildcard, OutExcludeWildcards); } void -BuildsCommand::ParseExcludeFolderAndExtension(std::vector& OutExcludeFolders, std::vector& OutExcludeExtensions) +BuildsSubCmdBase::ParseExcludeFolderAndExtension(std::vector& OutExcludeFolders, + std::vector& OutExcludeExtensions) { auto SplitAndAppendExclusion = [](const std::string_view Input, std::vector& Output) { ForEachStrTok(Input, ";,", [&Output](std::string_view Exclusion) { @@ -2741,34 +1030,31 @@ BuildsCommand::ParseExcludeFolderAndExtension(std::vector& OutExclu }); }; - SplitAndAppendExclusion(m_ExcludeFolders, OutExcludeFolders); - SplitAndAppendExclusion(m_ExcludeExtensions, OutExcludeExtensions); + SplitAndAppendExclusion(m_Config.ExcludeFolders, OutExcludeFolders); + SplitAndAppendExclusion(m_Config.ExcludeExtensions, OutExcludeExtensions); } void -BuildsCommand::ResolveZenFolderPath(const std::filesystem::path& DefaultPath) +BuildsSubCmdBase::ResolveZenFolderPath(const std::filesystem::path& DefaultPath) { - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = DefaultPath; - } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + m_ResolvedZenFolderPath = m_Config.ZenFolderPath.empty() ? DefaultPath : m_Config.ZenFolderPath; + MakeSafeAbsolutePathInPlace(m_ResolvedZenFolderPath); } EPartialBlockRequestMode -BuildsCommand::ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts) { - EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests); + EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_Config.AllowPartialBlockRequests); if (Mode == EPartialBlockRequestMode::Invalid) { - throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests), + throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_Config.AllowPartialBlockRequests), SubOpts.help()); } return Mode; } void -BuildsCommand::ParseZenProcessId(int& ZenProcessId) +BuildsSubCmdBase::ParseZenProcessId(int& ZenProcessId) { if (ZenProcessId == -1) { @@ -2789,20 +1075,15 @@ BuildsCommand::ParseZenProcessId(int& ZenProcessId) ////////////////////////////////////////////////////////////////////////// -// --------------------------------------------------------------------------- -// Subcommand implementations -// --------------------------------------------------------------------------- - -BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("list-namespaces", "List all namespaces and optionally their buckets") -, m_Parent(Parent) +BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "list-namespaces", "List all namespaces and optionally their buckets") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "recursive", "Enable fetch of buckets within namespaces also", cxxopts::value(m_Recursive), ""); Opts.add_option("", "", @@ -2817,36 +1098,21 @@ BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent) void BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + cxxopts::Options& Opts = SubOptions(); - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ false, - /*RequireBucket*/ false, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + std::string DummyBuildId; + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {.RequireNamespace = false, .RequireBucket = false}, Opts); + StorageInstance Storage = + CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); CbObject Response = Storage.BuildStorage->ListNamespaces(m_Recursive); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); @@ -2858,29 +1124,20 @@ BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - std::filesystem::path ResultPath = MakeSafeAbsolutePath(m_ResultPath); - if (ToLower(ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(MakeSafeAbsolutePath(m_ResultPath), Response); } } -BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list", "List builds matching a query"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsListSubCmd::BuildsListSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "list", "List builds matching a query") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "query-path", @@ -2900,19 +1157,16 @@ BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list" void BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + if (!m_ResultPath.empty()) + { + LogBanner(); + } + + cxxopts::Options& Opts = SubOptions(); MakeSafeAbsolutePathInPlace(m_QueryPath); MakeSafeAbsolutePathInPlace(m_ResultPath); - if (!m_ResultPath.empty()) - { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - } std::string JsonQuery; if (m_QueryPath.empty()) { @@ -2949,25 +1203,14 @@ BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } } + std::string DummyBuildId; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ false, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {.RequireBucket = false}, Opts); + StorageInstance Storage = + CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); CbObject Response = Storage.BuildStorage->ListBuilds(JsonQuery); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); @@ -2979,28 +1222,20 @@ BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } } -BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("list-blocks", "List blocks for a build") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "list-blocks", "List blocks for a build") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); Opts.add_option("", "", @@ -3016,50 +1251,35 @@ BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent) void BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - MakeSafeAbsolutePathInPlace(m_ResultPath); - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } + cxxopts::Options& Opts = SubOptions(); + + MakeSafeAbsolutePathInPlace(m_ResultPath); + if (m_MaxCount == 0) { throw OptionParseException(fmt::format("'--max-count' ('{}') is invalid", m_MaxCount), Opts.help()); } BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + StorageInstance Storage = + CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); CbObject Response = Storage.BuildStorage->FindBlocks(BuildId, m_MaxCount); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); std::vector BlockDescriptions = ParseChunkBlockDescriptionList(Response); - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Response contains {} block", BlockDescriptions.size()); } @@ -3074,35 +1294,25 @@ BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } } -BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("upload", "Upload a folder to build storage") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "upload", "Upload a folder to build storage") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddExcludeFolderOption(Opts); - Parent.AddExcludeExtensionsOption(Opts); - Parent.AddChunkingCacheOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddExcludeFolderOption(Opts); + Config.AddExcludeExtensionsOption(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); Opts.add_option("", "", @@ -3150,7 +1360,7 @@ BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) cxxopts::value(m_UploadToZenCache), ""); - Parent.AddMultipartOptions(Opts); + Config.AddMultipartOptions(Opts); Opts.add_option("", "", @@ -3175,50 +1385,30 @@ BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) void BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - ZenState InstanceState; + ParsePath(m_Path, Opts); - m_Parent.ParsePath(m_Path, Opts); + builds_impl::ZenState InstanceState; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + StorageInstance Storage = + CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); if (m_BuildPartName.empty() && m_ManifestPath.empty()) { m_BuildPartName = m_Path.filename().string(); } - const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId(m_BuildId, Opts); if (m_BuildId.empty()) { m_BuildId = BuildId.ToString(); @@ -3227,29 +1417,31 @@ BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) Oid BuildPartId; if (!m_BuildPartId.empty()) { - BuildPartId = m_Parent.ParseBuildPartId(m_BuildPartId, Opts); + BuildPartId = ParseBuildPartId(m_BuildPartId, Opts); } - CbObject MetaData = m_Parent.ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts); + CbObject MetaData = ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts); - const std::filesystem::path TempDir = ZenTempFolderPath(m_Parent.GetZenFolderPath()); + const std::filesystem::path TempDir = ZenTempFolderPath(GetZenFolderPath()); std::vector ExcludeFolders = DefaultExcludeFolders; std::vector ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); std::unique_ptr ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - std::unique_ptr ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); - std::unique_ptr Progress(CreateConsoleProgress(ProgressMode)); + std::unique_ptr Progress = CreateProgress(); std::vector> UploadedParts = UploadFolder(ConsoleLog(), *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId, BuildPartId, m_BuildPartName, @@ -3261,68 +1453,79 @@ BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = TempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = m_CreateBuild, .IgnoreExistingBlocks = m_Clean, .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, .ExcludeFolders = ExcludeFolders, .ExcludeExtensions = ExcludeExtensions}); - if (!AbortFlag) + if (!AbortFlag()) { if (m_PostUploadVerify) { for (const auto& Part : UploadedParts) { - ValidateBuildPart(ConsoleLog(), *Progress, Workers, *Storage.BuildStorage, BuildId, Part.first, Part.second); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + BuildId, + Part.first, + Part.second); } } } - if (true) + if (!m_Config.Quiet) { - if (!IsQuiet) - { - ZEN_CONSOLE( - "{}:\n" - "Read: {}\n" - "Write: {}\n" - "Requests: {}\n" - "Avg Request Time: {}\n" - "Avg I/O Time: {}", - Storage.BuildStorageHost.Name, - NiceBytes(StorageStats.TotalBytesRead.load()), - NiceBytes(StorageStats.TotalBytesWritten.load()), - StorageStats.TotalRequestCount.load(), - StorageStats.TotalExecutionTimeUs.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0, - StorageStats.TotalRequestCount.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0); - } + const BuildStorageResolveResult::Host& Host = Storage.BuildStorageHost; + const BuildStorageBase::Statistics& Stats = StorageStats; + ZEN_CONSOLE( + "{}:\n" + "Read: {}\n" + "Write: {}\n" + "Requests: {}\n" + "Avg Request Time: {}\n" + "Avg I/O Time: {}", + Host.Name, + NiceBytes(Stats.TotalBytesRead.load()), + NiceBytes(Stats.TotalBytesWritten.load()), + Stats.TotalRequestCount.load(), + Stats.TotalExecutionTimeUs.load() > 0 + ? NiceTimeSpanMs(Stats.TotalExecutionTimeUs.load() / 1000 / Stats.TotalRequestCount.load()) + : 0, + Stats.TotalRequestCount.load() > 0 ? NiceTimeSpanMs(Stats.TotalRequestTimeUs.load() / 1000 / Stats.TotalRequestCount.load()) + : 0); } - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Upload aborted"); } } -BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("download", "Download a build to a local folder") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "download", "Download a build to a local folder") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddWildcardOptions(Opts); - Parent.AddAppendNewContentOptions(Opts); - Parent.AddExcludeFolderOption(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddWildcardOptions(Opts); + Config.AddAppendNewContentOptions(Opts); + Config.AddExcludeFolderOption(Opts); Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); @@ -3357,9 +1560,9 @@ BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) "Upload data downloaded from remote host to zen cache", cxxopts::value(m_UploadToZenCache), ""); - Parent.AddMultipartOptions(Opts); + Config.AddMultipartOptions(Opts); - Parent.AddPartialBlockRequestOptions(Opts); + Config.AddPartialBlockRequestOptions(Opts); Opts.add_option( "", @@ -3389,76 +1592,61 @@ BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) void BuildsDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - ZenState InstanceState; + cxxopts::Options& Opts = SubOptions(); - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); std::vector IncludeWildcards; std::vector ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); + ParseFileFilters(IncludeWildcards, ExcludeWildcards); - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); + builds_impl::ZenState InstanceState; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - std::unique_ptr Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + StorageInstance Storage = CreateBuildStorage(m_Path / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); - std::vector BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + std::vector BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(Opts); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); - if (m_Parent.m_AppendNewContent && m_Clean) + if (m_Config.AppendNewContent && m_Clean) { throw OptionParseException("'--append' conflicts with '--clean'", Opts.help()); } std::vector ExcludeFolders = DefaultExcludeFolders; std::vector ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); - std::unique_ptr Progress(CreateConsoleProgress(ProgressMode)); + std::unique_ptr Progress = CreateProgress(); DownloadFolder( ConsoleLog(), *Progress, Workers, Storage, - StorageCacheStats, + AbortFlag(), + PauseFlag(), + CacheStats, BuildId, BuildPartIds, BuildPartNames, m_DownloadSpecPath, m_Path, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, - .ZenFolderPath = m_Parent.GetZenFolderPath(), - .AllowMultiparts = m_Parent.m_AllowMultiparts, + DownloadOptions{.SystemRootDir = m_Config.SystemRootDir, + .ZenFolderPath = GetZenFolderPath(), + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = m_Clean, .PostDownloadVerify = m_PostDownloadVerify, @@ -3467,28 +1655,33 @@ BuildsDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) .AllowFileClone = m_AllowFileClone, .IncludeWildcards = IncludeWildcards, .ExcludeWildcards = ExcludeWildcards, - .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Parent.m_BoostWorkerMemory), + .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory), .PopulateCache = m_UploadToZenCache, - .AppendNewContent = m_Parent.m_AppendNewContent, + .AppendNewContent = m_Config.AppendNewContent, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles, .ExcludeFolders = ExcludeFolders}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Download aborted"); } } -BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "List files in a build"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsLsSubCmd::BuildsLsSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "ls", "List files in a build") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddWildcardOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddWildcardOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); Opts.add_option("", @@ -3526,43 +1719,31 @@ BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "Lis void BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } - ZenState InstanceState; + cxxopts::Options& Opts = SubOptions(); + + builds_impl::ZenState InstanceState; std::vector IncludeWildcards; std::vector ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); - - m_Parent.ResolveZenFolderPath(m_Parent.m_StoragePath); // ls uses storage path context + ParseFileFilters(IncludeWildcards, ExcludeWildcards); + // ls uses storage path context BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - std::unique_ptr Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + StorageInstance Storage = CreateBuildStorage(m_Config.StoragePath, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); - std::vector BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + std::vector BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); std::unique_ptr StructuredOutput; if (!m_ResultPath.empty()) @@ -3571,38 +1752,30 @@ BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) StructuredOutput = std::make_unique(); } - ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get()); + ListBuild(m_Config.Quiet, Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get()); if (StructuredOutput) { CbObject Response = StructuredOutput->Save(); - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("List build aborted"); } } -BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff", "Diff two local folders"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "diff", "Diff two local folders") { - auto& Opts = SubOptions(); - Parent.AddOutputOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddExcludeFolderOption(Opts); - Parent.AddExcludeExtensionsOption(Opts); - Parent.AddChunkingCacheOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddOutputOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddExcludeFolderOption(Opts); + Config.AddExcludeExtensionsOption(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); Opts.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), ""); Opts.add_option("", @@ -3618,38 +1791,28 @@ BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff" void BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); if (m_DiffPath.empty()) { throw OptionParseException("'--compare-path' is required", Opts.help()); } MakeSafeAbsolutePathInPlace(m_DiffPath); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - std::vector ExcludeFolders = DefaultExcludeFolders; std::vector ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); StandardChunkingControllerSettings ChunkingSettings; std::unique_ptr ChunkController = CreateStandardChunkingController(ChunkingSettings); - std::unique_ptr ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); if (m_OnlyChunked) { @@ -3661,24 +1824,37 @@ BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ChunkingSettings.SplitAndCompressExtensions.end()); } - DiffFolders(Workers, m_Path, m_DiffPath, *ChunkController, *ChunkCache, ExcludeFolders, ExcludeExtensions); - if (AbortFlag) + std::unique_ptr Progress = CreateProgress(); + + DiffFolders(*Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + Workers, + m_Path, + m_DiffPath, + *ChunkController, + *ChunkCache, + ExcludeFolders, + ExcludeExtensions); + if (AbortFlag()) { throw std::runtime_error("Diff folders aborted"); } } -BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("fetch-blob", "Fetch and validate a specific blob") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "fetch-blob", "Fetch and validate a specific blob") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); Opts.add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), ""); Opts.parse_positional({"build-id", "blob-hash"}); @@ -3688,68 +1864,47 @@ BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent) void BuildsFetchBlobSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + StorageInstance Storage = + CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - IoHash BlobHash = m_Parent.ParseBlobHash(m_BlobHash, Opts); - - const Oid BuildId = Oid::FromHexString(m_BuildId); + IoHash BlobHash = ParseBlobHash(m_BlobHash, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); uint64_t CompressedSize; uint64_t DecompressedSize; - ValidateBlob(AbortFlag, *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); - if (AbortFlag) + ValidateBlob(AbortFlag(), *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); + if (AbortFlag()) { throw std::runtime_error("Fetch blob aborted"); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", BlobHash, CompressedSize, DecompressedSize); } } -BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("prime-cache", "Prime the zen cache with build data") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "prime-cache", "Prime the zen cache with build data") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); Opts.add_option("", "", @@ -3777,47 +1932,33 @@ BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent) void BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ true, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); - - std::vector BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + StorageInstance Storage = CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, + {.BoostCacheBackgroundWorkers = true}, + Opts, + StorageStats, + CacheStats, + Auth, + Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); + + const Oid BuildId = ParseBuildId(m_BuildId, Opts); + + std::vector BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); + CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId, m_Config.Quiet); std::vector> AllBuildParts = ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); @@ -3829,101 +1970,101 @@ BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) AllBuildPartIds.push_back(BuildPart.first); } - ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache"); - - std::unique_ptr Progress(CreateConsoleProgress(ProgressMode)); + std::unique_ptr Progress = CreateProgress(); + Progress->SetLogOperationName("Prime Cache"); BuildsOperationPrimeCache PrimeOp(ConsoleLog(), *Progress, Storage, - AbortFlag, - PauseFlag, + AbortFlag(), + PauseFlag(), Workers.GetNetworkPool(), BuildId, AllBuildPartIds, - BuildsOperationPrimeCache::Options{.IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .ZenFolderPath = m_Parent.GetZenFolderPath(), + BuildsOperationPrimeCache::Options{.IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .ZenFolderPath = GetZenFolderPath(), .LargeAttachmentSize = PreferredMultipartChunkSize * 4u, .PreferredMultipartChunkSize = PreferredMultipartChunkSize, .ForceUpload = m_Force}, - StorageCacheStats); + CacheStats); PrimeOp.Execute(); - if (!IsQuiet) + if (!m_Config.Quiet && Storage.CacheStorage) { - if (Storage.CacheStorage) - { - ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", - StorageCacheStats.PutBlobCount.load(), - NiceBytes(StorageCacheStats.PutBlobByteCount), - Storage.CacheHost.Name); - } + ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", + CacheStats.PutBlobCount.load(), + NiceBytes(CacheStats.PutBlobByteCount), + Storage.CacheHost.Name); } } -BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("pause", "Pause a running zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +namespace { + void AddProcessIdOption(cxxopts::Options& Opts, int& ZenProcessId) + { + Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(ZenProcessId), ""); + Opts.parse_positional({"process-id"}); + Opts.positional_help("process-id"); + } +} // namespace + +BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "pause", "Pause a running zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), ""); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsPauseSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Pause.store(true); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(1); } -BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("resume", "Resume a paused zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "resume", "Resume a paused zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), ""); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsResumeSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Pause.store(false); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(0); } -BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("abort", "Abort a running zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "abort", "Abort a running zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), ""); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsAbortSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Abort.store(true); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Abort.store(1); } -BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("validate-part", "Validate a build part") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "validate-part", "Validate a build part") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); Opts.add_option("", "", @@ -3944,42 +2085,23 @@ BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent) void BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } - - ZenState InstanceState; + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + StorageInstance Storage = + CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); + builds_impl::ZenState InstanceState; - std::unique_ptr Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + Oid BuildId = ParseBuildId(m_BuildId, Opts); if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) { @@ -3988,33 +2110,45 @@ BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) Opts.help()); } - const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : m_Parent.ParseBuildPartId(m_BuildPartId, Opts); + const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(m_BuildPartId, Opts); - std::unique_ptr Progress(CreateConsoleProgress(ProgressMode)); + std::unique_ptr Progress = CreateProgress(); - ValidateBuildPart(ConsoleLog(), *Progress, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + BuildId, + BuildPartId, + m_BuildPartName); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Validate build part failed"); } } -BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test", "Run an upload/download test cycle"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsTestSubCmd::BuildsTestSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "test", "Run an upload/download test cycle") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); - Parent.AddMultipartOptions(Opts); - Parent.AddPartialBlockRequestOptions(Opts); - Parent.AddWildcardOptions(Opts); - Parent.AddAppendNewContentOptions(Opts); - Parent.AddChunkingCacheOptions(Opts); + Config.AddMultipartOptions(Opts); + Config.AddPartialBlockRequestOptions(Opts); + Config.AddWildcardOptions(Opts); + Config.AddAppendNewContentOptions(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "", "enable-scavenge", @@ -4034,38 +2168,35 @@ BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test" void BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(m_Parent.m_SystemRootDir); - CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); - auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); }); + m_TestSystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_TestSystemRootDir); + CleanDirectory(m_TestSystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); + auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_TestSystemRootDir); }); - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); - if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty()) + const bool CreatedTempStorage = m_Config.OverrideHost.empty() && m_Config.StoragePath.empty(); + if (CreatedTempStorage) { - m_Parent.m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.m_StoragePath); - CreateDirectories(m_Parent.m_StoragePath); - m_Parent.m_StoragePath = m_Parent.m_StoragePath.generic_string(); + m_TestStoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), m_TestStoragePath); + CreateDirectories(m_TestStoragePath); + m_TestStoragePath = m_TestStoragePath.generic_string(); } - auto StorageGuard = MakeGuard([this]() { - if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty()) + auto StorageGuard = MakeGuard([this, CreatedTempStorage]() { + if (CreatedTempStorage) { - DeleteDirectories(m_Parent.m_StoragePath); + DeleteDirectories(m_TestStoragePath); } }); - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(Opts); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); BuildStorageBase::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -4076,30 +2207,20 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2"); const std::filesystem::path DownloadPath3 = m_Path.parent_path() / (m_BuildPartName + "_test3"); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath2); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath3); - auto DownloadGuard = MakeGuard([&Workers, DownloadPath, DownloadPath2, DownloadPath3]() { - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3); + auto DownloadGuard = MakeGuard([this, &Workers, DownloadPath, DownloadPath2, DownloadPath3]() { + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath2); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath3); }); - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - std::unique_ptr Auth; std::string TestBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - TestBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + const ResolvedStorage Resolved = ParseStorageOptions(TestBuildId, {}, Opts, m_TestSystemRootDir, m_TestStoragePath); + StorageInstance Storage = CreateBuildStorage(m_Path / ZenFolderName, {}, Opts, StorageStats, StorageCacheStats, Auth, Resolved); m_BuildId = Oid::NewOid().ToString(); m_BuildPartId = Oid::NewOid().ToString(); @@ -4129,19 +2250,21 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView()); } - const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path); + const std::filesystem::path UploadTempDir = builds_impl::UploadTempDirectory(m_Path); std::unique_ptr ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - std::unique_ptr ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); - std::unique_ptr Progress(CreateConsoleProgress(ProgressMode)); + std::unique_ptr Progress = CreateProgress(); UploadFolder(ConsoleLog(), *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId, BuildPartId, m_BuildPartName, @@ -4153,12 +2276,14 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload build)"); } @@ -4170,6 +2295,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), Oid::NewOid(), Oid::NewOid(), m_BuildPartName, @@ -4181,42 +2308,56 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload again, chunking is cached)"); } } - ValidateBuildPart(ConsoleLog(), *Progress, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + BuildId, + BuildPartId, + m_BuildPartName); - if (!m_Parent.m_IncludeWildcard.empty() || !m_Parent.m_ExcludeWildcard.empty()) + if (!m_Config.IncludeWildcard.empty() || !m_Config.ExcludeWildcard.empty()) { - auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); }); + auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); }); ZEN_CONSOLE("\nDownload Filtered Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); std::vector IncludeWildcards; std::vector ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); + ParseFileFilters(IncludeWildcards, ExcludeWildcards); DownloadFolder(ConsoleLog(), *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, @@ -4225,8 +2366,11 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) .AllowFileClone = m_AllowFileClone, .IncludeWildcards = IncludeWildcards, .ExcludeWildcards = ExcludeWildcards, - .AppendNewContent = false}); - if (AbortFlag) + .AppendNewContent = false, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } @@ -4240,15 +2384,17 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, @@ -4257,8 +2403,11 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) .AllowFileClone = m_AllowFileClone, .IncludeWildcards = ExcludeWildcards, .ExcludeWildcards = IncludeWildcards, - .AppendNewContent = true}); - if (AbortFlag) + .AppendNewContent = true, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } @@ -4268,15 +2417,17 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, @@ -4285,8 +2436,11 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) .AllowFileClone = m_AllowFileClone, .IncludeWildcards = {}, .ExcludeWildcards = {}, - .AppendNewContent = false}); - if (AbortFlag) + .AppendNewContent = false, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } @@ -4297,6 +2451,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4304,16 +2460,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = false, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } @@ -4323,6 +2482,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4330,16 +2491,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download identical target)"); } @@ -4371,7 +2535,7 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) return true; }; - ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + ParallelWork Work(AbortFlag(), PauseFlag(), WorkerThreadPool::EMode::EnableBacklog); uint32_t Randomizer = 0; auto FileSizeIt = DownloadContent.FileSizes.begin(); @@ -4389,8 +2553,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { Work.ScheduleWork( Workers.GetIOWorkerPool(), - [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic&) { - if (!AbortFlag) + [this, SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic&) { + if (!AbortFlag()) { bool WasReadOnly = SetFileReadOnly(FilePath, false); { @@ -4431,7 +2595,7 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ZEN_UNUSED(IsAborted, IsPaused); ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork); }); - ZEN_ASSERT(!AbortFlag.load()); + ZEN_ASSERT(!AbortFlag().load()); ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }; @@ -4441,6 +2605,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4448,16 +2614,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download scrambled target)"); } @@ -4478,6 +2647,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId2, BuildPartId2, m_BuildPartName, @@ -4489,23 +2660,37 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload scrambled)"); } - ValidateBuildPart(ConsoleLog(), *Progress, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + BuildId, + BuildPartId, + m_BuildPartName); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); DownloadFolder(ConsoleLog(), *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4513,16 +2698,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } @@ -4532,22 +2720,27 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId2, {BuildPartId2}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download scrambled)"); } @@ -4557,22 +2750,27 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId2, {BuildPartId2}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download scrambled)"); } @@ -4582,22 +2780,27 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath2, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath2 / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } @@ -4607,38 +2810,44 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath3, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath3 / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } } -BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("multi-test-download", "Download multiple builds sequentially as a test") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "multi-test-download", "Download multiple builds sequentially as a test") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); Opts.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), ""); Opts.add_option("", @@ -4660,42 +2869,30 @@ BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Pare void BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(m_Parent.m_SystemRootDir); - CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); - auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); }); + cxxopts::Options& Opts = SubOptions(); - m_Parent.ParsePath(m_Path, Opts); + m_TestSystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_TestSystemRootDir); + CleanDirectory(m_TestSystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); + auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_TestSystemRootDir); }); - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); - - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(Opts); + ParsePath(m_Path, Opts); + std::string DummyBuildId; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {}, Opts, m_TestSystemRootDir); + StorageInstance Storage = CreateBuildStorage(m_Path / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - std::unique_ptr Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - std::unique_ptr Progress(CreateConsoleProgress(ProgressMode)); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); + + std::unique_ptr Progress = CreateProgress(); Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) @@ -4709,31 +2906,36 @@ BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, - StorageCacheStats, + AbortFlag(), + PauseFlag(), + CacheStats, BuildId, /*BuildPartIds,*/ {}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, m_Path, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, - .ZenFolderPath = m_Parent.GetZenFolderPath(), - .AllowMultiparts = m_Parent.m_AllowMultiparts, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, + .ZenFolderPath = GetZenFolderPath(), + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = BuildIdString == m_BuildIds.front(), .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = false, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Multitest aborted"); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("\n"); } } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index ef7500fd6..78d9fac33 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -2,65 +2,206 @@ #pragma once -#include "../authutils.h" -#include "../zen.h" - -#include -#include +#include #include #include #include -#include +#include + +#include "authutils.h" +#include "consoleprogress.h" namespace zen { -class BuildsCommand; +class ProgressBase; +class AuthMgr; + +struct CreateBuildStorageOptions +{ + bool RequireNamespace = true; + bool RequireBucket = true; + bool BoostCacheBackgroundWorkers = false; +}; + +////////////////////////////////////////////////////////////////////////// + +struct BuildsConfiguration +{ + std::filesystem::path SystemRootDir; + bool UseSparseFiles = true; + + bool PlainProgress = false; + bool LogProgress = false; + bool Verbose = false; + bool Quiet = false; + ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; + bool BoostWorkerCount = false; + bool BoostWorkerMemory = false; + bool BoostWorkers = false; + + std::string OverrideHost; + std::string Host; + std::string Url; + bool AssumeHttp2 = false; + bool VerboseHttp = false; + bool AllowRedirect = false; + std::string Namespace; + std::string Bucket; + + std::filesystem::path StoragePath; + bool WriteMetadataAsJson = false; + + std::string ZenCacheHost; + + AuthCommandLineOptions AuthOptions; + + std::string IncludeWildcard; + std::string ExcludeWildcard; + std::string ExcludeFolders; + std::string ExcludeExtensions; + + std::filesystem::path ChunkingCachePath; + + bool AllowMultiparts = true; + std::string AllowPartialBlockRequests = "true"; + + bool AppendNewContent = false; + + std::filesystem::path ZenFolderPath; + + void AddSystemOptions(cxxopts::Options& Ops); + void AddCloudOptions(cxxopts::Options& Ops); + void AddFileOptions(cxxopts::Options& Ops); + void AddCacheOptions(cxxopts::Options& Ops); + void AddOutputOptions(cxxopts::Options& Ops); + void AddWorkerOptions(cxxopts::Options& Ops); + void AddZenFolderOptions(cxxopts::Options& Ops); + void AddChunkingCacheOptions(cxxopts::Options& Ops); + void AddWildcardOptions(cxxopts::Options& Ops); + void AddExcludeFolderOption(cxxopts::Options& Ops); + void AddExcludeExtensionsOption(cxxopts::Options& Ops); + void AddMultipartOptions(cxxopts::Options& Ops); + void AddPartialBlockRequestOptions(cxxopts::Options& Ops); + void AddAppendNewContentOptions(cxxopts::Options& Ops); +}; + +////////////////////////////////////////////////////////////////////////// + +class BuildsSubCmdBase : public ZenSubCmdBase +{ +public: + BuildsSubCmdBase(BuildsConfiguration& Config, std::string_view Name, std::string_view Description) + : ZenSubCmdBase(Name, Description) + , m_Config(Config) + { + } + +protected: + const BuildsConfiguration& m_Config; + std::filesystem::path m_ResolvedZenFolderPath; + + struct ResolvedStorage + { + std::filesystem::path SystemRootDir; + std::string Host; + std::string Namespace; + std::string Bucket; + std::filesystem::path StoragePath; + }; + + void LogBanner(); + void LogWorkersInfo(const TransferThreadWorkers& Workers); + + // SystemRootDirOverride / StoragePathOverride: empty = fall back to m_Config. + ResolvedStorage ParseStorageOptions(std::string& BuildId, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + const std::filesystem::path& SystemRootDirOverride = {}, + const std::filesystem::path& StoragePathOverride = {}); + + // Resolves the zen folder, creates it, and builds the storage instance using the supplied Resolved values. + // Caller owns Stats/Auth lifetime. Stats must outlive the returned StorageInstance. + StorageInstance CreateBuildStorage(const std::filesystem::path& ZenFolderDefault, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + BuildStorageBase::Statistics& OutStorageStats, + BuildStorageCache::Statistics& OutCacheStats, + std::unique_ptr& OutAuth, + const ResolvedStorage& Resolved); + Oid ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts); + Oid ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts); + std::vector ParseBuildPartIds(const std::vector& BuildPartIdStrs, cxxopts::Options& SubOpts); + std::vector ParseBuildPartNames(const std::vector& BuildPartNameStrs, cxxopts::Options& SubOpts); + CbObject ParseBuildMetadata(bool CreateBuild, + std::filesystem::path& BuildMetadataPath, + const std::string& BuildMetadata, + cxxopts::Options& SubOpts); + void ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts); + IoHash ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts); + EPartialBlockRequestMode ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts); + void ParseZenProcessId(int& ZenProcessId); + void ParseFileFilters(std::vector& OutIncludeWildcards, std::vector& OutExcludeWildcards); + void ParseExcludeFolderAndExtension(std::vector& OutExcludeFolders, std::vector& OutExcludeExtensions); + + void ResolveZenFolderPath(const std::filesystem::path& DefaultPath); + const std::filesystem::path& GetZenFolderPath() const { return m_ResolvedZenFolderPath; } + + std::atomic& AbortFlag() const; + std::atomic& PauseFlag() const; + std::unique_ptr CreateProgress() const; + + void CleanZenFolder(); +}; + +////////////////////////////////////////////////////////////////////////// -class BuildsListNamespacesSubCmd : public ZenSubCmdBase +class BuildsListNamespacesSubCmd : public BuildsSubCmdBase { public: - explicit BuildsListNamespacesSubCmd(BuildsCommand& Parent); + explicit BuildsListNamespacesSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; bool m_Recursive = false; std::filesystem::path m_ResultPath; }; -class BuildsListSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsListSubCmd : public BuildsSubCmdBase { public: - explicit BuildsListSubCmd(BuildsCommand& Parent); + explicit BuildsListSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::filesystem::path m_QueryPath; std::filesystem::path m_ResultPath; }; -class BuildsListBlocksSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsListBlocksSubCmd : public BuildsSubCmdBase { public: - explicit BuildsListBlocksSubCmd(BuildsCommand& Parent); + explicit BuildsListBlocksSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::string m_BuildId; std::filesystem::path m_ResultPath; uint32_t m_MaxCount = 16; }; -class BuildsUploadSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsUploadSubCmd : public BuildsSubCmdBase { public: - explicit BuildsUploadSubCmd(BuildsCommand& Parent); + explicit BuildsUploadSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::filesystem::path m_Path; std::string m_BuildId; std::string m_BuildPartId; @@ -76,14 +217,15 @@ private: bool m_UploadToZenCache = true; }; -class BuildsDownloadSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsDownloadSubCmd : public BuildsSubCmdBase { public: - explicit BuildsDownloadSubCmd(BuildsCommand& Parent); + explicit BuildsDownloadSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::filesystem::path m_Path; std::string m_BuildId; std::vector m_BuildPartIds; @@ -97,113 +239,126 @@ private: bool m_AllowFileClone = true; }; -class BuildsLsSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsLsSubCmd : public BuildsSubCmdBase { public: - explicit BuildsLsSubCmd(BuildsCommand& Parent); + explicit BuildsLsSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::string m_BuildId; std::vector m_BuildPartIds; std::vector m_BuildPartNames; std::filesystem::path m_ResultPath; }; -class BuildsDiffSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsDiffSubCmd : public BuildsSubCmdBase { public: - explicit BuildsDiffSubCmd(BuildsCommand& Parent); + explicit BuildsDiffSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::filesystem::path m_Path; std::filesystem::path m_DiffPath; bool m_OnlyChunked = false; }; -class BuildsFetchBlobSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsFetchBlobSubCmd : public BuildsSubCmdBase { public: - explicit BuildsFetchBlobSubCmd(BuildsCommand& Parent); + explicit BuildsFetchBlobSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - std::string m_BuildId; - std::string m_BlobHash; + std::string m_BuildId; + std::string m_BlobHash; }; -class BuildsPrimeCacheSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsPrimeCacheSubCmd : public BuildsSubCmdBase { public: - explicit BuildsPrimeCacheSubCmd(BuildsCommand& Parent); + explicit BuildsPrimeCacheSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; std::string m_BuildId; std::vector m_BuildPartIds; std::vector m_BuildPartNames; bool m_Force = false; }; -class BuildsPauseSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsPauseSubCmd : public BuildsSubCmdBase { public: - explicit BuildsPauseSubCmd(BuildsCommand& Parent); + explicit BuildsPauseSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - int m_ZenProcessId = -1; + int m_ZenProcessId = -1; }; -class BuildsResumeSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsResumeSubCmd : public BuildsSubCmdBase { public: - explicit BuildsResumeSubCmd(BuildsCommand& Parent); + explicit BuildsResumeSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - int m_ZenProcessId = -1; + int m_ZenProcessId = -1; }; -class BuildsAbortSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsAbortSubCmd : public BuildsSubCmdBase { public: - explicit BuildsAbortSubCmd(BuildsCommand& Parent); + explicit BuildsAbortSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - int m_ZenProcessId = -1; + int m_ZenProcessId = -1; }; -class BuildsValidatePartSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsValidatePartSubCmd : public BuildsSubCmdBase { public: - explicit BuildsValidatePartSubCmd(BuildsCommand& Parent); + explicit BuildsValidatePartSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; - std::string m_BuildId; - std::string m_BuildPartId; - std::string m_BuildPartName; + std::string m_BuildId; + std::string m_BuildPartId; + std::string m_BuildPartName; }; -class BuildsTestSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsTestSubCmd : public BuildsSubCmdBase { public: - explicit BuildsTestSubCmd(BuildsCommand& Parent); + explicit BuildsTestSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; + // Fixture-only overrides of SystemRootDir/StoragePath; passed to ParseStorageOptions explicitly. + std::filesystem::path m_TestSystemRootDir; + std::filesystem::path m_TestStoragePath; + std::filesystem::path m_Path; std::string m_BuildPartName; std::string m_BuildId; @@ -216,21 +371,27 @@ private: bool m_AllowFileClone = true; }; -class BuildsMultiTestDownloadSubCmd : public ZenSubCmdBase +////////////////////////////////////////////////////////////////////////// + +class BuildsMultiTestDownloadSubCmd : public BuildsSubCmdBase { public: - explicit BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent); + explicit BuildsMultiTestDownloadSubCmd(BuildsConfiguration& Config); void Run(const ZenCliOptions& GlobalOptions) override; private: - BuildsCommand& m_Parent; + // Fixture-only override of SystemRootDir; passed to CreateStorage explicitly. + std::filesystem::path m_TestSystemRootDir; + std::filesystem::path m_Path; std::vector m_BuildIds; bool m_EnableScavenging = true; bool m_AllowFileClone = true; }; -class BuildsCommand : public CacheStoreCmdWithSubCommands +////////////////////////////////////////////////////////////////////////// + +class BuildsCommand : public ZenCmdWithSubCommands { public: static constexpr char Name[] = "builds"; @@ -242,99 +403,14 @@ public: cxxopts::Options& Options() override { return m_Options; } - // Option-adding helpers (called by subcommand constructors) - void AddSystemOptions(cxxopts::Options& Ops); - void AddCloudOptions(cxxopts::Options& Ops); - void AddFileOptions(cxxopts::Options& Ops); - void AddCacheOptions(cxxopts::Options& Ops); - void AddOutputOptions(cxxopts::Options& Ops); - void AddWorkerOptions(cxxopts::Options& Ops); - void AddZenFolderOptions(cxxopts::Options& Ops); - void AddChunkingCacheOptions(cxxopts::Options& Ops); - void AddWildcardOptions(cxxopts::Options& Ops); - void AddExcludeFolderOption(cxxopts::Options& Ops); - void AddExcludeExtensionsOption(cxxopts::Options& Ops); - void AddMultipartOptions(cxxopts::Options& Ops); - void AddPartialBlockRequestOptions(cxxopts::Options& Ops); - void AddAppendNewContentOptions(cxxopts::Options& Ops); - - // Shared parsing/factory methods used by subcommand Run() implementations - void ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts); - StorageInstance CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, - BuildStorageCache::Statistics& StorageCacheStats, - const std::filesystem::path& TempPath, - std::string& BuildId, - bool RequireNamespace, - bool RequireBucket, - bool BoostCacheBackgroundWorkerPool, - std::unique_ptr& Auth, - cxxopts::Options& SubOpts); - Oid ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts); - Oid ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts); - std::vector ParseBuildPartIds(const std::vector& BuildPartIdStrs, cxxopts::Options& SubOpts); - std::vector ParseBuildPartNames(const std::vector& BuildPartNameStrs, cxxopts::Options& SubOpts); - CbObject ParseBuildMetadata(bool CreateBuild, - std::filesystem::path& BuildMetadataPath, - const std::string& BuildMetadata, - cxxopts::Options& SubOpts); - void ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts); - IoHash ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts); - EPartialBlockRequestMode ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts); - void ParseZenProcessId(int& ZenProcessId); - void ParseFileFilters(std::vector& OutIncludeWildcards, std::vector& OutExcludeWildcards); - void ParseExcludeFolderAndExtension(std::vector& OutExcludeFolders, std::vector& OutExcludeExtensions); - - void ResolveZenFolderPath(const std::filesystem::path& DefaultPath); - const std::filesystem::path& GetZenFolderPath() const { return m_ZenFolderPath; } - - cxxopts::Options m_Options{Name, Description}; - std::string m_SubCommand; - - // Shared state populated by AddXxxOptions helpers (bound via cxxopts::value references) - std::filesystem::path m_SystemRootDir; - bool m_UseSparseFiles = true; - - bool m_PlainProgress = false; - bool m_LogProgress = false; - bool m_Verbose = false; - bool m_Quiet = false; - bool m_BoostWorkerCount = false; - bool m_BoostWorkerMemory = false; - bool m_BoostWorkers = false; - - // cloud builds - std::string m_OverrideHost; - std::string m_Host; - std::string m_Url; - bool m_AssumeHttp2 = false; - bool m_VerboseHttp = false; - bool m_AllowRedirect = false; - std::string m_Namespace; - std::string m_Bucket; - - std::filesystem::path m_StoragePath; - bool m_WriteMetadataAsJson = false; - - std::string m_ZenCacheHost; - - AuthCommandLineOptions m_AuthOptions; - - std::string m_IncludeWildcard; - std::string m_ExcludeWildcard; - std::string m_ExcludeFolders; - std::string m_ExcludeExtensions; - - std::filesystem::path m_ChunkingCachePath; - - bool m_AllowMultiparts = true; - std::string m_AllowPartialBlockRequests = "true"; - - bool m_AppendNewContent = false; + BuildsConfiguration& GetConfiguration() { return m_Configuration; } + const BuildsConfiguration& GetConfiguration() const { return m_Configuration; } private: - std::filesystem::path m_ZenFolderPath; + cxxopts::Options m_Options{Name, Description}; + std::string m_SubCommand; + BuildsConfiguration m_Configuration; -protected: BuildsListNamespacesSubCmd m_ListNamespacesSubCmd; BuildsListSubCmd m_ListSubCmd; BuildsListBlocksSubCmd m_ListBlocksSubCmd; diff --git a/src/zen/cmds/exec_cmd.cpp b/src/zen/cmds/exec_cmd.cpp index dab53f13c..6cebfa430 100644 --- a/src/zen/cmds/exec_cmd.cpp +++ b/src/zen/cmds/exec_cmd.cpp @@ -23,7 +23,7 @@ #include #include -#include "../progressbar.h" +#include "consoleprogress.h" #include #include @@ -133,7 +133,7 @@ struct ExecSessionConfig bool Quiet = false; bool DumpActions = false; bool Binary = false; - ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; + ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; }; ////////////////////////////////////////////////////////////////////////// @@ -903,9 +903,11 @@ ExecSessionRunner::Run() std::atomic SubmittedWorkItems{0}; size_t TotalWorkItems = RemainingWorkItems.load(); - ProgressBar SubmitProgress(m_Config.ProgressMode, "Submit"); - SubmitProgress.UpdateState({.Task = "Submitting work items", .TotalCount = TotalWorkItems, .RemainingCount = RemainingWorkItems.load()}, - false); + std::unique_ptr ProgressOwner(CreateConsoleProgress(m_Config.ProgressMode)); + std::unique_ptr SubmitProgress = ProgressOwner->CreateProgressBar("Submit"); + SubmitProgress->UpdateState( + {.Task = "Submitting work items", .TotalCount = TotalWorkItems, .RemainingCount = RemainingWorkItems.load()}, + false); int OffsetCounter = m_Config.Offset; int StrideCounter = m_Config.Stride; @@ -1012,11 +1014,11 @@ ExecSessionRunner::Run() size_t Remaining = --RemainingWorkItems; int Submitted = ++SubmittedWorkItems; - SubmitProgress.UpdateState({.Task = "Submitting work items", - .Details = fmt::format("#{} LSN {}", Submitted, LsnField), - .TotalCount = TotalWorkItems, - .RemainingCount = Remaining}, - false); + SubmitProgress->UpdateState({.Task = "Submitting work items", + .Details = fmt::format("#{} LSN {}", Submitted, LsnField), + .TotalCount = TotalWorkItems, + .RemainingCount = Remaining}, + false); if (!m_Config.OutputPath.empty()) { @@ -1056,30 +1058,31 @@ ExecSessionRunner::Run() }, TargetParallelism); - SubmitProgress.Finish(); + SubmitProgress->Finish(); // Wait until all pending work is complete size_t TotalPendingJobs = m_PendingJobs.GetSize(); - ProgressBar CompletionProgress(m_Config.ProgressMode, "Execute"); + std::unique_ptr CompletionProgress = ProgressOwner->CreateProgressBar("Execute"); while (!m_PendingJobs.IsEmpty()) { size_t PendingCount = m_PendingJobs.GetSize(); - CompletionProgress.UpdateState({.Task = "Executing work items", - .Details = fmt::format("{} completed, {} remaining", TotalPendingJobs - PendingCount, PendingCount), - .TotalCount = TotalPendingJobs, - .RemainingCount = PendingCount}, - false); + CompletionProgress->UpdateState( + {.Task = "Executing work items", + .Details = fmt::format("{} completed, {} remaining", TotalPendingJobs - PendingCount, PendingCount), + .TotalCount = TotalPendingJobs, + .RemainingCount = PendingCount}, + false); - zen::Sleep(GetUpdateDelayMS(m_Config.ProgressMode)); + zen::Sleep(ProgressOwner->GetProgressUpdateDelayMS()); DrainCompletedJobs(); SendOrchestratorHeartbeat(); } - CompletionProgress.Finish(); + CompletionProgress->Finish(); // Write summary files @@ -1438,10 +1441,10 @@ ExecCommand::OnParentOptionsParsed(const ZenCliOptions& GlobalOptions) int ExecCommand::RunSession(zen::compute::ComputeServiceSession& ComputeSession, std::string_view OrchestratorUrl) { - ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; + ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; if (m_QuietLogging) { - ProgressMode = ProgressBar::Mode::Quiet; + ProgressMode = ConsoleProgressMode::Quiet; } ExecSessionConfig Config{ diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 7f94bf2df..bad2728ef 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -21,16 +21,18 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include -#include "../progressbar.h" +#include "consoleprogress.h" ZEN_THIRD_PARTY_INCLUDES_START #include @@ -131,13 +133,16 @@ namespace projectstore_impl { throw std::runtime_error(fmt::format("invalid job id returned, received '{}'", JobIdText)); } - ProgressBar ProgressBar(PlainProgress ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty, ""sv); + std::unique_ptr ProgressOwner( + CreateConsoleProgress(PlainProgress ? ConsoleProgressMode::Plain : ConsoleProgressMode::Pretty)); + std::unique_ptr Bar = ProgressOwner->CreateProgressBar(""sv); + std::string ActiveTask; - auto OuputMessages = [&](CbObjectView StatusObject) { + auto OutputMessages = [&](CbObjectView StatusObject) { CbArrayView Messages = StatusObject["Messages"sv].AsArrayView(); if (Messages.Num() > 0) { - ProgressBar.ForceLinebreak(); + Bar->ForceLinebreak(); for (auto M : Messages) { std::string_view Message = M.AsString(); @@ -169,33 +174,36 @@ namespace projectstore_impl { uint64_t RemainingCount = StatusObject["RemainingCount"sv].AsUInt64(); uint64_t ProgressElapsedTimeMs = StatusObject["ProgressElapsedTimeMs"sv].AsUInt64((uint64_t)-1); - if (!ProgressBar.IsSameTask(CurrentOp)) + if (ActiveTask != CurrentOp) { - ProgressBar.Finish(); + Bar->Finish(); + ActiveTask = ""; } - if (!ProgressBar.HasActiveTask()) + if (ActiveTask.empty()) { - OuputMessages(StatusObject); + OutputMessages(StatusObject); MessagesDone = true; + ActiveTask = std::string(CurrentOp); } - ProgressBar.UpdateState({.Task = std::string(CurrentOp), - .Details = std::string(CurrentOpDetails), - .TotalCount = TotalCount, - .RemainingCount = RemainingCount, - .OptionalElapsedTime = ProgressElapsedTimeMs}, - false); + Bar->UpdateState({.Task = std::string(CurrentOp), + .Details = std::string(CurrentOpDetails), + .TotalCount = TotalCount, + .RemainingCount = RemainingCount, + .OptionalElapsedTime = ProgressElapsedTimeMs}, + false); } if ((Status == "Complete") || (Status == "Aborted")) { - ProgressBar.Finish(); + Bar->Finish(); + ActiveTask = ""; } if (!MessagesDone) { - OuputMessages(StatusObject); + OutputMessages(StatusObject); } if (Status == "Complete") @@ -246,13 +254,13 @@ namespace projectstore_impl { #endif // ZEN_PLATFORM_WINDOWS if (HttpClient::Response DeleteResult = Http.Delete(fmt::format("/admin/jobs/{}", JobId))) { - ProgressBar.ForceLinebreak(); + Bar->ForceLinebreak(); ZEN_CONSOLE("Requested cancel..."); Cancelled = true; } else { - ProgressBar.ForceLinebreak(); + Bar->ForceLinebreak(); ZEN_CONSOLE("Failed cancelling job {}", DeleteResult); } continue; @@ -2196,17 +2204,18 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg ZEN_CONSOLE("Fetched oplog in {}", NiceTimeSpanMs(uint64_t(Response.ElapsedSeconds * 1000.0))); if (CbObject ResponseObject = Response.AsObject()) { - std::unique_ptr EmitProgressBar; + std::unique_ptr ProgressOwner2(CreateConsoleProgress(ConsoleProgressMode::Pretty)); + std::unique_ptr EmitProgressBar; { - ProgressBar ParseProgressBar(ProgressBar::Mode::Pretty, ""); - CbArrayView Entries = ResponseObject["entries"sv].AsArrayView(); - uint64_t Remaining = Entries.Num(); + std::unique_ptr ParseProgressBar = ProgressOwner2->CreateProgressBar(""); + CbArrayView Entries = ResponseObject["entries"sv].AsArrayView(); + uint64_t Remaining = Entries.Num(); for (auto EntryIter : Entries) { if (!AbortFlag) { CbObjectView Entry = EntryIter.AsObjectView(); - ParseProgressBar.UpdateState( + ParseProgressBar->UpdateState( {.Task = "Parsing oplog", .Details = "", .TotalCount = Entries.Num(), .RemainingCount = Remaining}, false); Remaining--; @@ -2219,7 +2228,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg } if (!EmitProgressBar) { - EmitProgressBar = std::make_unique(ProgressBar::Mode::Pretty, ""sv); + EmitProgressBar = ProgressOwner2->CreateProgressBar(""sv); WriteStopWatch.Reset(); } @@ -2229,7 +2238,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg ++OplogEntryCount; } } - ParseProgressBar.Finish(); + ParseProgressBar->Finish(); } WorkRemaining.CountDown(); @@ -2472,7 +2481,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a }; ParseSystemOptions(); - ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; + ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; auto ParseOutputOptions = [&]() { if (m_Verbose && m_Quiet) @@ -2494,19 +2503,19 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a if (m_LogProgress) { - ProgressMode = ProgressBar::Mode::Log; + ProgressMode = ConsoleProgressMode::Log; } else if (m_PlainProgress) { - ProgressMode = ProgressBar::Mode::Plain; + ProgressMode = ConsoleProgressMode::Plain; } else if (m_Quiet) { - ProgressMode = ProgressBar::Mode::Quiet; + ProgressMode = ConsoleProgressMode::Quiet; } else { - ProgressMode = ProgressBar::Mode::Pretty; + ProgressMode = ConsoleProgressMode::Pretty; } }; ParseOutputOptions(); diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp index c027f0d67..713c5e386 100644 --- a/src/zen/cmds/wipe_cmd.cpp +++ b/src/zen/cmds/wipe_cmd.cpp @@ -12,7 +12,7 @@ #include #include -#include "../progressbar.h" +#include "consoleprogress.h" #include @@ -35,13 +35,13 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { namespace wipe_impl { - static std::atomic AbortFlag = false; - static std::atomic PauseFlag = false; - static bool IsVerbose = false; - static bool Quiet = false; - static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; - const bool SingleThreaded = false; - bool BoostWorkerThreads = true; + static std::atomic AbortFlag = false; + static std::atomic PauseFlag = false; + static bool IsVerbose = false; + static bool Quiet = false; + static ConsoleProgressMode ProgressMode = ConsoleProgressMode::Pretty; + const bool SingleThreaded = false; + bool BoostWorkerThreads = true; WorkerThreadPool& GetIOWorkerPool() { @@ -168,7 +168,8 @@ namespace wipe_impl { ZEN_TRACE_CPU("CleanDirectory"); Stopwatch Timer; - ProgressBar Progress(ProgressMode, "Clean Folder"); + std::unique_ptr ProgressOwner(CreateConsoleProgress(ProgressMode)); + std::unique_ptr Progress = ProgressOwner->CreateProgressBar("Clean Folder"); std::atomic CleanWipe = true; std::atomic DiscoveredItemCount = 0; @@ -414,7 +415,7 @@ namespace wipe_impl { GetIOWorkerPool(), Work.PendingWork()); - Work.Wait(ProgressMode == ProgressBar::Mode::Pretty ? 200 : 5000, [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) { + Work.Wait(ProgressOwner->GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) { ZEN_UNUSED(PendingWork); if (Quiet) { @@ -425,12 +426,12 @@ namespace wipe_impl { uint64_t Deleted = DeletedItemCount.load(); uint64_t DeletedBytes = DeletedByteCount.load(); uint64_t Discovered = DiscoveredItemCount.load(); - Progress.UpdateState({.Task = "Removing files ", - .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), - .TotalCount = Discovered, - .RemainingCount = Discovered - Deleted, - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); + Progress->UpdateState({.Task = "Removing files ", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = Discovered, + .RemainingCount = Discovered - Deleted, + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); }); std::vector DirectoriesToDelete; @@ -474,22 +475,22 @@ namespace wipe_impl { } uint64_t NowMs = Timer.GetElapsedTimeMs(); - if ((NowMs - LastUpdateTimeMs) >= GetUpdateDelayMS(ProgressMode)) + if ((NowMs - LastUpdateTimeMs) >= ProgressOwner->GetProgressUpdateDelayMS()) { LastUpdateTimeMs = NowMs; uint64_t Deleted = DeletedItemCount.load(); uint64_t DeletedBytes = DeletedByteCount.load(); uint64_t Discovered = DiscoveredItemCount.load(); - Progress.UpdateState({.Task = "Removing folders", - .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), - .TotalCount = DirectoriesToDelete.size(), - .RemainingCount = DirectoriesToDelete.size() - SubDirectoryIndex}, - false); + Progress->UpdateState({.Task = "Removing folders", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = DirectoriesToDelete.size(), + .RemainingCount = DirectoriesToDelete.size() - SubDirectoryIndex}, + false); } } - Progress.Finish(); + Progress->Finish(); uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); if (!Quiet) @@ -549,7 +550,7 @@ WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Quiet = m_Quiet; IsVerbose = m_Verbose; - ProgressMode = m_PlainProgress ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty; + ProgressMode = m_PlainProgress ? ConsoleProgressMode::Plain : ConsoleProgressMode::Pretty; BoostWorkerThreads = m_BoostWorkerThreads; MakeSafeAbsolutePathInPlace(m_Directory); diff --git a/src/zen/consoleprogress.cpp b/src/zen/consoleprogress.cpp new file mode 100644 index 000000000..1726a08aa --- /dev/null +++ b/src/zen/consoleprogress.cpp @@ -0,0 +1,582 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +// Zen command line client utility +// + +#include "consoleprogress.h" + +#include +#include +#include +#include +#include +#include + +#if !ZEN_PLATFORM_WINDOWS +# include +#endif + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +////////////////////////////////////////////////////////////////////////// + +namespace zen { + +// Global tracking for scroll region cleanup on abnormal termination (Ctrl+C etc.) +// Only one ProgressBar can own a scroll region at a time. +static std::atomic g_ActiveScrollRegionOwner{nullptr}; +static std::atomic g_ActiveScrollRegionRows{0}; + +static void +ResetScrollRegionRaw() +{ + // Signal-safe: emit raw escape sequences to restore terminal state. + // These are async-signal-safe on POSIX (write()) and safe in console + // ctrl handlers on Windows (WriteConsole is allowed). + uint32_t Rows = g_ActiveScrollRegionRows.load(std::memory_order_acquire); + if (Rows >= 3) + { + // Move to status line, erase it, reset scroll region, move cursor to end of content + TuiMoveCursor(Rows, 1); + TuiEraseLine(); + TuiResetScrollRegion(); + TuiMoveCursor(Rows - 1, 1); + } + else + { + TuiResetScrollRegion(); + } + TuiShowCursor(true); + TuiFlush(); +} + +#if ZEN_PLATFORM_WINDOWS +static BOOL WINAPI +ScrollRegionCtrlHandler(DWORD CtrlType) +{ + if (CtrlType == CTRL_C_EVENT || CtrlType == CTRL_BREAK_EVENT) + { + ResetScrollRegionRaw(); + } + // Return FALSE so the default handler (process termination) still runs + return FALSE; +} +#else +static struct sigaction s_PrevSigIntAction; +static struct sigaction s_PrevSigTermAction; + +static void +ScrollRegionSignalHandler(int Signal) +{ + ResetScrollRegionRaw(); + + // Re-raise with the previous handler + struct sigaction* PrevAction = (Signal == SIGINT) ? &s_PrevSigIntAction : &s_PrevSigTermAction; + sigaction(Signal, PrevAction, nullptr); + raise(Signal); +} +#endif + +static void +InstallScrollRegionCleanupHandler() +{ +#if ZEN_PLATFORM_WINDOWS + SetConsoleCtrlHandler(ScrollRegionCtrlHandler, TRUE); +#else + struct sigaction Action = {}; + Action.sa_handler = ScrollRegionSignalHandler; + Action.sa_flags = SA_RESETHAND; // one-shot + sigemptyset(&Action.sa_mask); + sigaction(SIGINT, &Action, &s_PrevSigIntAction); + sigaction(SIGTERM, &Action, &s_PrevSigTermAction); +#endif +} + +static void +RemoveScrollRegionCleanupHandler() +{ +#if ZEN_PLATFORM_WINDOWS + SetConsoleCtrlHandler(ScrollRegionCtrlHandler, FALSE); +#else + sigaction(SIGINT, &s_PrevSigIntAction, nullptr); + sigaction(SIGTERM, &s_PrevSigTermAction, nullptr); +#endif +} + +#if ZEN_PLATFORM_WINDOWS +static HANDLE +GetConsoleHandle() +{ + static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); + return hStdOut; +} +#endif + +static void +OutputToConsoleRaw(const char* String, size_t Length) +{ +#if ZEN_PLATFORM_WINDOWS + HANDLE hStdOut = GetConsoleHandle(); + if (TuiIsStdoutTty()) + { + WriteConsoleA(hStdOut, String, (DWORD)Length, 0, 0); + } + else + { + ::WriteFile(hStdOut, (LPCVOID)String, (DWORD)Length, 0, 0); + } +#else + fwrite(String, 1, Length, stdout); +#endif +} + +static void +OutputToConsoleRaw(const std::string& String) +{ + OutputToConsoleRaw(String.c_str(), String.length()); +} + +static void +OutputToConsoleRaw(const StringBuilderBase& SB) +{ + OutputToConsoleRaw(SB.c_str(), SB.Size()); +} + +static uint32_t +GetUpdateDelayMS(ConsoleProgressMode InMode) +{ + switch (InMode) + { + case ConsoleProgressMode::Plain: + return 5000; + case ConsoleProgressMode::Pretty: + return 200; + case ConsoleProgressMode::Log: + return 2000; + case ConsoleProgressMode::Quiet: + return 5000; + default: + ZEN_ASSERT(false); + return 0; + } +} + +class ConsoleProgressBar : public ProgressBase::ProgressBar +{ +public: + explicit ConsoleProgressBar(ConsoleProgressMode InMode, std::string_view InSubTask); + ~ConsoleProgressBar(); + + void UpdateState(const State& NewState, bool DoLinebreak) override; + void ForceLinebreak() override; + void Finish() override; + +private: + void SetupScrollRegion(); + void TeardownScrollRegion(); + void RenderStatusLine(std::string_view Line); + + const ConsoleProgressMode m_Mode; + Stopwatch m_SW; + uint64_t m_LastUpdateMS; + uint64_t m_PausedMS; + State m_State; + const std::string m_SubTask; + size_t m_LastOutputLength = 0; + bool m_ScrollRegionActive = false; + uint32_t m_ScrollRegionRows = 0; +}; + +ConsoleProgressBar::ConsoleProgressBar(ConsoleProgressMode InMode, std::string_view InSubTask) +: m_Mode((!TuiIsStdoutTty() && InMode == ConsoleProgressMode::Pretty) ? ConsoleProgressMode::Plain : InMode) +, m_LastUpdateMS((uint64_t)-1) +, m_PausedMS(0) +, m_SubTask(InSubTask) +{ + ZEN_ASSERT(InSubTask.find('\"') == std::string_view::npos); + if (!m_SubTask.empty()) + { + if (m_Mode == ConsoleProgressMode::Log) + { + std::string String = fmt::format("@progress push \"{}\"\n", m_SubTask); + OutputToConsoleRaw(String); + } + } + + if (m_Mode == ConsoleProgressMode::Pretty) + { + SetupScrollRegion(); + } +} + +ConsoleProgressBar::~ConsoleProgressBar() +{ + try + { + TeardownScrollRegion(); + ForceLinebreak(); + if (!m_SubTask.empty()) + { + if (m_Mode == ConsoleProgressMode::Log) + { + const std::string String("@progress pop\n"); + OutputToConsoleRaw(String); + } + } + } + catch (const std::exception& Ex) + { + ZEN_ERROR("ConsoleProgressBar::~ConsoleProgressBar() failed with {}", Ex.what()); + } +} + +void +ConsoleProgressBar::SetupScrollRegion() +{ + // Only one scroll region owner at a time; nested bars fall back to the inline \r path. + if (g_ActiveScrollRegionOwner.load(std::memory_order_acquire) != nullptr) + { + return; + } + + uint32_t Rows = TuiConsoleRows(0); + if (Rows < 3) + { + return; + } + + TuiEnableOutput(); + + // Ensure cursor is not on the last row before we install the region. + // Print a newline to push content up if needed, then set the region. + OutputToConsoleRaw("\n", 1); + TuiSetScrollRegion(1, Rows - 1); + + // Move cursor into the scroll region so normal output stays there + TuiMoveCursor(Rows - 1, 1); + + m_ScrollRegionActive = true; + m_ScrollRegionRows = Rows; + + g_ActiveScrollRegionRows.store(Rows, std::memory_order_release); + g_ActiveScrollRegionOwner.store(this, std::memory_order_release); + InstallScrollRegionCleanupHandler(); +} + +void +ConsoleProgressBar::TeardownScrollRegion() +{ + if (!m_ScrollRegionActive) + { + return; + } + m_ScrollRegionActive = false; + + RemoveScrollRegionCleanupHandler(); + g_ActiveScrollRegionOwner.store(nullptr, std::memory_order_release); + g_ActiveScrollRegionRows.store(0, std::memory_order_release); + + // Emit all teardown escape sequences as a single atomic write + ExtendableStringBuilder<128> Buf; + Buf << fmt::format("\x1b[{};1H", m_ScrollRegionRows) // move to status line + << "\x1b[2K" // erase it + << "\x1b[r" // reset scroll region + << fmt::format("\x1b[{};1H", m_ScrollRegionRows - 1); // move to end of content + OutputToConsoleRaw(Buf); + TuiFlush(); +} + +void +ConsoleProgressBar::RenderStatusLine(std::string_view Line) +{ + // Handle terminal resizes by re-querying row count + uint32_t CurrentRows = TuiConsoleRows(0); + if (CurrentRows >= 3 && CurrentRows != m_ScrollRegionRows) + { + // Terminal was resized - reinstall scroll region + TuiSetScrollRegion(1, CurrentRows - 1); + m_ScrollRegionRows = CurrentRows; + } + + // Build the entire escape sequence as a single string so the console write + // is atomic and log output from other threads cannot interleave. + ExtendableStringBuilder<512> Buf; + Buf << "\x1b" + "7" // ESC 7 - save cursor + << fmt::format("\x1b[{};1H", m_ScrollRegionRows) // move to bottom row + << "\x1b[2K" // erase entire line + << Line // progress bar content + << "\x1b" + "8"; // ESC 8 - restore cursor + OutputToConsoleRaw(Buf); +} + +void +ConsoleProgressBar::UpdateState(const State& NewState, bool DoLinebreak) +{ + ZEN_ASSERT(NewState.TotalCount >= NewState.RemainingCount); + ZEN_ASSERT(NewState.Task.find('\"') == std::string::npos); + if (DoLinebreak == false && m_State == NewState) + { + return; + } + + uint64_t ElapsedTimeMS = NewState.OptionalElapsedTime == (uint64_t)-1 ? m_SW.GetElapsedTimeMs() : NewState.OptionalElapsedTime; + if (m_LastUpdateMS != (uint64_t)-1) + { + if (!DoLinebreak && (NewState.Status == m_State.Status) && (NewState.Task == m_State.Task) && + ((m_LastUpdateMS + 200) > ElapsedTimeMS)) + { + return; + } + if (m_State.Status == State::EStatus::Paused) + { + uint64_t ElapsedSinceLast = ElapsedTimeMS - m_LastUpdateMS; + m_PausedMS += ElapsedSinceLast; + } + } + + m_LastUpdateMS = ElapsedTimeMS; + + std::string Task = NewState.Task; + switch (NewState.Status) + { + case State::EStatus::Aborted: + Task = "Aborting"; + break; + case State::EStatus::Paused: + Task = "Paused"; + break; + default: + break; + } + if (NewState.Task.length() > Task.length()) + { + Task += std::string(NewState.Task.length() - Task.length(), ' '); + } + + const size_t PercentDone = + NewState.TotalCount > 0u ? gsl::narrow((100 * (NewState.TotalCount - NewState.RemainingCount)) / NewState.TotalCount) : 0u; + + uint64_t Completed = NewState.TotalCount - NewState.RemainingCount; + uint64_t ETAElapsedMS = ElapsedTimeMS - m_PausedMS; + uint64_t ETAMS = ((m_State.TotalCount == NewState.TotalCount) && (NewState.Status == State::EStatus::Running)) && (PercentDone > 5) + ? (ETAElapsedMS * NewState.RemainingCount) / Completed + : 0; + const std::string ETAString = (ETAMS > 0) ? fmt::format(" ETA {}", NiceTimeSpanMs(ETAMS)) : ""; + + if (m_Mode == ConsoleProgressMode::Plain) + { + const std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; + const std::string Output = fmt::format("{} {}% {}{}{}\n", Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), ETAString, Details); + OutputToConsoleRaw(Output); + m_State = NewState; + } + else if (m_Mode == ConsoleProgressMode::Pretty) + { + size_t ProgressBarSize = 20; + + size_t ProgressBarCount = (ProgressBarSize * PercentDone) / 100; + + uint32_t ConsoleColumns = TuiConsoleColumns(1024); + + const std::string PercentString = fmt::format("{:#3}%", PercentDone); + + const std::string ProgressBarString = + fmt::format(": |{}{}|", std::string(ProgressBarCount, '#'), std::string(ProgressBarSize - ProgressBarCount, ' ')); + + const std::string ElapsedString = fmt::format(": {}", NiceTimeSpanMs(ElapsedTimeMS)); + + const std::string DetailsString = (!NewState.Details.empty()) ? fmt::format(". {}", NewState.Details) : ""; + + ExtendableStringBuilder<256> OutputBuilder; + + OutputBuilder << Task << " " << PercentString; + if (OutputBuilder.Size() + 1 < ConsoleColumns) + { + size_t RemainingSpace = ConsoleColumns - (OutputBuilder.Size() + 1); + bool ElapsedFits = RemainingSpace >= ElapsedString.length(); + RemainingSpace -= ElapsedString.length(); + bool ETAFits = ElapsedFits && RemainingSpace >= ETAString.length(); + RemainingSpace -= ETAString.length(); + bool DetailsFits = ETAFits && RemainingSpace >= DetailsString.length(); + RemainingSpace -= DetailsString.length(); + bool ProgressBarFits = DetailsFits && RemainingSpace >= ProgressBarString.length(); + RemainingSpace -= ProgressBarString.length(); + + if (ProgressBarFits) + { + OutputBuilder << ProgressBarString; + } + if (ElapsedFits) + { + OutputBuilder << ElapsedString; + } + if (ETAFits) + { + OutputBuilder << ETAString; + } + if (DetailsFits) + { + OutputBuilder << DetailsString; + } + } + + if (m_ScrollRegionActive) + { + // Render on the pinned bottom status line + RenderStatusLine(OutputBuilder.ToView()); + } + else + { + // Fallback: inline \r-based overwrite (terminal too small for scroll region) + std::string_view Output = OutputBuilder.ToView(); + std::string::size_type EraseLength = + m_LastOutputLength > (Output.length() + 1) ? (m_LastOutputLength - Output.length() - 1) : 0; + ExtendableStringBuilder<256> LineToPrint; + + if (Output.length() + 1 + EraseLength >= ConsoleColumns) + { + if (m_LastOutputLength > 0) + { + LineToPrint << "\n"; + } + LineToPrint << Output; + DoLinebreak = true; + } + else + { + LineToPrint << "\r" << Output << std::string(EraseLength, ' '); + } + + if (DoLinebreak) + { + LineToPrint << "\n"; + } + + OutputToConsoleRaw(LineToPrint); + m_LastOutputLength = DoLinebreak ? 0 : (Output.length() + 1); // +1 for \r prefix + } + + m_State = NewState; + } + else if (m_Mode == ConsoleProgressMode::Log) + { + if (m_State.Task != NewState.Task || + m_State.Details != NewState.Details) // TODO: Should we output just because details change? Will this spam the log collector? + { + std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; + for (std::string::value_type& Char : Details) + { + if (Char == '"') + { + Char = '\''; + } + } + const std::string Message = + fmt::format("@progress \"{} {}{}{}\"\n", NewState.Task, NiceTimeSpanMs(ElapsedTimeMS), ETAString, Details); + OutputToConsoleRaw(Message); + } + + const size_t OldPercentDone = + m_State.TotalCount > 0u ? gsl::narrow((100 * (m_State.TotalCount - m_State.RemainingCount)) / m_State.TotalCount) : 0u; + + if (OldPercentDone != PercentDone) + { + const std::string Progress = fmt::format("@progress {}%\n", PercentDone); + OutputToConsoleRaw(Progress); + } + m_State = NewState; + } +} + +void +ConsoleProgressBar::ForceLinebreak() +{ + if (m_LastOutputLength > 0) + { + State NewState = m_State; + UpdateState(NewState, /*DoLinebreak*/ true); + } +} + +void +ConsoleProgressBar::Finish() +{ + TeardownScrollRegion(); + + if (m_LastOutputLength > 0 || m_State.RemainingCount > 0) + { + State NewState = m_State; + NewState.RemainingCount = 0; + NewState.Details = ""; + UpdateState(NewState, /*DoLinebreak*/ true); + } + m_State = State{}; + m_LastOutputLength = 0; + m_SW.Reset(); +} + +class ConsoleProgress : public ProgressBase +{ +public: + ConsoleProgress(ConsoleProgressMode InMode) : m_Mode(InMode) {} + + virtual void SetLogOperationName(std::string_view Name) override + { + ZEN_ASSERT(Name.find('\"') == std::string_view::npos); + if (m_Mode == ConsoleProgressMode::Log) + { + std::string String = fmt::format("@progress \"{}\"\n", Name); + OutputToConsoleRaw(String); + } + } + + virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) override + { + if (m_Mode == ConsoleProgressMode::Log) + { + const size_t PercentDone = StepCount > 0u ? gsl::narrow((100 * StepIndex) / StepCount) : 0u; + std::string String = fmt::format("@progress {}%\n", PercentDone); + OutputToConsoleRaw(String); + } + } + virtual void PushLogOperation(std::string_view Name) override + { + ZEN_ASSERT(Name.find('\"') == std::string_view::npos); + if (m_Mode == ConsoleProgressMode::Log) + { + std::string String = fmt::format("@progress push \"{}\"\n", Name); + OutputToConsoleRaw(String); + } + } + + virtual void PopLogOperation() override + { + if (m_Mode == ConsoleProgressMode::Log) + { + const std::string String("@progress pop\n"); + OutputToConsoleRaw(String); + } + } + + virtual uint32_t GetProgressUpdateDelayMS() const override { return GetUpdateDelayMS(m_Mode); } + + virtual std::unique_ptr CreateProgressBar(std::string_view InSubTask) override + { + return std::make_unique(m_Mode, InSubTask); + } + +private: + ConsoleProgressMode m_Mode; +}; + +ProgressBase* +CreateConsoleProgress(ConsoleProgressMode InMode) +{ + return new ConsoleProgress(InMode); +} + +} // namespace zen diff --git a/src/zen/consoleprogress.h b/src/zen/consoleprogress.h new file mode 100644 index 000000000..3b68f4ca5 --- /dev/null +++ b/src/zen/consoleprogress.h @@ -0,0 +1,19 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +namespace zen { + +enum class ConsoleProgressMode +{ + Plain, + Pretty, + Log, + Quiet +}; + +ProgressBase* CreateConsoleProgress(ConsoleProgressMode InMode); + +} // namespace zen diff --git a/src/zen/progressbar.cpp b/src/zen/progressbar.cpp deleted file mode 100644 index 780b08707..000000000 --- a/src/zen/progressbar.cpp +++ /dev/null @@ -1,567 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -// Zen command line client utility -// - -#include "progressbar.h" - -#include -#include -#include -#include -#include - -#if !ZEN_PLATFORM_WINDOWS -# include -#endif - -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END - -////////////////////////////////////////////////////////////////////////// - -namespace zen { - -// Global tracking for scroll region cleanup on abnormal termination (Ctrl+C etc.) -// Only one ProgressBar can own a scroll region at a time. -static std::atomic g_ActiveScrollRegionOwner{nullptr}; -static std::atomic g_ActiveScrollRegionRows{0}; - -static void -ResetScrollRegionRaw() -{ - // Signal-safe: emit raw escape sequences to restore terminal state. - // These are async-signal-safe on POSIX (write()) and safe in console - // ctrl handlers on Windows (WriteConsole is allowed). - uint32_t Rows = g_ActiveScrollRegionRows.load(std::memory_order_acquire); - if (Rows >= 3) - { - // Move to status line, erase it, reset scroll region, move cursor to end of content - TuiMoveCursor(Rows, 1); - TuiEraseLine(); - TuiResetScrollRegion(); - TuiMoveCursor(Rows - 1, 1); - } - else - { - TuiResetScrollRegion(); - } - TuiShowCursor(true); - TuiFlush(); -} - -#if ZEN_PLATFORM_WINDOWS -static BOOL WINAPI -ScrollRegionCtrlHandler(DWORD CtrlType) -{ - if (CtrlType == CTRL_C_EVENT || CtrlType == CTRL_BREAK_EVENT) - { - ResetScrollRegionRaw(); - } - // Return FALSE so the default handler (process termination) still runs - return FALSE; -} -#else -static struct sigaction s_PrevSigIntAction; -static struct sigaction s_PrevSigTermAction; - -static void -ScrollRegionSignalHandler(int Signal) -{ - ResetScrollRegionRaw(); - - // Re-raise with the previous handler - struct sigaction* PrevAction = (Signal == SIGINT) ? &s_PrevSigIntAction : &s_PrevSigTermAction; - sigaction(Signal, PrevAction, nullptr); - raise(Signal); -} -#endif - -static void -InstallScrollRegionCleanupHandler() -{ -#if ZEN_PLATFORM_WINDOWS - SetConsoleCtrlHandler(ScrollRegionCtrlHandler, TRUE); -#else - struct sigaction Action = {}; - Action.sa_handler = ScrollRegionSignalHandler; - Action.sa_flags = SA_RESETHAND; // one-shot - sigemptyset(&Action.sa_mask); - sigaction(SIGINT, &Action, &s_PrevSigIntAction); - sigaction(SIGTERM, &Action, &s_PrevSigTermAction); -#endif -} - -static void -RemoveScrollRegionCleanupHandler() -{ -#if ZEN_PLATFORM_WINDOWS - SetConsoleCtrlHandler(ScrollRegionCtrlHandler, FALSE); -#else - sigaction(SIGINT, &s_PrevSigIntAction, nullptr); - sigaction(SIGTERM, &s_PrevSigTermAction, nullptr); -#endif -} - -#if ZEN_PLATFORM_WINDOWS -static HANDLE -GetConsoleHandle() -{ - static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); - return hStdOut; -} -#endif - -static void -OutputToConsoleRaw(const char* String, size_t Length) -{ -#if ZEN_PLATFORM_WINDOWS - HANDLE hStdOut = GetConsoleHandle(); - if (TuiIsStdoutTty()) - { - WriteConsoleA(hStdOut, String, (DWORD)Length, 0, 0); - } - else - { - ::WriteFile(hStdOut, (LPCVOID)String, (DWORD)Length, 0, 0); - } -#else - fwrite(String, 1, Length, stdout); -#endif -} - -static void -OutputToConsoleRaw(const std::string& String) -{ - OutputToConsoleRaw(String.c_str(), String.length()); -} - -static void -OutputToConsoleRaw(const StringBuilderBase& SB) -{ - OutputToConsoleRaw(SB.c_str(), SB.Size()); -} - -uint32_t -GetUpdateDelayMS(ProgressBar::Mode InMode) -{ - switch (InMode) - { - case ProgressBar::Mode::Plain: - return 5000; - case ProgressBar::Mode::Pretty: - return 200; - case ProgressBar::Mode::Log: - return 2000; - default: - ZEN_ASSERT(false); - return 0; - } -} - -void -ProgressBar::SetLogOperationName(Mode InMode, std::string_view Name) -{ - ZEN_ASSERT(Name.find('\"') == std::string_view::npos); - if (InMode == Mode::Log) - { - std::string String = fmt::format("@progress \"{}\"\n", Name); - OutputToConsoleRaw(String); - } -} - -void -ProgressBar::SetLogOperationProgress(Mode InMode, uint32_t StepIndex, uint32_t StepCount) -{ - if (InMode == Mode::Log) - { - const size_t PercentDone = StepCount > 0u ? gsl::narrow((100 * StepIndex) / StepCount) : 0u; - - std::string String = fmt::format("@progress {}%\n", PercentDone); - OutputToConsoleRaw(String); - } -} - -void -ProgressBar::PushLogOperation(Mode InMode, std::string_view Name) -{ - if (InMode == Mode::Log) - { - std::string String = fmt::format("@progress push \"{}\"\n", Name); - OutputToConsoleRaw(String); - } -} - -void -ProgressBar::PopLogOperation(Mode InMode) -{ - if (InMode == Mode::Log) - { - const std::string String("@progress pop\n"); - OutputToConsoleRaw(String); - } -} - -ProgressBar::ProgressBar(Mode InMode, std::string_view InSubTask) -: m_Mode((!TuiIsStdoutTty() && InMode == Mode::Pretty) ? Mode::Plain : InMode) -, m_LastUpdateMS((uint64_t)-1) -, m_PausedMS(0) -, m_SubTask(InSubTask) -{ - ZEN_ASSERT(InSubTask.find('\"') == std::string_view::npos); - if (!m_SubTask.empty()) - { - PushLogOperation(InMode, m_SubTask); - } - - if (m_Mode == Mode::Pretty) - { - SetupScrollRegion(); - } -} - -ProgressBar::~ProgressBar() -{ - try - { - TeardownScrollRegion(); - ForceLinebreak(); - if (!m_SubTask.empty()) - { - PopLogOperation(m_Mode); - } - } - catch (const std::exception& Ex) - { - ZEN_ERROR("ProgressBar::~ProgressBar() failed with {}", Ex.what()); - } -} - -void -ProgressBar::SetupScrollRegion() -{ - // Only one scroll region owner at a time; nested bars fall back to the inline \r path. - if (g_ActiveScrollRegionOwner.load(std::memory_order_acquire) != nullptr) - { - return; - } - - uint32_t Rows = TuiConsoleRows(0); - if (Rows < 3) - { - return; - } - - TuiEnableOutput(); - - // Ensure cursor is not on the last row before we install the region. - // Print a newline to push content up if needed, then set the region. - OutputToConsoleRaw("\n"); - TuiSetScrollRegion(1, Rows - 1); - - // Move cursor into the scroll region so normal output stays there - TuiMoveCursor(Rows - 1, 1); - - m_ScrollRegionActive = true; - m_ScrollRegionRows = Rows; - - g_ActiveScrollRegionRows.store(Rows, std::memory_order_release); - g_ActiveScrollRegionOwner.store(this, std::memory_order_release); - InstallScrollRegionCleanupHandler(); -} - -void -ProgressBar::TeardownScrollRegion() -{ - if (!m_ScrollRegionActive) - { - return; - } - m_ScrollRegionActive = false; - - RemoveScrollRegionCleanupHandler(); - g_ActiveScrollRegionOwner.store(nullptr, std::memory_order_release); - g_ActiveScrollRegionRows.store(0, std::memory_order_release); - - // Emit all teardown escape sequences as a single atomic write - ExtendableStringBuilder<128> Buf; - Buf << fmt::format("\x1b[{};1H", m_ScrollRegionRows) // move to status line - << "\x1b[2K" // erase it - << "\x1b[r" // reset scroll region - << fmt::format("\x1b[{};1H", m_ScrollRegionRows - 1); // move to end of content - OutputToConsoleRaw(Buf); - TuiFlush(); -} - -void -ProgressBar::RenderStatusLine(std::string_view Line) -{ - // Handle terminal resizes by re-querying row count - uint32_t CurrentRows = TuiConsoleRows(0); - if (CurrentRows >= 3 && CurrentRows != m_ScrollRegionRows) - { - // Terminal was resized - reinstall scroll region - TuiSetScrollRegion(1, CurrentRows - 1); - m_ScrollRegionRows = CurrentRows; - } - - // Build the entire escape sequence as a single string so the console write - // is atomic and log output from other threads cannot interleave. - ExtendableStringBuilder<512> Buf; - Buf << "\x1b" - "7" // ESC 7 - save cursor - << fmt::format("\x1b[{};1H", m_ScrollRegionRows) // move to bottom row - << "\x1b[2K" // erase entire line - << Line // progress bar content - << "\x1b" - "8"; // ESC 8 - restore cursor - OutputToConsoleRaw(Buf); -} - -void -ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) -{ - ZEN_ASSERT(NewState.TotalCount >= NewState.RemainingCount); - ZEN_ASSERT(NewState.Task.find('\"') == std::string::npos); - if (DoLinebreak == false && m_State == NewState) - { - return; - } - - uint64_t ElapsedTimeMS = NewState.OptionalElapsedTime == (uint64_t)-1 ? m_SW.GetElapsedTimeMs() : NewState.OptionalElapsedTime; - if (m_LastUpdateMS != (uint64_t)-1) - { - if (!DoLinebreak && (NewState.Status == m_State.Status) && (NewState.Task == m_State.Task) && - ((m_LastUpdateMS + 200) > ElapsedTimeMS)) - { - return; - } - if (m_State.Status == State::EStatus::Paused) - { - uint64_t ElapsedSinceLast = ElapsedTimeMS - m_LastUpdateMS; - m_PausedMS += ElapsedSinceLast; - } - } - - m_LastUpdateMS = ElapsedTimeMS; - - std::string Task = NewState.Task; - switch (NewState.Status) - { - case State::EStatus::Aborted: - Task = "Aborting"; - break; - case State::EStatus::Paused: - Task = "Paused"; - break; - default: - break; - } - if (NewState.Task.length() > Task.length()) - { - Task += std::string(NewState.Task.length() - Task.length(), ' '); - } - - const size_t PercentDone = - NewState.TotalCount > 0u ? gsl::narrow((100 * (NewState.TotalCount - NewState.RemainingCount)) / NewState.TotalCount) : 0u; - - uint64_t Completed = NewState.TotalCount - NewState.RemainingCount; - uint64_t ETAElapsedMS = ElapsedTimeMS - m_PausedMS; - uint64_t ETAMS = ((m_State.TotalCount == NewState.TotalCount) && (NewState.Status == State::EStatus::Running)) && (PercentDone > 5) - ? (ETAElapsedMS * NewState.RemainingCount) / Completed - : 0; - const std::string ETAString = (ETAMS > 0) ? fmt::format(" ETA {}", NiceTimeSpanMs(ETAMS)) : ""; - - if (m_Mode == Mode::Plain) - { - const std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; - const std::string Output = fmt::format("{} {}% {}{}{}\n", Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), ETAString, Details); - OutputToConsoleRaw(Output); - m_State = NewState; - } - else if (m_Mode == Mode::Pretty) - { - size_t ProgressBarSize = 20; - - size_t ProgressBarCount = (ProgressBarSize * PercentDone) / 100; - - uint32_t ConsoleColumns = TuiConsoleColumns(1024); - - const std::string PercentString = fmt::format("{:#3}%", PercentDone); - - const std::string ProgressBarString = - fmt::format(": |{}{}|", std::string(ProgressBarCount, '#'), std::string(ProgressBarSize - ProgressBarCount, ' ')); - - const std::string ElapsedString = fmt::format(": {}", NiceTimeSpanMs(ElapsedTimeMS)); - - const std::string DetailsString = (!NewState.Details.empty()) ? fmt::format(". {}", NewState.Details) : ""; - - ExtendableStringBuilder<256> OutputBuilder; - - OutputBuilder << Task << " " << PercentString; - if (OutputBuilder.Size() + 1 < ConsoleColumns) - { - size_t RemainingSpace = ConsoleColumns - (OutputBuilder.Size() + 1); - bool ElapsedFits = RemainingSpace >= ElapsedString.length(); - RemainingSpace -= ElapsedString.length(); - bool ETAFits = ElapsedFits && RemainingSpace >= ETAString.length(); - RemainingSpace -= ETAString.length(); - bool DetailsFits = ETAFits && RemainingSpace >= DetailsString.length(); - RemainingSpace -= DetailsString.length(); - bool ProgressBarFits = DetailsFits && RemainingSpace >= ProgressBarString.length(); - RemainingSpace -= ProgressBarString.length(); - - if (ProgressBarFits) - { - OutputBuilder << ProgressBarString; - } - if (ElapsedFits) - { - OutputBuilder << ElapsedString; - } - if (ETAFits) - { - OutputBuilder << ETAString; - } - if (DetailsFits) - { - OutputBuilder << DetailsString; - } - } - - if (m_ScrollRegionActive) - { - // Render on the pinned bottom status line - RenderStatusLine(OutputBuilder.ToView()); - } - else - { - // Fallback: inline \r-based overwrite (terminal too small for scroll region) - std::string_view Output = OutputBuilder.ToView(); - std::string::size_type EraseLength = - m_LastOutputLength > (Output.length() + 1) ? (m_LastOutputLength - Output.length() - 1) : 0; - ExtendableStringBuilder<256> LineToPrint; - - if (Output.length() + 1 + EraseLength >= ConsoleColumns) - { - if (m_LastOutputLength > 0) - { - LineToPrint << "\n"; - } - LineToPrint << Output; - DoLinebreak = true; - } - else - { - LineToPrint << "\r" << Output << std::string(EraseLength, ' '); - } - - if (DoLinebreak) - { - LineToPrint << "\n"; - } - - OutputToConsoleRaw(LineToPrint); - m_LastOutputLength = DoLinebreak ? 0 : (Output.length() + 1); // +1 for \r prefix - } - - m_State = NewState; - } - else if (m_Mode == Mode::Log) - { - if (m_State.Task != NewState.Task || - m_State.Details != NewState.Details) // TODO: Should we output just because details change? Will this spam the log collector? - { - std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; - for (std::string::value_type& Char : Details) - { - if (Char == '"') - { - Char = '\''; - } - } - const std::string Message = - fmt::format("@progress \"{} {}{}{}\"\n", NewState.Task, NiceTimeSpanMs(ElapsedTimeMS), ETAString, Details); - OutputToConsoleRaw(Message); - } - - const size_t OldPercentDone = - m_State.TotalCount > 0u ? gsl::narrow((100 * (m_State.TotalCount - m_State.RemainingCount)) / m_State.TotalCount) : 0u; - - if (OldPercentDone != PercentDone) - { - const std::string Progress = fmt::format("@progress {}%\n", PercentDone); - OutputToConsoleRaw(Progress); - } - m_State = NewState; - } -} - -void -ProgressBar::ForceLinebreak() -{ - if (m_LastOutputLength > 0) - { - State NewState = m_State; - UpdateState(NewState, /*DoLinebreak*/ true); - } -} - -void -ProgressBar::Finish() -{ - TeardownScrollRegion(); - - if (m_LastOutputLength > 0 || m_State.RemainingCount > 0) - { - State NewState = m_State; - NewState.RemainingCount = 0; - NewState.Details = ""; - UpdateState(NewState, /*DoLinebreak*/ true); - } - m_State = State{}; - m_LastOutputLength = 0; - m_SW.Reset(); -} - -bool -ProgressBar::IsSameTask(std::string_view Task) const -{ - return Task == m_State.Task; -} - -bool -ProgressBar::HasActiveTask() const -{ - return !m_State.Task.empty(); -} - -class ConsoleOpLogOutput : public ProgressBase -{ -public: - ConsoleOpLogOutput(zen::ProgressBar::Mode InMode) : m_Mode(InMode) {} - - virtual void SetLogOperationName(std::string_view Name) override { zen::ProgressBar::SetLogOperationName(m_Mode, Name); } - virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) override - { - zen::ProgressBar::SetLogOperationProgress(m_Mode, StepIndex, StepCount); - } - virtual uint32_t GetProgressUpdateDelayMS() const override { return GetUpdateDelayMS(m_Mode); } - - virtual std::unique_ptr CreateProgressBar(std::string_view InSubTask) override - { - return std::make_unique(m_Mode, InSubTask); - } - -private: - zen::ProgressBar::Mode m_Mode; -}; - -ProgressBase* -CreateConsoleProgress(ProgressBar::Mode InMode) -{ - return new ConsoleOpLogOutput(InMode); -} - -} // namespace zen diff --git a/src/zen/progressbar.h b/src/zen/progressbar.h deleted file mode 100644 index 26bb9b9c4..000000000 --- a/src/zen/progressbar.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include - -namespace zen { - -class ProgressBar : public ProgressBase::ProgressBar -{ -public: - enum class Mode - { - Plain, - Pretty, - Log, - Quiet - }; - - static void SetLogOperationName(Mode InMode, std::string_view Name); - static void SetLogOperationProgress(Mode InMode, uint32_t StepIndex, uint32_t StepCount); - static void PushLogOperation(Mode InMode, std::string_view Name); - static void PopLogOperation(Mode InMode); - - explicit ProgressBar(Mode InMode, std::string_view InSubTask); - ~ProgressBar(); - - void UpdateState(const State& NewState, bool DoLinebreak) override; - void ForceLinebreak(); - void Finish() override; - bool IsSameTask(std::string_view Task) const; - bool HasActiveTask() const; - -private: - void SetupScrollRegion(); - void TeardownScrollRegion(); - void RenderStatusLine(std::string_view Line); - - const Mode m_Mode; - Stopwatch m_SW; - uint64_t m_LastUpdateMS; - uint64_t m_PausedMS; - State m_State; - const std::string m_SubTask; - size_t m_LastOutputLength = 0; - bool m_ScrollRegionActive = false; - uint32_t m_ScrollRegionRows = 0; -}; - -uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode); - -ProgressBase* CreateConsoleProgress(ProgressBar::Mode InMode); - -} // namespace zen diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 0229db4a8..a09f923fc 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -56,7 +56,7 @@ #include #include -#include "progressbar.h" +#include "consoleprogress.h" #if ZEN_WITH_TESTS # include diff --git a/src/zenremotestore/builds/buildinspect.cpp b/src/zenremotestore/builds/buildinspect.cpp new file mode 100644 index 000000000..1af9e20af --- /dev/null +++ b/src/zenremotestore/builds/buildinspect.cpp @@ -0,0 +1,463 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +using namespace std::literals; + +ChunkedFolderContent +ScanAndChunkFolder(ProgressBase& Progress, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + bool IsQuiet, + TransferThreadWorkers& Workers, + GetFolderContentStatistics& GetFolderContentStats, + ChunkingStatistics& ChunkingStats, + const std::filesystem::path& Path, + std::function&& IsAcceptedFolder, + std::function&& IsAcceptedFile, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache) +{ + Stopwatch Timer; + + ZEN_TRACE_CPU("ScanAndChunkFolder"); + + FolderContent Content = GetFolderContent( + GetFolderContentStats, + Path, + std::move(IsAcceptedFolder), + std::move(IsAcceptedFile), + Workers.GetIOWorkerPool(), + Progress.GetProgressUpdateDelayMS(), + [](bool, std::ptrdiff_t) {}, + AbortFlag); + if (AbortFlag) + { + return {}; + } + + BuildState LocalContent = GetLocalContent(Progress, + AbortFlag, + PauseFlag, + IsQuiet, + Workers, + GetFolderContentStats, + ChunkingStats, + Path, + ZenStateFilePath(Path / ZenFolderName), + ChunkController, + ChunkCache) + .State; + + std::vector UntrackedPaths = GetNewPaths(LocalContent.ChunkedContent.Paths, Content.Paths); + + BuildState UntrackedLocalContent = GetLocalStateFromPaths(Progress, + AbortFlag, + PauseFlag, + Workers, + GetFolderContentStats, + ChunkingStats, + Path, + ChunkController, + ChunkCache, + UntrackedPaths) + .State; + + ChunkedFolderContent Result = + MergeChunkedFolderContents(LocalContent.ChunkedContent, std::vector{UntrackedLocalContent.ChunkedContent}); + + const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0)); + const uint64_t ChunkedRawSize = + std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0)); + + if (!IsQuiet) + { + ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", + Result.Paths.size(), + NiceBytes(TotalRawSize), + Result.ChunkedContent.ChunkHashes.size(), + NiceBytes(ChunkedRawSize), + Path, + NiceTimeSpanMs(Timer.GetElapsedTimeMs()), + NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + } + return Result; +}; + +void +ListBuild(bool IsQuiet, + StorageInstance& Storage, + const Oid& BuildId, + const std::vector& BuildPartIds, + std::span BuildPartNames, + std::span IncludeWildcards, + std::span ExcludeWildcards, + CbObjectWriter* OptionalStructuredOutput) +{ + std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; + + CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId, IsQuiet); + + if (OptionalStructuredOutput != nullptr) + { + OptionalStructuredOutput->AddObjectId("buildId"sv, BuildId); + OptionalStructuredOutput->AddObject("build"sv, BuildObject); + } + + std::vector> AllBuildParts = + ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); + + if (!AllBuildParts.empty()) + { + Stopwatch GetBuildPartTimer; + + if (OptionalStructuredOutput != nullptr) + { + OptionalStructuredOutput->BeginArray("parts"sv); + } + + for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++) + { + const Oid BuildPartId = AllBuildParts[BuildPartIndex].first; + const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second; + CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); + + if (OptionalStructuredOutput != nullptr) + { + OptionalStructuredOutput->BeginObject(); + OptionalStructuredOutput->AddObjectId("id"sv, BuildPartId); + OptionalStructuredOutput->AddString("partName"sv, BuildPartName); + } + { + if (OptionalStructuredOutput != nullptr) + { + } + else if (!IsQuiet) + { + ZEN_CONSOLE("{}Part: {} ('{}'):\n", + BuildPartIndex > 0 ? "\n" : "", + BuildPartId, + BuildPartName, + NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), + NiceBytes(BuildPartManifest.GetSize())); + } + + std::vector Paths; + std::vector RawHashes; + std::vector RawSizes; + std::vector Attributes; + + SourcePlatform Platform; + std::vector SequenceRawHashes; + std::vector ChunkCounts; + std::vector AbsoluteChunkOrders; + std::vector LooseChunkHashes; + std::vector LooseChunkRawSizes; + std::vector BlockRawHashes; + + ReadBuildContentFromCompactBinary(BuildPartManifest, + Platform, + Paths, + RawHashes, + RawSizes, + Attributes, + SequenceRawHashes, + ChunkCounts, + AbsoluteChunkOrders, + LooseChunkHashes, + LooseChunkRawSizes, + BlockRawHashes); + + std::vector Order(Paths.size()); + std::iota(Order.begin(), Order.end(), 0); + + std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) { + const std::filesystem::path& LhsPath = Paths[Lhs]; + const std::filesystem::path& RhsPath = Paths[Rhs]; + return LhsPath < RhsPath; + }); + + if (OptionalStructuredOutput != nullptr) + { + OptionalStructuredOutput->BeginArray("files"sv); + } + { + for (size_t Index : Order) + { + const std::filesystem::path& Path = Paths[Index]; + if (IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(Path.generic_string()), /*CaseSensitive*/ true)) + { + const IoHash& RawHash = RawHashes[Index]; + const uint64_t RawSize = RawSizes[Index]; + const uint32_t Attribute = Attributes[Index]; + + if (OptionalStructuredOutput != nullptr) + { + OptionalStructuredOutput->BeginObject(); + { + OptionalStructuredOutput->AddString("path"sv, fmt::format("{}", Path)); + OptionalStructuredOutput->AddInteger("rawSize"sv, RawSize); + OptionalStructuredOutput->AddHash("rawHash"sv, RawHash); + switch (Platform) + { + case SourcePlatform::Windows: + OptionalStructuredOutput->AddInteger("attributes"sv, Attribute); + break; + case SourcePlatform::MacOS: + case SourcePlatform::Linux: + OptionalStructuredOutput->AddString("chmod"sv, fmt::format("{:#04o}", Attribute)); + break; + default: + throw std::runtime_error(fmt::format("Unsupported platform: {}", (int)Platform)); + } + } + OptionalStructuredOutput->EndObject(); + } + else + { + ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash); + } + } + } + } + if (OptionalStructuredOutput != nullptr) + { + OptionalStructuredOutput->EndArray(); // "files" + } + } + if (OptionalStructuredOutput != nullptr) + { + OptionalStructuredOutput->EndObject(); + } + } + if (OptionalStructuredOutput != nullptr) + { + OptionalStructuredOutput->EndArray(); // parts + } + } +} + +void +DiffFolders(ProgressBase& Progress, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + bool IsQuiet, + TransferThreadWorkers& Workers, + const std::filesystem::path& BasePath, + const std::filesystem::path& ComparePath, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + const std::vector& ExcludeFolders, + const std::vector& ExcludeExtensions) +{ + ZEN_TRACE_CPU("DiffFolders"); + + Progress.SetLogOperationName("Diff Folders"); + + enum TaskSteps : uint32_t + { + CheckBase, + CheckCompare, + Diff, + Cleanup, + StepCount + }; + + auto EndProgress = MakeGuard([&]() { Progress.SetLogOperationProgress(TaskSteps::StepCount, TaskSteps::StepCount); }); + + ChunkedFolderContent BaseFolderContent; + ChunkedFolderContent CompareFolderContent; + + { + auto IsAcceptedFolder = [ExcludeFolders](const std::string_view& RelativePath) -> bool { + for (const std::string& ExcludeFolder : ExcludeFolders) + { + if (RelativePath.starts_with(ExcludeFolder)) + { + if (RelativePath.length() == ExcludeFolder.length()) + { + return false; + } + else if (RelativePath[ExcludeFolder.length()] == '/') + { + return false; + } + } + } + return true; + }; + + auto IsAcceptedFile = [ExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool { + for (const std::string& ExcludeExtension : ExcludeExtensions) + { + if (RelativePath.ends_with(ExcludeExtension)) + { + return false; + } + } + return true; + }; + + Progress.SetLogOperationProgress(TaskSteps::CheckBase, TaskSteps::StepCount); + + GetFolderContentStatistics BaseGetFolderContentStats; + ChunkingStatistics BaseChunkingStats; + BaseFolderContent = ScanAndChunkFolder(Progress, + AbortFlag, + PauseFlag, + IsQuiet, + Workers, + BaseGetFolderContentStats, + BaseChunkingStats, + BasePath, + IsAcceptedFolder, + IsAcceptedFile, + ChunkController, + ChunkCache); + if (AbortFlag) + { + return; + } + + Progress.SetLogOperationProgress(TaskSteps::CheckCompare, TaskSteps::StepCount); + + GetFolderContentStatistics CompareGetFolderContentStats; + ChunkingStatistics CompareChunkingStats; + CompareFolderContent = ScanAndChunkFolder(Progress, + AbortFlag, + PauseFlag, + IsQuiet, + Workers, + CompareGetFolderContentStats, + CompareChunkingStats, + ComparePath, + IsAcceptedFolder, + IsAcceptedFile, + ChunkController, + ChunkCache); + + if (AbortFlag) + { + return; + } + } + + Progress.SetLogOperationProgress(TaskSteps::Diff, TaskSteps::StepCount); + + std::vector AddedHashes; + std::vector RemovedHashes; + uint64_t RemovedSize = 0; + uint64_t AddedSize = 0; + + tsl::robin_map BaseRawHashLookup; + for (size_t PathIndex = 0; PathIndex < BaseFolderContent.RawHashes.size(); PathIndex++) + { + const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex]; + BaseRawHashLookup.insert_or_assign(RawHash, PathIndex); + } + tsl::robin_map CompareRawHashLookup; + for (size_t PathIndex = 0; PathIndex < CompareFolderContent.RawHashes.size(); PathIndex++) + { + const IoHash& RawHash = CompareFolderContent.RawHashes[PathIndex]; + if (!BaseRawHashLookup.contains(RawHash)) + { + AddedHashes.push_back(RawHash); + AddedSize += CompareFolderContent.RawSizes[PathIndex]; + } + CompareRawHashLookup.insert_or_assign(RawHash, PathIndex); + } + for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++) + { + const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex]; + if (!CompareRawHashLookup.contains(RawHash)) + { + RemovedHashes.push_back(RawHash); + RemovedSize += BaseFolderContent.RawSizes[PathIndex]; + } + } + + uint64_t BaseTotalRawSize = 0; + for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++) + { + BaseTotalRawSize += BaseFolderContent.RawSizes[PathIndex]; + } + + double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0; + + ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept", + RemovedHashes.size(), + NiceBytes(RemovedSize), + AddedHashes.size(), + NiceBytes(AddedSize), + BaseFolderContent.Paths.size() - RemovedHashes.size(), + NiceBytes(BaseTotalRawSize - RemovedSize), + KeptPercent); + + uint64_t CompareTotalRawSize = 0; + + uint64_t FoundChunkCount = 0; + uint64_t FoundChunkSize = 0; + uint64_t NewChunkCount = 0; + uint64_t NewChunkSize = 0; + const ChunkedContentLookup BaseFolderLookup = BuildChunkedContentLookup(BaseFolderContent); + for (uint32_t ChunkIndex = 0; ChunkIndex < CompareFolderContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++) + { + const IoHash& ChunkHash = CompareFolderContent.ChunkedContent.ChunkHashes[ChunkIndex]; + if (BaseFolderLookup.ChunkHashToChunkIndex.contains(ChunkHash)) + { + FoundChunkCount++; + FoundChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + } + else + { + NewChunkCount++; + NewChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + } + CompareTotalRawSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + } + + double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0; + double NewPercent = CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0; + + ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.", + FoundChunkCount, + NiceBytes(FoundChunkSize), + FoundPercent, + CompareFolderContent.ChunkedContent.ChunkHashes.size(), + NiceBytes(CompareTotalRawSize), + BaseFolderContent.ChunkedContent.ChunkHashes.size(), + NiceBytes(BaseTotalRawSize), + NewChunkCount, + NiceBytes(NewChunkSize), + NewPercent); + + Progress.SetLogOperationProgress(TaskSteps::Cleanup, TaskSteps::StepCount); +} + +} // namespace zen diff --git a/src/zenremotestore/builds/buildprimecache.cpp b/src/zenremotestore/builds/buildprimecache.cpp new file mode 100644 index 000000000..12791f718 --- /dev/null +++ b/src/zenremotestore/builds/buildprimecache.cpp @@ -0,0 +1,350 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace zen { + +using namespace std::literals; + +BuildsOperationPrimeCache::BuildsOperationPrimeCache(LoggerRef Log, + ProgressBase& Progress, + StorageInstance& Storage, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + WorkerThreadPool& NetworkPool, + const Oid& BuildId, + std::span BuildPartIds, + const Options& Options, + BuildStorageCache::Statistics& StorageCacheStats) +: m_Log(Log) +, m_Progress(Progress) +, m_Storage(Storage) +, m_AbortFlag(AbortFlag) +, m_PauseFlag(PauseFlag) +, m_NetworkPool(NetworkPool) +, m_BuildId(BuildId) +, m_BuildPartIds(BuildPartIds.begin(), BuildPartIds.end()) +, m_Options(Options) +, m_StorageCacheStats(StorageCacheStats) +{ + m_TempPath = m_Options.ZenFolderPath / "tmp"; + CreateDirectories(m_TempPath); +} + +void +BuildsOperationPrimeCache::Execute() +{ + ZEN_TRACE_CPU("BuildsOperationPrimeCache::Execute"); + + Stopwatch PrimeTimer; + + tsl::robin_map LooseChunkRawSizes; + tsl::robin_set BuildBlobs; + CollectReferencedBlobs(BuildBlobs, LooseChunkRawSizes); + + if (!m_Options.IsQuiet) + { + ZEN_INFO("Found {} referenced blobs", BuildBlobs.size()); + } + + if (BuildBlobs.empty()) + { + return; + } + + std::vector BlobsToDownload = FilterAlreadyCachedBlobs(BuildBlobs); + + if (BlobsToDownload.empty()) + { + return; + } + + std::atomic MultipartAttachmentCount; + std::atomic CompletedDownloadCount; + FilteredRate FilteredDownloadedBytesPerSecond; + + ScheduleBlobDownloads(BlobsToDownload, + LooseChunkRawSizes, + MultipartAttachmentCount, + CompletedDownloadCount, + FilteredDownloadedBytesPerSecond); + + if (m_AbortFlag) + { + return; + } + + if (m_Storage.CacheStorage) + { + m_Storage.CacheStorage->Flush(m_Progress.GetProgressUpdateDelayMS(), [this](intptr_t Remaining) -> bool { + ZEN_UNUSED(Remaining); + if (!m_Options.IsQuiet) + { + ZEN_INFO("Waiting for {} blobs to finish upload to '{}'", Remaining, m_Storage.CacheHost.Name); + } + return !m_AbortFlag; + }); + } + + if (!m_Options.IsQuiet) + { + uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() + m_DownloadStats.DownloadedBlockByteCount.load(); + ZEN_INFO("Downloaded {} ({}bits/s) in {}. {} as multipart. Completed in {}", + NiceBytes(DownloadedBytes), + NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)), + NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000), + MultipartAttachmentCount.load(), + NiceTimeSpanMs(PrimeTimer.GetElapsedTimeMs())); + } +} + +void +BuildsOperationPrimeCache::CollectReferencedBlobs(tsl::robin_set& OutBuildBlobs, + tsl::robin_map& OutLooseChunkRawSizes) +{ + for (const Oid& BuildPartId : m_BuildPartIds) + { + CbObject BuildPart = m_Storage.BuildStorage->GetBuildPart(m_BuildId, BuildPartId); + + CbObjectView BlockAttachmentsView = BuildPart["blockAttachments"sv].AsObjectView(); + std::vector BlockAttachments = compactbinary_helpers::ReadBinaryAttachmentArray("rawHashes"sv, BlockAttachmentsView); + + CbObjectView ChunkAttachmentsView = BuildPart["chunkAttachments"sv].AsObjectView(); + std::vector ChunkAttachments = compactbinary_helpers::ReadBinaryAttachmentArray("rawHashes"sv, ChunkAttachmentsView); + std::vector ChunkRawSizes = compactbinary_helpers::ReadArray("chunkRawSizes"sv, ChunkAttachmentsView); + if (ChunkAttachments.size() != ChunkRawSizes.size()) + { + throw std::runtime_error(fmt::format("Mismatch of loose chunk raw size array, expected {}, found {}", + ChunkAttachments.size(), + ChunkRawSizes.size())); + } + + OutBuildBlobs.reserve(ChunkAttachments.size() + BlockAttachments.size()); + OutBuildBlobs.insert(BlockAttachments.begin(), BlockAttachments.end()); + OutBuildBlobs.insert(ChunkAttachments.begin(), ChunkAttachments.end()); + + for (size_t ChunkAttachmentIndex = 0; ChunkAttachmentIndex < ChunkAttachments.size(); ChunkAttachmentIndex++) + { + OutLooseChunkRawSizes.insert_or_assign(ChunkAttachments[ChunkAttachmentIndex], ChunkRawSizes[ChunkAttachmentIndex]); + } + } +} + +std::vector +BuildsOperationPrimeCache::FilterAlreadyCachedBlobs(const tsl::robin_set& BuildBlobs) +{ + std::vector BlobsToDownload; + BlobsToDownload.reserve(BuildBlobs.size()); + + if (m_Storage.CacheStorage && !BuildBlobs.empty() && !m_Options.ForceUpload) + { + ZEN_TRACE_CPU("BlobCacheExistCheck"); + Stopwatch Timer; + + const std::vector BlobHashes(BuildBlobs.begin(), BuildBlobs.end()); + const std::vector CacheExistsResult = + m_Storage.CacheStorage->BlobsExists(m_BuildId, BlobHashes); + + if (CacheExistsResult.size() == BlobHashes.size()) + { + for (size_t BlobIndex = 0; BlobIndex < BlobHashes.size(); BlobIndex++) + { + if (!CacheExistsResult[BlobIndex].HasBody) + { + BlobsToDownload.push_back(BlobHashes[BlobIndex]); + } + } + size_t FoundCount = BuildBlobs.size() - BlobsToDownload.size(); + + if (FoundCount > 0 && !m_Options.IsQuiet) + { + ZEN_INFO("Remote cache : Found {} out of {} needed blobs in {}", + FoundCount, + BuildBlobs.size(), + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } + } + } + else + { + BlobsToDownload.insert(BlobsToDownload.end(), BuildBlobs.begin(), BuildBlobs.end()); + } + return BlobsToDownload; +} + +void +BuildsOperationPrimeCache::ScheduleBlobDownloads(std::span BlobsToDownload, + const tsl::robin_map& LooseChunkRawSizes, + std::atomic& MultipartAttachmentCount, + std::atomic& CompletedDownloadCount, + FilteredRate& FilteredDownloadedBytesPerSecond) +{ + std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Downloading"); + + ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + + const size_t BlobCount = BlobsToDownload.size(); + + for (size_t BlobIndex = 0; BlobIndex < BlobCount; BlobIndex++) + { + Work.ScheduleWork( + m_NetworkPool, + [this, + &Work, + BlobsToDownload, + BlobCount, + &LooseChunkRawSizes, + &CompletedDownloadCount, + &FilteredDownloadedBytesPerSecond, + &MultipartAttachmentCount, + BlobIndex](std::atomic&) { + if (!m_AbortFlag) + { + const IoHash& BlobHash = BlobsToDownload[BlobIndex]; + bool IsLargeBlob = false; + if (auto It = LooseChunkRawSizes.find(BlobHash); It != LooseChunkRawSizes.end()) + { + IsLargeBlob = It->second >= m_Options.LargeAttachmentSize; + } + + FilteredDownloadedBytesPerSecond.Start(); + + if (IsLargeBlob) + { + DownloadLargeBlobForCache(Work, + BlobHash, + BlobCount, + CompletedDownloadCount, + MultipartAttachmentCount, + FilteredDownloadedBytesPerSecond); + } + else + { + DownloadSingleBlobForCache(BlobHash, BlobCount, CompletedDownloadCount, FilteredDownloadedBytesPerSecond); + } + } + }); + } + + Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); + + uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() + m_DownloadStats.DownloadedBlockByteCount.load(); + FilteredDownloadedBytesPerSecond.Update(DownloadedBytes); + + std::string DownloadRateString = (CompletedDownloadCount == BlobCount) + ? "" + : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8)); + std::string UploadDetails = m_Storage.CacheStorage ? fmt::format(" {} ({}) uploaded.", + m_StorageCacheStats.PutBlobCount.load(), + NiceBytes(m_StorageCacheStats.PutBlobByteCount.load())) + : ""; + + std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}", + CompletedDownloadCount.load(), + BlobCount, + NiceBytes(DownloadedBytes), + DownloadRateString, + UploadDetails); + ProgressBar->UpdateState({.Task = "Downloading", + .Details = Details, + .TotalCount = BlobCount, + .RemainingCount = BlobCount - CompletedDownloadCount.load(), + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); + }); + + FilteredDownloadedBytesPerSecond.Stop(); + ProgressBar->Finish(); +} + +void +BuildsOperationPrimeCache::DownloadLargeBlobForCache(ParallelWork& Work, + const IoHash& BlobHash, + size_t BlobCount, + std::atomic& CompletedDownloadCount, + std::atomic& MultipartAttachmentCount, + FilteredRate& FilteredDownloadedBytesPerSecond) +{ + DownloadLargeBlob(*m_Storage.BuildStorage, + m_TempPath, + m_BuildId, + BlobHash, + m_Options.PreferredMultipartChunkSize, + Work, + m_NetworkPool, + m_DownloadStats.DownloadedChunkByteCount, + MultipartAttachmentCount, + [this, BlobCount, BlobHash, &FilteredDownloadedBytesPerSecond, &CompletedDownloadCount](IoBuffer&& Payload) { + m_DownloadStats.DownloadedChunkCount++; + m_DownloadStats.RequestsCompleteCount++; + + if (!m_AbortFlag) + { + if (Payload && m_Storage.CacheStorage) + { + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, + BlobHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(Payload))); + } + } + if (CompletedDownloadCount.fetch_add(1) + 1 == BlobCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + }); +} + +void +BuildsOperationPrimeCache::DownloadSingleBlobForCache(const IoHash& BlobHash, + size_t BlobCount, + std::atomic& CompletedDownloadCount, + FilteredRate& FilteredDownloadedBytesPerSecond) +{ + IoBuffer Payload; + try + { + Payload = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, BlobHash); + + m_DownloadStats.DownloadedBlockCount++; + m_DownloadStats.DownloadedBlockByteCount += Payload.GetSize(); + m_DownloadStats.RequestsCompleteCount++; + } + catch (const std::exception&) + { + // Silence http errors due to abort + if (!m_AbortFlag) + { + throw; + } + } + + if (!m_AbortFlag) + { + if (Payload && m_Storage.CacheStorage) + { + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, + BlobHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(std::move(Payload)))); + } + if (CompletedDownloadCount.fetch_add(1) + 1 == BlobCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + } +} + +} // namespace zen diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp deleted file mode 100644 index 6d93d8de6..000000000 --- a/src/zenremotestore/builds/buildstorageoperations.cpp +++ /dev/null @@ -1,8560 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -ZEN_THIRD_PARTY_INCLUDES_START -#include -#include -ZEN_THIRD_PARTY_INCLUDES_END - -#if ZEN_WITH_TESTS -# include -# include -# include -# include -#endif // ZEN_WITH_TESTS - -namespace zen { - -using namespace std::literals; - -namespace { - std::filesystem::path ZenTempCacheFolderPath(const std::filesystem::path& ZenFolderPath) - { - return ZenTempFolderPath(ZenFolderPath) / "cache"; // Decompressed and verified data - chunks & sequences - } - std::filesystem::path ZenTempBlockFolderPath(const std::filesystem::path& ZenFolderPath) - { - return ZenTempFolderPath(ZenFolderPath) / "blocks"; // Temp storage for whole and partial blocks - } - std::filesystem::path ZenTempDownloadFolderPath(const std::filesystem::path& ZenFolderPath) - { - return ZenTempFolderPath(ZenFolderPath) / "download"; // Temp storage for decompressed and validated chunks - } - - uint64_t GetBytesPerSecond(uint64_t ElapsedWallTimeUS, uint64_t Count) - { - if (ElapsedWallTimeUS == 0) - { - return 0; - } - return Count * 1000000 / ElapsedWallTimeUS; - } - - std::filesystem::path GetTempChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash) - { - return CacheFolderPath / (RawHash.ToHexString() + ".tmp"); - } - - std::filesystem::path GetFinalChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash) - { - return CacheFolderPath / RawHash.ToHexString(); - } - - bool CleanDirectory(LoggerRef InLog, - ProgressBase& Progress, - WorkerThreadPool& IOWorkerPool, - std::atomic& AbortFlag, - std::atomic& PauseFlag, - bool IsQuiet, - const std::filesystem::path& Path, - std::span ExcludeDirectories) - { - ZEN_TRACE_CPU("CleanDirectory"); - ZEN_SCOPED_LOG(InLog); - Stopwatch Timer; - - std::unique_ptr ProgressBar = Progress.CreateProgressBar("Clean Folder"); - - CleanDirectoryResult Result = CleanDirectory( - IOWorkerPool, - AbortFlag, - PauseFlag, - Path, - ExcludeDirectories, - [&](const std::string_view Details, uint64_t TotalCount, uint64_t RemainingCount, bool IsPaused, bool IsAborted) { - ProgressBar->UpdateState({.Task = "Cleaning folder ", - .Details = std::string(Details), - .TotalCount = TotalCount, - .RemainingCount = RemainingCount, - .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }, - Progress.GetProgressUpdateDelayMS()); - - ProgressBar->Finish(); - - if (AbortFlag) - { - return false; - } - - uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); - - if (!Result.FailedRemovePaths.empty()) - { - ExtendableStringBuilder<512> SB; - for (size_t FailedPathIndex = 0; FailedPathIndex < Result.FailedRemovePaths.size(); FailedPathIndex++) - { - SB << fmt::format("\n '{}': ({}) {}", - Result.FailedRemovePaths[FailedPathIndex].first, - Result.FailedRemovePaths[FailedPathIndex].second.value(), - Result.FailedRemovePaths[FailedPathIndex].second.message()); - } - ZEN_WARN("Clean failed to remove files from '{}': {}", Path, SB.ToView()); - } - - if (ElapsedTimeMs >= 200 && !IsQuiet) - { - ZEN_INFO("Wiped folder '{}' {} ({}) in {}", - Path, - Result.FoundCount, - NiceBytes(Result.DeletedByteCount), - NiceTimeSpanMs(ElapsedTimeMs)); - } - - return Result.FailedRemovePaths.empty(); - } - - bool IsExtensionHashCompressable(const tsl::robin_set& NonCompressableExtensionHashes, const uint32_t PathHash) - { - return !NonCompressableExtensionHashes.contains(PathHash); - } - - bool IsChunkCompressable(const tsl::robin_set& NonCompressableExtensionHashes, - const ChunkedContentLookup& Lookup, - uint32_t ChunkIndex) - { - const uint32_t ChunkLocationCount = Lookup.ChunkSequenceLocationCounts[ChunkIndex]; - if (ChunkLocationCount == 0) - { - return false; - } - const size_t ChunkLocationOffset = Lookup.ChunkSequenceLocationOffset[ChunkIndex]; - const uint32_t SequenceIndex = Lookup.ChunkSequenceLocations[ChunkLocationOffset].SequenceIndex; - const uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[SequenceIndex]; - const uint32_t ExtensionHash = Lookup.PathExtensionHash[PathIndex]; - - const bool IsCompressable = IsExtensionHashCompressable(NonCompressableExtensionHashes, ExtensionHash); - return IsCompressable; - } - - template - std::string FormatArray(std::span Items, std::string_view Prefix) - { - ExtendableStringBuilder<512> SB; - for (const T& Item : Items) - { - SB.Append(fmt::format("{}{}", Prefix, Item)); - } - return SB.ToString(); - } - - uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes) - { -#if ZEN_PLATFORM_WINDOWS - if (SourcePlatform == SourcePlatform::Windows) - { - SetFileAttributesToPath(FilePath, Attributes); - return Attributes; - } - else - { - uint32_t CurrentAttributes = GetFileAttributesFromPath(FilePath); - uint32_t NewAttributes = zen::MakeFileAttributeReadOnly(CurrentAttributes, zen::IsFileModeReadOnly(Attributes)); - if (CurrentAttributes != NewAttributes) - { - SetFileAttributesToPath(FilePath, NewAttributes); - } - return NewAttributes; - } -#endif // ZEN_PLATFORM_WINDOWS -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - if (SourcePlatform != SourcePlatform::Windows) - { - zen::SetFileMode(FilePath, Attributes); - return Attributes; - } - else - { - uint32_t CurrentMode = zen::GetFileMode(FilePath); - uint32_t NewMode = zen::MakeFileModeReadOnly(CurrentMode, zen::IsFileAttributeReadOnly(Attributes)); - if (CurrentMode != NewMode) - { - zen::SetFileMode(FilePath, NewMode); - } - return NewMode; - } -#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - }; - - uint32_t GetNativeFileAttributes(const std::filesystem::path FilePath) - { -#if ZEN_PLATFORM_WINDOWS - return GetFileAttributesFromPath(FilePath); -#endif // ZEN_PLATFORM_WINDOWS -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - return GetFileMode(FilePath); -#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - } - - void DownloadLargeBlob(BuildStorageBase& Storage, - const std::filesystem::path& DownloadFolder, - const Oid& BuildId, - const IoHash& ChunkHash, - const std::uint64_t PreferredMultipartChunkSize, - ParallelWork& Work, - WorkerThreadPool& NetworkPool, - std::atomic& DownloadedChunkByteCount, - std::atomic& MultipartAttachmentCount, - std::function&& OnDownloadComplete) - { - ZEN_TRACE_CPU("DownloadLargeBlob"); - - struct WorkloadData - { - TemporaryFile TempFile; - }; - std::shared_ptr Workload(std::make_shared()); - - std::error_code Ec; - Workload->TempFile.CreateTemporary(DownloadFolder, Ec); - if (Ec) - { - throw std::runtime_error( - fmt::format("Failed opening temporary file '{}', reason: ({}) {}", Workload->TempFile.GetPath(), Ec.message(), Ec.value())); - } - std::vector> WorkItems = Storage.GetLargeBuildBlob( - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - [&Work, Workload, &DownloadedChunkByteCount](uint64_t Offset, const IoBuffer& Chunk) { - DownloadedChunkByteCount += Chunk.GetSize(); - - if (!Work.IsAborted()) - { - ZEN_TRACE_CPU("Async_DownloadLargeBlob_OnReceive"); - Workload->TempFile.Write(Chunk.GetView(), Offset); - } - }, - [&Work, Workload, OnDownloadComplete = std::move(OnDownloadComplete)]() { - if (!Work.IsAborted()) - { - ZEN_TRACE_CPU("Async_DownloadLargeBlob_OnComplete"); - - uint64_t PayloadSize = Workload->TempFile.FileSize(); - void* FileHandle = Workload->TempFile.Detach(); - ZEN_ASSERT(FileHandle != nullptr); - IoBuffer Payload(IoBuffer::File, FileHandle, 0, PayloadSize, true); - Payload.SetDeleteOnClose(true); - OnDownloadComplete(std::move(Payload)); - } - }); - if (!WorkItems.empty()) - { - MultipartAttachmentCount++; - } - for (auto& WorkItem : WorkItems) - { - Work.ScheduleWork(NetworkPool, [WorkItem = std::move(WorkItem)](std::atomic& AbortFlag) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("Async_DownloadLargeBlob_Work"); - - WorkItem(); - } - }); - } - } - - CompositeBuffer ValidateBlob(std::atomic& AbortFlag, - IoBuffer&& Payload, - const IoHash& BlobHash, - uint64_t& OutCompressedSize, - uint64_t& OutDecompressedSize) - { - ZEN_TRACE_CPU("ValidateBlob"); - - if (Payload.GetContentType() != ZenContentType::kCompressedBinary) - { - throw std::runtime_error(fmt::format("Blob {} ({} bytes) has unexpected content type '{}'", - BlobHash, - Payload.GetSize(), - ToString(Payload.GetContentType()))); - } - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize); - if (!Compressed) - { - throw std::runtime_error(fmt::format("Blob {} ({} bytes) compressed header is invalid", BlobHash, Payload.GetSize())); - } - if (RawHash != BlobHash) - { - throw std::runtime_error( - fmt::format("Blob {} ({} bytes) compressed header has a mismatching raw hash {}", BlobHash, Payload.GetSize(), RawHash)); - } - - IoHashStream Hash; - bool CouldDecompress = Compressed.DecompressToStream( - 0, - RawSize, - [&AbortFlag, &Hash](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { - ZEN_UNUSED(SourceOffset, SourceSize, Offset); - if (!AbortFlag) - { - for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) - { - Hash.Append(Segment.GetView()); - } - return true; - } - return false; - }); - - if (AbortFlag) - { - return CompositeBuffer{}; - } - - if (!CouldDecompress) - { - throw std::runtime_error( - fmt::format("Blob {} ({} bytes) failed to decompress - header information mismatch", BlobHash, Payload.GetSize())); - } - IoHash ValidateRawHash = Hash.GetHash(); - if (ValidateRawHash != BlobHash) - { - throw std::runtime_error(fmt::format("Blob {} ({} bytes) decompressed hash {} does not match header information", - BlobHash, - Payload.GetSize(), - ValidateRawHash)); - } - OodleCompressor Compressor; - OodleCompressionLevel CompressionLevel; - uint64_t BlockSize; - if (!Compressed.TryGetCompressParameters(Compressor, CompressionLevel, BlockSize)) - { - throw std::runtime_error(fmt::format("Blob {} ({} bytes) failed to get compression details", BlobHash, Payload.GetSize())); - } - OutCompressedSize = Payload.GetSize(); - OutDecompressedSize = RawSize; - if (CompressionLevel == OodleCompressionLevel::None) - { - // Only decompress to composite if we need it for block verification - CompositeBuffer DecompressedComposite = Compressed.DecompressToComposite(); - if (!DecompressedComposite) - { - throw std::runtime_error(fmt::format("Blob {} ({} bytes) failed to decompress to composite", BlobHash, Payload.GetSize())); - } - return DecompressedComposite; - } - return CompositeBuffer{}; - } - - std::filesystem::path TryMoveDownloadedChunk(IoBuffer& BlockBuffer, const std::filesystem::path& Path, bool ForceDiskBased) - { - uint64_t BlockSize = BlockBuffer.GetSize(); - IoBufferFileReference FileRef; - if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && (FileRef.FileChunkSize == BlockSize)) - { - ZEN_TRACE_CPU("MoveTempFullBlock"); - std::error_code Ec; - std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); - if (!Ec) - { - BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - RenameFile(TempBlobPath, Path, Ec); - if (Ec) - { - // Re-open the temp file again - BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); - BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); - BlockBuffer.SetDeleteOnClose(true); - } - else - { - return Path; - } - } - } - - if (ForceDiskBased) - { - // Could not be moved and rather large, lets store it on disk - ZEN_TRACE_CPU("WriteTempFullBlock"); - TemporaryFile::SafeWriteFile(Path, BlockBuffer); - BlockBuffer = {}; - return Path; - } - - return {}; - } - -} // namespace - -class ReadFileCache -{ -public: - // A buffered file reader that provides CompositeBuffer where the buffers are owned and the memory never overwritten - ReadFileCache(std::atomic& OpenReadCount, - std::atomic& CurrentOpenFileCount, - std::atomic& ReadCount, - std::atomic& ReadByteCount, - const std::filesystem::path& Path, - const ChunkedFolderContent& LocalContent, - const ChunkedContentLookup& LocalLookup, - size_t MaxOpenFileCount) - : m_Path(Path) - , m_LocalContent(LocalContent) - , m_LocalLookup(LocalLookup) - , m_OpenReadCount(OpenReadCount) - , m_CurrentOpenFileCount(CurrentOpenFileCount) - , m_ReadCount(ReadCount) - , m_ReadByteCount(ReadByteCount) - { - m_OpenFiles.reserve(MaxOpenFileCount); - } - ~ReadFileCache() { m_OpenFiles.clear(); } - - CompositeBuffer GetRange(uint32_t SequenceIndex, uint64_t Offset, uint64_t Size) - { - ZEN_TRACE_CPU("ReadFileCache::GetRange"); - - auto CacheIt = - std::find_if(m_OpenFiles.begin(), m_OpenFiles.end(), [SequenceIndex](const auto& Lhs) { return Lhs.first == SequenceIndex; }); - if (CacheIt != m_OpenFiles.end()) - { - if (CacheIt != m_OpenFiles.begin()) - { - auto CachedFile(std::move(CacheIt->second)); - m_OpenFiles.erase(CacheIt); - m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(SequenceIndex, std::move(CachedFile))); - } - CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size); - return Result; - } - const uint32_t LocalPathIndex = m_LocalLookup.SequenceIndexFirstPathIndex[SequenceIndex]; - const std::filesystem::path LocalFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred(); - if (Size == m_LocalContent.RawSizes[LocalPathIndex]) - { - IoBuffer Result = IoBufferBuilder::MakeFromFile(LocalFilePath); - return CompositeBuffer(SharedBuffer(Result)); - } - if (m_OpenFiles.size() == m_OpenFiles.capacity()) - { - m_OpenFiles.pop_back(); - } - m_OpenFiles.insert( - m_OpenFiles.begin(), - std::make_pair( - SequenceIndex, - std::make_unique(LocalFilePath, m_OpenReadCount, m_CurrentOpenFileCount, m_ReadCount, m_ReadByteCount))); - CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size); - return Result; - } - -private: - const std::filesystem::path m_Path; - const ChunkedFolderContent& m_LocalContent; - const ChunkedContentLookup& m_LocalLookup; - std::vector>> m_OpenFiles; - std::atomic& m_OpenReadCount; - std::atomic& m_CurrentOpenFileCount; - std::atomic& m_ReadCount; - std::atomic& m_ReadByteCount; -}; - -bool -IsSingleFileChunk(const ChunkedFolderContent& RemoteContent, - const std::vector Locations) -{ - if (Locations.size() == 1) - { - const uint32_t FirstSequenceIndex = Locations[0]->SequenceIndex; - if (RemoteContent.ChunkedContent.ChunkCounts[FirstSequenceIndex] == 1) - { - ZEN_ASSERT_SLOW(Locations[0]->Offset == 0); - return true; - } - } - return false; -} - -IoBuffer -MakeBufferMemoryBased(const CompositeBuffer& PartialBlockBuffer) -{ - ZEN_TRACE_CPU("MakeBufferMemoryBased"); - IoBuffer BlockMemoryBuffer; - std::span Segments = PartialBlockBuffer.GetSegments(); - if (Segments.size() == 1) - { - IoBufferFileReference FileRef = {}; - if (PartialBlockBuffer.GetSegments().front().AsIoBuffer().GetFileReference(FileRef)) - { - BlockMemoryBuffer = UniqueBuffer::Alloc(FileRef.FileChunkSize).MoveToShared().AsIoBuffer(); - BasicFile Reader; - Reader.Attach(FileRef.FileHandle); - auto _ = MakeGuard([&Reader]() { Reader.Detach(); }); - MutableMemoryView ReadMem = BlockMemoryBuffer.GetMutableView(); - Reader.Read(ReadMem.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset); - return BlockMemoryBuffer; - } - else - { - return PartialBlockBuffer.GetSegments().front().AsIoBuffer(); - } - } - else - { - // Not a homogenous memory buffer, read all to memory - - BlockMemoryBuffer = UniqueBuffer::Alloc(PartialBlockBuffer.GetSize()).MoveToShared().AsIoBuffer(); - MutableMemoryView ReadMem = BlockMemoryBuffer.GetMutableView(); - for (const SharedBuffer& Segment : Segments) - { - IoBufferFileReference FileRef = {}; - if (Segment.AsIoBuffer().GetFileReference(FileRef)) - { - BasicFile Reader; - Reader.Attach(FileRef.FileHandle); - auto _ = MakeGuard([&Reader]() { Reader.Detach(); }); - Reader.Read(ReadMem.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset); - ReadMem = ReadMem.Mid(FileRef.FileChunkSize); - } - else - { - ReadMem = ReadMem.CopyFrom(Segment.AsIoBuffer().GetView()); - } - } - return BlockMemoryBuffer; - } -} - -class FilteredRate -{ -public: - FilteredRate() {} - - void Start() - { - if (StartTimeUS == (uint64_t)-1) - { - uint64_t Expected = (uint64_t)-1; - if (StartTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs())) - { - LastTimeUS = StartTimeUS.load(); - } - } - } - void Stop() - { - if (EndTimeUS == (uint64_t)-1) - { - uint64_t Expected = (uint64_t)-1; - EndTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs()); - } - } - - void Update(uint64_t Count) - { - if (LastTimeUS == (uint64_t)-1) - { - return; - } - uint64_t TimeUS = Timer.GetElapsedTimeUs(); - uint64_t TimeDeltaUS = TimeUS - LastTimeUS; - if (TimeDeltaUS >= 2000000) - { - uint64_t Delta = Count - LastCount; - uint64_t PerSecond = (Delta * 1000000) / TimeDeltaUS; - - LastPerSecond = PerSecond; - - LastCount = Count; - - FilteredPerSecond = (PerSecond + (LastPerSecond * 7)) / 8; - - LastTimeUS = TimeUS; - } - } - - uint64_t GetCurrent() const // If Stopped - return total count / total time - { - if (LastTimeUS == (uint64_t)-1) - { - return 0; - } - return FilteredPerSecond; - } - - uint64_t GetElapsedTimeUS() const - { - if (StartTimeUS == (uint64_t)-1) - { - return 0; - } - if (EndTimeUS == (uint64_t)-1) - { - return 0; - } - uint64_t TimeDeltaUS = EndTimeUS - StartTimeUS; - return TimeDeltaUS; - } - - bool IsActive() const { return (StartTimeUS != (uint64_t)-1) && (EndTimeUS == (uint64_t)-1); } - -private: - Stopwatch Timer; - std::atomic StartTimeUS = (uint64_t)-1; - std::atomic EndTimeUS = (uint64_t)-1; - std::atomic LastTimeUS = (uint64_t)-1; - uint64_t LastCount = 0; - uint64_t LastPerSecond = 0; - uint64_t FilteredPerSecond = 0; -}; - -std::filesystem::path -ZenStateFilePath(const std::filesystem::path& ZenFolderPath) -{ - return ZenFolderPath / "current_state.cbo"; -} -std::filesystem::path -ZenTempFolderPath(const std::filesystem::path& ZenFolderPath) -{ - return ZenFolderPath / "tmp"; -} - -////////////////////// BuildsOperationUpdateFolder - -BuildsOperationUpdateFolder::BuildsOperationUpdateFolder(LoggerRef Log, - ProgressBase& Progress, - StorageInstance& Storage, - std::atomic& AbortFlag, - std::atomic& PauseFlag, - WorkerThreadPool& IOWorkerPool, - WorkerThreadPool& NetworkPool, - const Oid& BuildId, - const std::filesystem::path& Path, - const ChunkedFolderContent& LocalContent, - const ChunkedContentLookup& LocalLookup, - const ChunkedFolderContent& RemoteContent, - const ChunkedContentLookup& RemoteLookup, - const std::vector& BlockDescriptions, - const std::vector& LooseChunkHashes, - const Options& Options) -: m_Log(Log) -, m_Progress(Progress) -, m_Storage(Storage) -, m_AbortFlag(AbortFlag) -, m_PauseFlag(PauseFlag) -, m_IOWorkerPool(IOWorkerPool) -, m_NetworkPool(NetworkPool) -, m_BuildId(BuildId) -, m_Path(Path) -, m_LocalContent(LocalContent) -, m_LocalLookup(LocalLookup) -, m_RemoteContent(RemoteContent) -, m_RemoteLookup(RemoteLookup) -, m_BlockDescriptions(BlockDescriptions) -, m_LooseChunkHashes(LooseChunkHashes) -, m_Options(Options) -, m_CacheFolderPath(ZenTempCacheFolderPath(m_Options.ZenFolderPath)) -, m_TempDownloadFolderPath(ZenTempDownloadFolderPath(m_Options.ZenFolderPath)) -, m_TempBlockFolderPath(ZenTempBlockFolderPath(m_Options.ZenFolderPath)) -{ -} - -void -BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) -{ - ZEN_TRACE_CPU("BuildsOperationUpdateFolder::Execute"); - try - { - enum class TaskSteps : uint32_t - { - ScanExistingData, - WriteChunks, - PrepareTarget, - FinalizeTarget, - Cleanup, - StepCount - }; - - auto EndProgress = - MakeGuard([&]() { m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::StepCount, (uint32_t)TaskSteps::StepCount); }); - - m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::ScanExistingData, (uint32_t)TaskSteps::StepCount); - - CreateDirectories(m_CacheFolderPath); - CreateDirectories(m_TempDownloadFolderPath); - CreateDirectories(m_TempBlockFolderPath); - - std::vector> SequenceIndexChunksLeftToWriteCounters(m_RemoteContent.ChunkedContent.SequenceRawHashes.size()); - std::vector RemoteChunkIndexNeedsCopyFromLocalFileFlags(m_RemoteContent.ChunkedContent.ChunkHashes.size()); - std::vector> RemoteChunkIndexNeedsCopyFromSourceFlags(m_RemoteContent.ChunkedContent.ChunkHashes.size()); - - tsl::robin_map CachedChunkHashesFound; - tsl::robin_map CachedSequenceHashesFound; - ScanCacheFolder(CachedChunkHashesFound, CachedSequenceHashesFound); - - tsl::robin_map CachedBlocksFound; - ScanTempBlocksFolder(CachedBlocksFound); - - tsl::robin_map SequenceIndexesLeftToFindToRemoteIndex; - InitializeSequenceCounters(SequenceIndexChunksLeftToWriteCounters, - SequenceIndexesLeftToFindToRemoteIndex, - CachedChunkHashesFound, - CachedSequenceHashesFound); - - std::vector ScavengedContents; - std::vector ScavengedLookups; - std::vector ScavengedPaths; - - std::vector ScavengedSequenceCopyOperations; - uint64_t ScavengedPathsCount = 0; - - if (m_Options.EnableOtherDownloadsScavenging) - { - ZEN_TRACE_CPU("GetScavengedSequences"); - - Stopwatch ScavengeTimer; - - if (!SequenceIndexesLeftToFindToRemoteIndex.empty()) - { - std::vector ScavengeSources = FindScavengeSources(); - ScanScavengeSources(ScavengeSources, ScavengedContents, ScavengedLookups, ScavengedPaths); - if (m_AbortFlag) - { - return; - } - - MatchScavengedSequencesToRemote(ScavengedContents, - ScavengedLookups, - ScavengedPaths, - SequenceIndexesLeftToFindToRemoteIndex, - SequenceIndexChunksLeftToWriteCounters, - ScavengedSequenceCopyOperations, - ScavengedPathsCount); - } - m_CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs(); - } - - uint32_t RemainingChunkCount = 0; - for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < m_RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++) - { - uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex); - if (ChunkWriteCount > 0) - { - RemainingChunkCount++; - } - } - - // Pick up all chunks in current local state - tsl::robin_map RawHashToCopyChunkDataIndex; - std::vector CopyChunkDatas; - - if (m_Options.EnableTargetFolderScavenging) - { - ZEN_TRACE_CPU("GetLocalChunks"); - - Stopwatch LocalTimer; - - ScavengeSourceForChunks(RemainingChunkCount, - RemoteChunkIndexNeedsCopyFromLocalFileFlags, - RawHashToCopyChunkDataIndex, - SequenceIndexChunksLeftToWriteCounters, - m_LocalContent, - m_LocalLookup, - CopyChunkDatas, - uint32_t(-1), - m_CacheMappingStats.LocalChunkMatchingRemoteCount, - m_CacheMappingStats.LocalChunkMatchingRemoteByteCount); - - m_CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs(); - } - - if (m_Options.EnableOtherDownloadsScavenging) - { - ZEN_TRACE_CPU("GetScavengeChunks"); - - Stopwatch ScavengeTimer; - - for (uint32_t ScavengedContentIndex = 0; ScavengedContentIndex < ScavengedContents.size() && (RemainingChunkCount > 0); - ScavengedContentIndex++) - { - const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengedContentIndex]; - const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengedContentIndex]; - - ScavengeSourceForChunks(RemainingChunkCount, - RemoteChunkIndexNeedsCopyFromLocalFileFlags, - RawHashToCopyChunkDataIndex, - SequenceIndexChunksLeftToWriteCounters, - ScavengedContent, - ScavengedLookup, - CopyChunkDatas, - ScavengedContentIndex, - m_CacheMappingStats.ScavengedChunkMatchingRemoteCount, - m_CacheMappingStats.ScavengedChunkMatchingRemoteByteCount); - } - m_CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs(); - } - - if (!m_Options.IsQuiet) - { - if (m_CacheMappingStats.CacheSequenceHashesCount > 0 || m_CacheMappingStats.CacheChunkCount > 0 || - m_CacheMappingStats.CacheBlockCount > 0) - { - ZEN_INFO("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}", - m_CacheMappingStats.CacheSequenceHashesCount, - NiceBytes(m_CacheMappingStats.CacheSequenceHashesByteCount), - m_CacheMappingStats.CacheChunkCount, - NiceBytes(m_CacheMappingStats.CacheChunkByteCount), - m_CacheMappingStats.CacheBlockCount, - NiceBytes(m_CacheMappingStats.CacheBlocksByteCount), - NiceTimeSpanMs(m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000)); - } - - if (m_CacheMappingStats.LocalPathsMatchingSequencesCount > 0 || m_CacheMappingStats.LocalChunkMatchingRemoteCount > 0) - { - ZEN_INFO("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}", - m_CacheMappingStats.LocalPathsMatchingSequencesCount, - NiceBytes(m_CacheMappingStats.LocalPathsMatchingSequencesByteCount), - m_CacheMappingStats.LocalChunkMatchingRemoteCount, - NiceBytes(m_CacheMappingStats.LocalChunkMatchingRemoteByteCount), - NiceTimeSpanMs(m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000)); - } - if (m_CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || m_CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0) - { - ZEN_INFO("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}", - ScavengedPathsCount, - m_CacheMappingStats.ScavengedPathsMatchingSequencesCount, - NiceBytes(m_CacheMappingStats.ScavengedPathsMatchingSequencesByteCount), - m_CacheMappingStats.ScavengedChunkMatchingRemoteCount, - NiceBytes(m_CacheMappingStats.ScavengedChunkMatchingRemoteByteCount), - NiceTimeSpanMs(m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000)); - } - } - - uint64_t BytesToWrite = CalculateBytesToWriteAndFlagNeededChunks(SequenceIndexChunksLeftToWriteCounters, - RemoteChunkIndexNeedsCopyFromLocalFileFlags, - RemoteChunkIndexNeedsCopyFromSourceFlags); - - for (const ScavengedSequenceCopyOperation& ScavengeCopyOp : ScavengedSequenceCopyOperations) - { - BytesToWrite += ScavengeCopyOp.RawSize; - } - - uint64_t BytesToValidate = m_Options.ValidateCompletedSequences ? BytesToWrite : 0; - - uint64_t TotalRequestCount = 0; - uint64_t TotalPartWriteCount = 0; - std::atomic WritePartsComplete = 0; - - tsl::robin_map RemotePathToRemoteIndex; - RemotePathToRemoteIndex.reserve(m_RemoteContent.Paths.size()); - for (uint32_t RemotePathIndex = 0; RemotePathIndex < m_RemoteContent.Paths.size(); RemotePathIndex++) - { - RemotePathToRemoteIndex.insert({m_RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex}); - } - - CheckRequiredDiskSpace(RemotePathToRemoteIndex); - - BlobsExistsResult ExistsResult; - { - ChunkBlockAnalyser BlockAnalyser( - Log(), - m_BlockDescriptions, - ChunkBlockAnalyser::Options{.IsQuiet = m_Options.IsQuiet, - .IsVerbose = m_Options.IsVerbose, - .HostLatencySec = m_Storage.BuildStorageHost.LatencySec, - .HostHighSpeedLatencySec = m_Storage.CacheHost.LatencySec, - .HostMaxRangeCountPerRequest = m_Storage.BuildStorageHost.Caps.MaxRangeCountPerRequest, - .HostHighSpeedMaxRangeCountPerRequest = m_Storage.CacheHost.Caps.MaxRangeCountPerRequest}); - - std::vector NeededBlocks = BlockAnalyser.GetNeeded( - m_RemoteLookup.ChunkHashToChunkIndex, - [&](uint32_t RemoteChunkIndex) -> bool { return RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]; }); - - std::vector FetchBlockIndexes; - std::vector CachedChunkBlockIndexes; - ClassifyCachedAndFetchBlocks(NeededBlocks, CachedBlocksFound, TotalPartWriteCount, CachedChunkBlockIndexes, FetchBlockIndexes); - - std::vector NeededLooseChunkIndexes = DetermineNeededLooseChunkIndexes(SequenceIndexChunksLeftToWriteCounters, - RemoteChunkIndexNeedsCopyFromLocalFileFlags, - RemoteChunkIndexNeedsCopyFromSourceFlags); - - ExistsResult = QueryBlobCacheExists(NeededLooseChunkIndexes, FetchBlockIndexes); - - std::vector BlockPartialDownloadModes = - DeterminePartialDownloadModes(ExistsResult); - ZEN_ASSERT(BlockPartialDownloadModes.size() == m_BlockDescriptions.size()); - - ChunkBlockAnalyser::BlockResult PartialBlocks = - BlockAnalyser.CalculatePartialBlockDownloads(NeededBlocks, BlockPartialDownloadModes); - - TotalRequestCount += NeededLooseChunkIndexes.size(); - TotalPartWriteCount += NeededLooseChunkIndexes.size(); - TotalRequestCount += PartialBlocks.BlockRanges.size(); - TotalPartWriteCount += PartialBlocks.BlockRanges.size(); - TotalRequestCount += PartialBlocks.FullBlockIndexes.size(); - TotalPartWriteCount += PartialBlocks.FullBlockIndexes.size(); - - std::vector LooseChunkHashWorks = - BuildLooseChunkHashWorks(NeededLooseChunkIndexes, SequenceIndexChunksLeftToWriteCounters); - - ZEN_TRACE_CPU("WriteChunks"); - - m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::WriteChunks, (uint32_t)TaskSteps::StepCount); - - Stopwatch WriteTimer; - - FilteredRate FilteredDownloadedBytesPerSecond; - FilteredRate FilteredWrittenBytesPerSecond; - - std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Writing"); - ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - - TotalPartWriteCount += CopyChunkDatas.size(); - TotalPartWriteCount += ScavengedSequenceCopyOperations.size(); - - BufferedWriteFileCache WriteCache; - - WriteChunksContext Context{.Work = Work, - .WriteCache = WriteCache, - .SequenceIndexChunksLeftToWriteCounters = SequenceIndexChunksLeftToWriteCounters, - .RemoteChunkIndexNeedsCopyFromSourceFlags = RemoteChunkIndexNeedsCopyFromSourceFlags, - .WritePartsComplete = WritePartsComplete, - .TotalPartWriteCount = TotalPartWriteCount, - .TotalRequestCount = TotalRequestCount, - .ExistsResult = ExistsResult, - .FilteredDownloadedBytesPerSecond = FilteredDownloadedBytesPerSecond, - .FilteredWrittenBytesPerSecond = FilteredWrittenBytesPerSecond}; - - ScheduleScavengedSequenceWrites(Context, ScavengedSequenceCopyOperations, ScavengedContents, ScavengedPaths); - ScheduleLooseChunkWrites(Context, LooseChunkHashWorks); - - std::unique_ptr CloneQuery = - m_Options.AllowFileClone ? GetCloneQueryInterface(m_CacheFolderPath) : nullptr; - - ScheduleLocalChunkCopies(Context, CopyChunkDatas, CloneQuery.get(), ScavengedContents, ScavengedLookups, ScavengedPaths); - ScheduleCachedBlockWrites(Context, CachedChunkBlockIndexes); - SchedulePartialBlockDownloads(Context, PartialBlocks); - ScheduleFullBlockDownloads(Context, PartialBlocks.FullBlockIndexes); - - { - ZEN_TRACE_CPU("WriteChunks_Wait"); - - Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(PendingWork); - uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() + - m_DownloadStats.DownloadedBlockByteCount.load() + - +m_DownloadStats.DownloadedPartialBlockByteCount.load(); - FilteredWrittenBytesPerSecond.Update(m_DiskStats.WriteByteCount.load()); - FilteredDownloadedBytesPerSecond.Update(DownloadedBytes); - std::string DownloadRateString = - (m_DownloadStats.RequestsCompleteCount == TotalRequestCount) - ? "" - : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8)); - std::string CloneDetails; - if (m_DiskStats.CloneCount.load() > 0) - { - CloneDetails = fmt::format(" ({} cloned)", NiceBytes(m_DiskStats.CloneByteCount.load())); - } - std::string WriteDetails = fmt::format(" {}/{} ({}B/s) written{}", - NiceBytes(m_WrittenChunkByteCount.load()), - NiceBytes(BytesToWrite), - NiceNum(FilteredWrittenBytesPerSecond.GetCurrent()), - CloneDetails); - - std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}", - m_DownloadStats.RequestsCompleteCount.load(), - TotalRequestCount, - NiceBytes(DownloadedBytes), - DownloadRateString, - WriteDetails); - - std::string Task; - if ((m_WrittenChunkByteCount < BytesToWrite) || (BytesToValidate == 0)) - { - Task = "Writing chunks "; - } - else - { - Task = "Verifying chunks "; - } - - ProgressBar->UpdateState({.Task = Task, - .Details = Details, - .TotalCount = (BytesToWrite + BytesToValidate), - .RemainingCount = ((BytesToWrite + BytesToValidate) - - (m_WrittenChunkByteCount.load() + m_ValidatedChunkByteCount.load())), - .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }); - } - - CloneQuery.reset(); - - FilteredWrittenBytesPerSecond.Stop(); - FilteredDownloadedBytesPerSecond.Stop(); - - ProgressBar->Finish(); - if (m_AbortFlag) - { - return; - } - - VerifyWriteChunksComplete(SequenceIndexChunksLeftToWriteCounters, BytesToWrite, BytesToValidate); - - const uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() + - m_DownloadStats.DownloadedBlockByteCount.load() + - m_DownloadStats.DownloadedPartialBlockByteCount.load(); - if (!m_Options.IsQuiet) - { - std::string CloneDetails; - if (m_DiskStats.CloneCount.load() > 0) - { - CloneDetails = fmt::format(" ({} cloned)", NiceBytes(m_DiskStats.CloneByteCount.load())); - } - ZEN_INFO("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s){} in {}. Completed in {}", - NiceBytes(DownloadedBytes), - NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)), - NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000), - NiceBytes(m_WrittenChunkByteCount.load()), - NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), m_DiskStats.WriteByteCount.load())), - CloneDetails, - NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000), - NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs())); - } - - m_WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs(); - m_WriteChunkStats.DownloadTimeUs = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(); - m_WriteChunkStats.WriteTimeUs = FilteredWrittenBytesPerSecond.GetElapsedTimeUS(); - } - - m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::PrepareTarget, (uint32_t)TaskSteps::StepCount); - - if (m_AbortFlag) - { - return; - } - - LocalPathCategorization Categorization = CategorizeLocalPaths(RemotePathToRemoteIndex); - - if (m_AbortFlag) - { - return; - } - - std::atomic CachedCount = 0; - std::atomic CachedByteCount = 0; - ScheduleLocalFileCaching(Categorization.FilesToCache, CachedCount, CachedByteCount); - if (m_AbortFlag) - { - return; - } - - ZEN_DEBUG( - "Local state prep: Match: {}, PathMismatch: {}, HashMismatch: {}, Cached: {} ({}), Skipped: {}, " - "Delete: {}", - Categorization.MatchCount, - Categorization.PathMismatchCount, - Categorization.HashMismatchCount, - CachedCount.load(), - NiceBytes(CachedByteCount.load()), - Categorization.SkippedCount, - Categorization.DeleteCount); - - m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::FinalizeTarget, (uint32_t)TaskSteps::StepCount); - - if (m_Options.WipeTargetFolder) - { - ZEN_TRACE_CPU("WipeTarget"); - Stopwatch Timer; - - // Clean target folder - if (!CleanDirectory(Log(), - m_Progress, - m_IOWorkerPool, - m_AbortFlag, - m_PauseFlag, - m_Options.IsQuiet, - m_Path, - m_Options.ExcludeFolders)) - { - ZEN_WARN("Some files in {} could not be removed", m_Path); - } - m_RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs(); - } - - if (m_AbortFlag) - { - return; - } - - { - ZEN_TRACE_CPU("FinalizeTree"); - - Stopwatch Timer; - - std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Rebuild State"); - ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - - OutLocalFolderState.Paths.resize(m_RemoteContent.Paths.size()); - OutLocalFolderState.RawSizes.resize(m_RemoteContent.Paths.size()); - OutLocalFolderState.Attributes.resize(m_RemoteContent.Paths.size()); - OutLocalFolderState.ModificationTicks.resize(m_RemoteContent.Paths.size()); - - std::atomic DeletedCount = 0; - std::atomic TargetsComplete = 0; - - ScheduleLocalFileRemovals(Work, Categorization.RemoveLocalPathIndexes, DeletedCount); - - std::vector Targets = BuildSortedFinalizeTargets(); - - ScheduleTargetFinalization(Work, - Targets, - Categorization.SequenceHashToLocalPathIndex, - Categorization.RemotePathIndexToLocalPathIndex, - OutLocalFolderState, - TargetsComplete); - - { - ZEN_TRACE_CPU("FinalizeTree_Wait"); - - Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(PendingWork); - const uint64_t WorkTotal = Targets.size() + Categorization.RemoveLocalPathIndexes.size(); - const uint64_t WorkComplete = TargetsComplete.load() + DeletedCount.load(); - std::string Details = fmt::format("{}/{} files", WorkComplete, WorkTotal); - ProgressBar->UpdateState({.Task = "Rebuilding state ", - .Details = Details, - .TotalCount = gsl::narrow(WorkTotal), - .RemainingCount = gsl::narrow(WorkTotal - WorkComplete), - .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }); - } - - m_RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs = Timer.GetElapsedTimeUs(); - ProgressBar->Finish(); - } - m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::Cleanup, (uint32_t)TaskSteps::StepCount); - } - catch (const std::exception&) - { - m_AbortFlag = true; - throw; - } -} - -void -BuildsOperationUpdateFolder::ScanCacheFolder(tsl::robin_map& OutCachedChunkHashesFound, - tsl::robin_map& OutCachedSequenceHashesFound) -{ - ZEN_TRACE_CPU("ScanCacheFolder"); - - Stopwatch CacheTimer; - - DirectoryContent CacheDirContent; - GetDirectoryContent(m_CacheFolderPath, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, CacheDirContent); - for (size_t Index = 0; Index < CacheDirContent.Files.size(); Index++) - { - if (m_Options.EnableTargetFolderScavenging) - { - IoHash FileHash; - if (IoHash::TryParse(CacheDirContent.Files[Index].filename().string(), FileHash)) - { - if (auto ChunkIt = m_RemoteLookup.ChunkHashToChunkIndex.find(FileHash); - ChunkIt != m_RemoteLookup.ChunkHashToChunkIndex.end()) - { - const uint32_t ChunkIndex = ChunkIt->second; - const uint64_t ChunkSize = m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - if (ChunkSize == CacheDirContent.FileSizes[Index]) - { - OutCachedChunkHashesFound.insert({FileHash, ChunkIndex}); - m_CacheMappingStats.CacheChunkCount++; - m_CacheMappingStats.CacheChunkByteCount += ChunkSize; - continue; - } - } - else if (auto SequenceIt = m_RemoteLookup.RawHashToSequenceIndex.find(FileHash); - SequenceIt != m_RemoteLookup.RawHashToSequenceIndex.end()) - { - const uint32_t SequenceIndex = SequenceIt->second; - const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; - const uint64_t SequenceSize = m_RemoteContent.RawSizes[PathIndex]; - if (SequenceSize == CacheDirContent.FileSizes[Index]) - { - OutCachedSequenceHashesFound.insert({FileHash, SequenceIndex}); - m_CacheMappingStats.CacheSequenceHashesCount++; - m_CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize; - - const std::filesystem::path CacheFilePath = - GetFinalChunkedSequenceFileName(m_CacheFolderPath, - m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); - ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); - - continue; - } - } - } - } - std::error_code Ec = TryRemoveFile(CacheDirContent.Files[Index]); - if (Ec) - { - ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", CacheDirContent.Files[Index], Ec.value(), Ec.message()); - } - } - m_CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs(); -} - -void -BuildsOperationUpdateFolder::ScanTempBlocksFolder(tsl::robin_map& OutCachedBlocksFound) -{ - ZEN_TRACE_CPU("ScanTempBlocksFolder"); - - Stopwatch CacheTimer; - - tsl::robin_map AllBlockSizes; - AllBlockSizes.reserve(m_BlockDescriptions.size()); - for (uint32_t BlockIndex = 0; BlockIndex < m_BlockDescriptions.size(); BlockIndex++) - { - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; - AllBlockSizes.insert({BlockDescription.BlockHash, BlockIndex}); - } - - DirectoryContent BlockDirContent; - GetDirectoryContent(m_TempBlockFolderPath, - DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, - BlockDirContent); - OutCachedBlocksFound.reserve(BlockDirContent.Files.size()); - for (size_t Index = 0; Index < BlockDirContent.Files.size(); Index++) - { - if (m_Options.EnableTargetFolderScavenging) - { - IoHash FileHash; - if (IoHash::TryParse(BlockDirContent.Files[Index].filename().string(), FileHash)) - { - if (auto BlockIt = AllBlockSizes.find(FileHash); BlockIt != AllBlockSizes.end()) - { - const uint32_t BlockIndex = BlockIt->second; - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; - uint64_t BlockSize = CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize; - for (uint64_t ChunkSize : BlockDescription.ChunkCompressedLengths) - { - BlockSize += ChunkSize; - } - - if (BlockSize == BlockDirContent.FileSizes[Index]) - { - OutCachedBlocksFound.insert({FileHash, BlockIndex}); - m_CacheMappingStats.CacheBlockCount++; - m_CacheMappingStats.CacheBlocksByteCount += BlockSize; - continue; - } - } - } - } - std::error_code Ec = TryRemoveFile(BlockDirContent.Files[Index]); - if (Ec) - { - ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", BlockDirContent.Files[Index], Ec.value(), Ec.message()); - } - } - - m_CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs(); -} - -void -BuildsOperationUpdateFolder::InitializeSequenceCounters(std::vector>& OutSequenceCounters, - tsl::robin_map& OutSequencesLeftToFind, - const tsl::robin_map& CachedChunkHashesFound, - const tsl::robin_map& CachedSequenceHashesFound) -{ - if (m_Options.EnableTargetFolderScavenging) - { - // Pick up all whole files we can use from current local state - ZEN_TRACE_CPU("GetLocalSequences"); - - std::vector MissingSequenceIndexes = ScanTargetFolder(CachedChunkHashesFound, CachedSequenceHashesFound); - - for (uint32_t RemoteSequenceIndex : MissingSequenceIndexes) - { - // We must write the sequence - const uint32_t ChunkCount = m_RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; - const IoHash& RemoteSequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - OutSequenceCounters[RemoteSequenceIndex] = ChunkCount; - OutSequencesLeftToFind.insert({RemoteSequenceRawHash, RemoteSequenceIndex}); - } - } - else - { - for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < m_RemoteContent.ChunkedContent.SequenceRawHashes.size(); - RemoteSequenceIndex++) - { - OutSequenceCounters[RemoteSequenceIndex] = m_RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; - } - } -} - -void -BuildsOperationUpdateFolder::MatchScavengedSequencesToRemote(std::span Contents, - std::span Lookups, - std::span Paths, - tsl::robin_map& InOutSequencesLeftToFind, - std::vector>& InOutSequenceCounters, - std::vector& OutCopyOperations, - uint64_t& OutScavengedPathsCount) -{ - for (uint32_t ScavengedContentIndex = 0; ScavengedContentIndex < Contents.size() && !InOutSequencesLeftToFind.empty(); - ScavengedContentIndex++) - { - const std::filesystem::path& ScavengePath = Paths[ScavengedContentIndex]; - if (ScavengePath.empty()) - { - continue; - } - const ChunkedFolderContent& ScavengedLocalContent = Contents[ScavengedContentIndex]; - const ChunkedContentLookup& ScavengedLookup = Lookups[ScavengedContentIndex]; - - for (uint32_t ScavengedSequenceIndex = 0; ScavengedSequenceIndex < ScavengedLocalContent.ChunkedContent.SequenceRawHashes.size(); - ScavengedSequenceIndex++) - { - const IoHash& SequenceRawHash = ScavengedLocalContent.ChunkedContent.SequenceRawHashes[ScavengedSequenceIndex]; - auto It = InOutSequencesLeftToFind.find(SequenceRawHash); - if (It == InOutSequencesLeftToFind.end()) - { - continue; - } - const uint32_t RemoteSequenceIndex = It->second; - const uint64_t RawSize = m_RemoteContent.RawSizes[m_RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]]; - ZEN_ASSERT(RawSize > 0); - - const uint32_t ScavengedPathIndex = ScavengedLookup.SequenceIndexFirstPathIndex[ScavengedSequenceIndex]; - ZEN_ASSERT_SLOW(IsFile((ScavengePath / ScavengedLocalContent.Paths[ScavengedPathIndex]).make_preferred())); - - OutCopyOperations.push_back({.ScavengedContentIndex = ScavengedContentIndex, - .ScavengedPathIndex = ScavengedPathIndex, - .RemoteSequenceIndex = RemoteSequenceIndex, - .RawSize = RawSize}); - - InOutSequencesLeftToFind.erase(SequenceRawHash); - InOutSequenceCounters[RemoteSequenceIndex] = 0; - - m_CacheMappingStats.ScavengedPathsMatchingSequencesCount++; - m_CacheMappingStats.ScavengedPathsMatchingSequencesByteCount += RawSize; - } - OutScavengedPathsCount++; - } -} - -uint64_t -BuildsOperationUpdateFolder::CalculateBytesToWriteAndFlagNeededChunks(std::span> SequenceCounters, - const std::vector& NeedsCopyFromLocalFileFlags, - std::span> OutNeedsCopyFromSourceFlags) -{ - uint64_t BytesToWrite = 0; - for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < m_RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++) - { - const uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceCounters, RemoteChunkIndex); - if (ChunkWriteCount > 0) - { - BytesToWrite += m_RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] * ChunkWriteCount; - if (!NeedsCopyFromLocalFileFlags[RemoteChunkIndex]) - { - OutNeedsCopyFromSourceFlags[RemoteChunkIndex] = true; - } - } - } - return BytesToWrite; -} - -void -BuildsOperationUpdateFolder::ClassifyCachedAndFetchBlocks(std::span NeededBlocks, - const tsl::robin_map& CachedBlocksFound, - uint64_t& TotalPartWriteCount, - std::vector& OutCachedChunkBlockIndexes, - std::vector& OutFetchBlockIndexes) -{ - ZEN_TRACE_CPU("BlockCacheFileExists"); - for (const ChunkBlockAnalyser::NeededBlock& NeededBlock : NeededBlocks) - { - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[NeededBlock.BlockIndex]; - bool UsingCachedBlock = false; - if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end()) - { - TotalPartWriteCount++; - - std::filesystem::path BlockPath = m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString(); - if (IsFile(BlockPath)) - { - OutCachedChunkBlockIndexes.push_back(NeededBlock.BlockIndex); - UsingCachedBlock = true; - } - } - if (!UsingCachedBlock) - { - OutFetchBlockIndexes.push_back(NeededBlock.BlockIndex); - } - } -} - -std::vector -BuildsOperationUpdateFolder::DetermineNeededLooseChunkIndexes(std::span> SequenceCounters, - const std::vector& NeedsCopyFromLocalFileFlags, - std::span> NeedsCopyFromSourceFlags) -{ - std::vector NeededLooseChunkIndexes; - NeededLooseChunkIndexes.reserve(m_LooseChunkHashes.size()); - for (uint32_t LooseChunkIndex = 0; LooseChunkIndex < m_LooseChunkHashes.size(); LooseChunkIndex++) - { - const IoHash& ChunkHash = m_LooseChunkHashes[LooseChunkIndex]; - auto RemoteChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); - ZEN_ASSERT(RemoteChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end()); - const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second; - - if (NeedsCopyFromLocalFileFlags[RemoteChunkIndex]) - { - if (m_Options.IsVerbose) - { - ZEN_INFO("Skipping chunk {} due to cache reuse", m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]); - } - continue; - } - - bool NeedsCopy = true; - if (NeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false)) - { - const uint64_t WriteCount = GetChunkWriteCount(SequenceCounters, RemoteChunkIndex); - if (WriteCount == 0) - { - if (m_Options.IsVerbose) - { - ZEN_INFO("Skipping chunk {} due to cache reuse", m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]); - } - } - else - { - NeededLooseChunkIndexes.push_back(LooseChunkIndex); - } - } - } - return NeededLooseChunkIndexes; -} - -BuildsOperationUpdateFolder::BlobsExistsResult -BuildsOperationUpdateFolder::QueryBlobCacheExists(std::span NeededLooseChunkIndexes, - std::span FetchBlockIndexes) -{ - BlobsExistsResult Result; - if (!m_Storage.CacheStorage) - { - return Result; - } - - ZEN_TRACE_CPU("BlobCacheExistCheck"); - Stopwatch Timer; - - std::vector BlobHashes; - BlobHashes.reserve(NeededLooseChunkIndexes.size() + FetchBlockIndexes.size()); - - for (const uint32_t LooseChunkIndex : NeededLooseChunkIndexes) - { - BlobHashes.push_back(m_LooseChunkHashes[LooseChunkIndex]); - } - - for (uint32_t BlockIndex : FetchBlockIndexes) - { - BlobHashes.push_back(m_BlockDescriptions[BlockIndex].BlockHash); - } - - const std::vector CacheExistsResult = m_Storage.CacheStorage->BlobsExists(m_BuildId, BlobHashes); - - if (CacheExistsResult.size() == BlobHashes.size()) - { - Result.ExistingBlobs.reserve(CacheExistsResult.size()); - for (size_t BlobIndex = 0; BlobIndex < BlobHashes.size(); BlobIndex++) - { - if (CacheExistsResult[BlobIndex].HasBody) - { - Result.ExistingBlobs.insert(BlobHashes[BlobIndex]); - } - } - } - Result.ElapsedTimeMs = Timer.GetElapsedTimeMs(); - if (!Result.ExistingBlobs.empty() && !m_Options.IsQuiet) - { - ZEN_INFO("Remote cache : Found {} out of {} needed blobs in {}", - Result.ExistingBlobs.size(), - BlobHashes.size(), - NiceTimeSpanMs(Result.ElapsedTimeMs)); - } - return Result; -} - -std::vector -BuildsOperationUpdateFolder::DeterminePartialDownloadModes(const BlobsExistsResult& ExistsResult) -{ - std::vector Modes; - - if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::Off) - { - Modes.resize(m_BlockDescriptions.size(), ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); - return Modes; - } - - const bool MultiRangeCache = m_Storage.CacheHost.Caps.MaxRangeCountPerRequest > 1; - const bool MultiRangeBuild = m_Storage.BuildStorageHost.Caps.MaxRangeCountPerRequest > 1; - ChunkBlockAnalyser::EPartialBlockDownloadMode CachePartialDownloadMode = - MultiRangeCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed - : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange; - ChunkBlockAnalyser::EPartialBlockDownloadMode CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::Off; - - switch (m_Options.PartialBlockRequestMode) - { - case EPartialBlockRequestMode::Off: - break; - case EPartialBlockRequestMode::ZenCacheOnly: - CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::Off; - break; - case EPartialBlockRequestMode::Mixed: - CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange; - break; - case EPartialBlockRequestMode::All: - CloudPartialDownloadMode = MultiRangeBuild ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange - : ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange; - break; - default: - ZEN_ASSERT(false); - break; - } - - Modes.reserve(m_BlockDescriptions.size()); - for (uint32_t BlockIndex = 0; BlockIndex < m_BlockDescriptions.size(); BlockIndex++) - { - const bool BlockExistInCache = ExistsResult.ExistingBlobs.contains(m_BlockDescriptions[BlockIndex].BlockHash); - Modes.push_back(BlockExistInCache ? CachePartialDownloadMode : CloudPartialDownloadMode); - } - return Modes; -} - -std::vector -BuildsOperationUpdateFolder::BuildLooseChunkHashWorks(std::span NeededLooseChunkIndexes, - std::span> SequenceCounters) -{ - std::vector LooseChunkHashWorks; - LooseChunkHashWorks.reserve(NeededLooseChunkIndexes.size()); - for (uint32_t LooseChunkIndex : NeededLooseChunkIndexes) - { - const IoHash& ChunkHash = m_LooseChunkHashes[LooseChunkIndex]; - auto RemoteChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); - ZEN_ASSERT(RemoteChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end()); - const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second; - - std::vector ChunkTargetPtrs = - GetRemainingChunkTargets(SequenceCounters, RemoteChunkIndex); - - ZEN_ASSERT(!ChunkTargetPtrs.empty()); - LooseChunkHashWorks.push_back(LooseChunkHashWorkData{.ChunkTargetPtrs = ChunkTargetPtrs, .RemoteChunkIndex = RemoteChunkIndex}); - } - return LooseChunkHashWorks; -} - -void -BuildsOperationUpdateFolder::VerifyWriteChunksComplete(std::span> SequenceCounters, - uint64_t BytesToWrite, - uint64_t BytesToValidate) -{ - uint32_t RawSequencesMissingWriteCount = 0; - for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceCounters.size(); SequenceIndex++) - { - const auto& Counter = SequenceCounters[SequenceIndex]; - if (Counter.load() != 0) - { - RawSequencesMissingWriteCount++; - const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; - const std::filesystem::path& IncompletePath = m_RemoteContent.Paths[PathIndex]; - ZEN_ASSERT(!IncompletePath.empty()); - const uint32_t ExpectedSequenceCount = m_RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; - if (!m_Options.IsQuiet) - { - ZEN_INFO("{}: Max count {}, Current count {}", IncompletePath, ExpectedSequenceCount, Counter.load()); - } - ZEN_ASSERT(Counter.load() <= ExpectedSequenceCount); - } - } - ZEN_ASSERT(RawSequencesMissingWriteCount == 0); - ZEN_ASSERT(m_WrittenChunkByteCount == BytesToWrite); - ZEN_ASSERT(m_ValidatedChunkByteCount == BytesToValidate); -} - -std::vector -BuildsOperationUpdateFolder::BuildSortedFinalizeTargets() -{ - std::vector Targets; - Targets.reserve(m_RemoteContent.Paths.size()); - for (uint32_t RemotePathIndex = 0; RemotePathIndex < m_RemoteContent.Paths.size(); RemotePathIndex++) - { - Targets.push_back(FinalizeTarget{.RawHash = m_RemoteContent.RawHashes[RemotePathIndex], .RemotePathIndex = RemotePathIndex}); - } - std::sort(Targets.begin(), Targets.end(), [](const FinalizeTarget& Lhs, const FinalizeTarget& Rhs) { - return std::tie(Lhs.RawHash, Lhs.RemotePathIndex) < std::tie(Rhs.RawHash, Rhs.RemotePathIndex); - }); - return Targets; -} - -void -BuildsOperationUpdateFolder::ScanScavengeSources(std::span Sources, - std::vector& OutContents, - std::vector& OutLookups, - std::vector& OutPaths) -{ - ZEN_TRACE_CPU("ScanScavengeSources"); - - const size_t ScavengePathCount = Sources.size(); - OutContents.resize(ScavengePathCount); - OutLookups.resize(ScavengePathCount); - OutPaths.resize(ScavengePathCount); - - std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Scavenging"); - - ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - - std::atomic PathsFound(0); - std::atomic ChunksFound(0); - std::atomic PathsScavenged(0); - - for (size_t ScavengeIndex = 0; ScavengeIndex < ScavengePathCount; ScavengeIndex++) - { - Work.ScheduleWork(m_IOWorkerPool, - [this, &Sources, &OutContents, &OutPaths, &OutLookups, &PathsFound, &ChunksFound, &PathsScavenged, ScavengeIndex]( - std::atomic&) { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Async_FindScavengeContent"); - - const ScavengeSource& Source = Sources[ScavengeIndex]; - ChunkedFolderContent& ScavengedLocalContent = OutContents[ScavengeIndex]; - ChunkedContentLookup& ScavengedLookup = OutLookups[ScavengeIndex]; - - if (FindScavengeContent(Source, ScavengedLocalContent, ScavengedLookup)) - { - OutPaths[ScavengeIndex] = Source.Path; - PathsFound += ScavengedLocalContent.Paths.size(); - ChunksFound += ScavengedLocalContent.ChunkedContent.ChunkHashes.size(); - } - else - { - OutPaths[ScavengeIndex].clear(); - } - PathsScavenged++; - } - }); - } - { - ZEN_TRACE_CPU("ScavengeScan_Wait"); - - Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(PendingWork); - std::string Details = fmt::format("{}/{} scanned. {} paths and {} chunks found for scavenging", - PathsScavenged.load(), - ScavengePathCount, - PathsFound.load(), - ChunksFound.load()); - ProgressBar->UpdateState({.Task = "Scavenging ", - .Details = Details, - .TotalCount = ScavengePathCount, - .RemainingCount = ScavengePathCount - PathsScavenged.load(), - .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }); - } - - ProgressBar->Finish(); -} - -BuildsOperationUpdateFolder::LocalPathCategorization -BuildsOperationUpdateFolder::CategorizeLocalPaths(const tsl::robin_map& RemotePathToRemoteIndex) -{ - ZEN_TRACE_CPU("PrepareTarget"); - - LocalPathCategorization Result; - tsl::robin_set CachedRemoteSequences; - - Result.RemotePathIndexToLocalPathIndex.reserve(m_RemoteContent.Paths.size()); - - for (uint32_t LocalPathIndex = 0; LocalPathIndex < m_LocalContent.Paths.size(); LocalPathIndex++) - { - if (m_AbortFlag) - { - break; - } - const IoHash& RawHash = m_LocalContent.RawHashes[LocalPathIndex]; - const std::filesystem::path& LocalPath = m_LocalContent.Paths[LocalPathIndex]; - - ZEN_ASSERT_SLOW(IsFile((m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred())); - - if (m_Options.EnableTargetFolderScavenging) - { - if (!m_Options.WipeTargetFolder) - { - // Check if it is already in the correct place - if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string()); - RemotePathIt != RemotePathToRemoteIndex.end()) - { - const uint32_t RemotePathIndex = RemotePathIt->second; - if (m_RemoteContent.RawHashes[RemotePathIndex] == RawHash) - { - // It is already in it's correct place - Result.RemotePathIndexToLocalPathIndex[RemotePathIndex] = LocalPathIndex; - Result.SequenceHashToLocalPathIndex.insert({RawHash, LocalPathIndex}); - Result.MatchCount++; - continue; - } - else - { - Result.HashMismatchCount++; - } - } - else - { - Result.PathMismatchCount++; - } - } - - // Do we need it? - if (m_RemoteLookup.RawHashToSequenceIndex.contains(RawHash)) - { - if (!CachedRemoteSequences.contains(RawHash)) - { - // We need it, make sure we move it to the cache - Result.FilesToCache.push_back(LocalPathIndex); - CachedRemoteSequences.insert(RawHash); - continue; - } - else - { - Result.SkippedCount++; - } - } - } - - if (!m_Options.WipeTargetFolder) - { - // Explicitly delete the unneeded local file - Result.RemoveLocalPathIndexes.push_back(LocalPathIndex); - Result.DeleteCount++; - } - } - - return Result; -} - -void -BuildsOperationUpdateFolder::ScheduleLocalFileCaching(std::span FilesToCache, - std::atomic& OutCachedCount, - std::atomic& OutCachedByteCount) -{ - ZEN_TRACE_CPU("CopyToCache"); - - std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Cache Local Data"); - ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - - for (uint32_t LocalPathIndex : FilesToCache) - { - if (m_AbortFlag) - { - break; - } - Work.ScheduleWork(m_IOWorkerPool, [this, &OutCachedCount, &OutCachedByteCount, LocalPathIndex](std::atomic&) { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Async_CopyToCache"); - - const IoHash& RawHash = m_LocalContent.RawHashes[LocalPathIndex]; - const std::filesystem::path& LocalPath = m_LocalContent.Paths[LocalPathIndex]; - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath)); - const std::filesystem::path LocalFilePath = (m_Path / LocalPath).make_preferred(); - - std::error_code Ec = RenameFileWithRetry(LocalFilePath, CacheFilePath); - if (Ec) - { - ZEN_WARN("Failed to move file from '{}' to '{}', reason: ({}) {}, retrying...", - LocalFilePath, - CacheFilePath, - Ec.value(), - Ec.message()); - Ec = RenameFileWithRetry(LocalFilePath, CacheFilePath); - if (Ec) - { - throw std::system_error(std::error_code(Ec.value(), std::system_category()), - fmt::format("Failed to file from '{}' to '{}', reason: ({}) {}", - LocalFilePath, - CacheFilePath, - Ec.value(), - Ec.message())); - } - } - - OutCachedCount++; - OutCachedByteCount += m_LocalContent.RawSizes[LocalPathIndex]; - } - }); - } - - { - ZEN_TRACE_CPU("CopyToCache_Wait"); - - Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(PendingWork); - const uint64_t WorkTotal = FilesToCache.size(); - const uint64_t WorkComplete = OutCachedCount.load(); - std::string Details = fmt::format("{}/{} ({}) files", WorkComplete, WorkTotal, NiceBytes(OutCachedByteCount)); - ProgressBar->UpdateState({.Task = "Caching local ", - .Details = Details, - .TotalCount = gsl::narrow(WorkTotal), - .RemainingCount = gsl::narrow(WorkTotal - WorkComplete), - .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }); - } - - ProgressBar->Finish(); -} - -void -BuildsOperationUpdateFolder::ScheduleScavengedSequenceWrites(WriteChunksContext& Context, - std::span CopyOperations, - const std::vector& ScavengedContents, - const std::vector& ScavengedPaths) -{ - for (uint32_t ScavengeOpIndex = 0; ScavengeOpIndex < CopyOperations.size(); ScavengeOpIndex++) - { - if (m_AbortFlag) - { - break; - } - Context.Work.ScheduleWork( - m_IOWorkerPool, - [this, &Context, &CopyOperations, &ScavengedContents, &ScavengedPaths, ScavengeOpIndex](std::atomic&) { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Async_WriteScavenged"); - - Context.FilteredWrittenBytesPerSecond.Start(); - - const ScavengedSequenceCopyOperation& ScavengeOp = CopyOperations[ScavengeOpIndex]; - const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengeOp.ScavengedContentIndex]; - const std::filesystem::path& ScavengeRootPath = ScavengedPaths[ScavengeOp.ScavengedContentIndex]; - - WriteScavengedSequenceToCache(ScavengeRootPath, ScavengedContent, ScavengeOp); - - if (Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount) - { - Context.FilteredWrittenBytesPerSecond.Stop(); - } - } - }); - } -} - -void -BuildsOperationUpdateFolder::ScheduleLooseChunkWrites(WriteChunksContext& Context, std::vector& LooseChunkHashWorks) -{ - for (uint32_t LooseChunkHashWorkIndex = 0; LooseChunkHashWorkIndex < LooseChunkHashWorks.size(); LooseChunkHashWorkIndex++) - { - if (m_AbortFlag) - { - break; - } - - Context.Work.ScheduleWork( - m_IOWorkerPool, - [this, &Context, &LooseChunkHashWorks, LooseChunkHashWorkIndex](std::atomic&) { - ZEN_TRACE_CPU("Async_ReadPreDownloadedChunk"); - if (!m_AbortFlag) - { - LooseChunkHashWorkData& LooseChunkHashWork = LooseChunkHashWorks[LooseChunkHashWorkIndex]; - const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex; - WriteLooseChunk(RemoteChunkIndex, - Context.ExistsResult, - Context.SequenceIndexChunksLeftToWriteCounters, - Context.WritePartsComplete, - std::move(LooseChunkHashWork.ChunkTargetPtrs), - Context.WriteCache, - Context.Work, - Context.TotalRequestCount, - Context.TotalPartWriteCount, - Context.FilteredDownloadedBytesPerSecond, - Context.FilteredWrittenBytesPerSecond); - } - }, - WorkerThreadPool::EMode::EnableBacklog); - } -} - -void -BuildsOperationUpdateFolder::ScheduleLocalChunkCopies(WriteChunksContext& Context, - std::span CopyChunkDatas, - CloneQueryInterface* CloneQuery, - const std::vector& ScavengedContents, - const std::vector& ScavengedLookups, - const std::vector& ScavengedPaths) -{ - for (size_t CopyDataIndex = 0; CopyDataIndex < CopyChunkDatas.size(); CopyDataIndex++) - { - if (m_AbortFlag) - { - break; - } - - Context.Work.ScheduleWork( - m_IOWorkerPool, - [this, &Context, CloneQuery, &CopyChunkDatas, &ScavengedContents, &ScavengedLookups, &ScavengedPaths, CopyDataIndex]( - std::atomic&) { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Async_CopyLocal"); - - Context.FilteredWrittenBytesPerSecond.Start(); - const CopyChunkData& CopyData = CopyChunkDatas[CopyDataIndex]; - - std::vector WrittenSequenceIndexes = WriteLocalChunkToCache(CloneQuery, - CopyData, - ScavengedContents, - ScavengedLookups, - ScavengedPaths, - Context.WriteCache); - bool WritePartsDone = Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount; - if (!m_AbortFlag) - { - if (WritePartsDone) - { - Context.FilteredWrittenBytesPerSecond.Stop(); - } - - // Write tracking, updating this must be done without any files open - std::vector CompletedChunkSequences; - for (uint32_t RemoteSequenceIndex : WrittenSequenceIndexes) - { - if (CompleteSequenceChunk(RemoteSequenceIndex, Context.SequenceIndexChunksLeftToWriteCounters)) - { - CompletedChunkSequences.push_back(RemoteSequenceIndex); - } - } - Context.WriteCache.Close(CompletedChunkSequences); - VerifyAndCompleteChunkSequencesAsync(CompletedChunkSequences, Context.Work); - } - } - }); - } -} - -void -BuildsOperationUpdateFolder::ScheduleCachedBlockWrites(WriteChunksContext& Context, std::span CachedBlockIndexes) -{ - for (uint32_t BlockIndex : CachedBlockIndexes) - { - if (m_AbortFlag) - { - break; - } - - Context.Work.ScheduleWork(m_IOWorkerPool, [this, &Context, BlockIndex](std::atomic&) { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Async_WriteCachedBlock"); - - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; - Context.FilteredWrittenBytesPerSecond.Start(); - - std::filesystem::path BlockChunkPath = m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString(); - IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockBuffer) - { - throw std::runtime_error(fmt::format("Can not read block {} at {}", BlockDescription.BlockHash, BlockChunkPath)); - } - - if (!m_AbortFlag) - { - if (!WriteChunksBlockToCache(BlockDescription, - Context.SequenceIndexChunksLeftToWriteCounters, - Context.Work, - CompositeBuffer(std::move(BlockBuffer)), - Context.RemoteChunkIndexNeedsCopyFromSourceFlags, - Context.WriteCache)) - { - std::error_code DummyEc; - RemoveFile(BlockChunkPath, DummyEc); - throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); - } - - std::error_code Ec = TryRemoveFile(BlockChunkPath); - if (Ec) - { - ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", BlockChunkPath, Ec.value(), Ec.message()); - } - - if (Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount) - { - Context.FilteredWrittenBytesPerSecond.Stop(); - } - } - } - }); - } -} - -void -BuildsOperationUpdateFolder::SchedulePartialBlockDownloads(WriteChunksContext& Context, - const ChunkBlockAnalyser::BlockResult& PartialBlocks) -{ - for (size_t BlockRangeIndex = 0; BlockRangeIndex < PartialBlocks.BlockRanges.size();) - { - if (m_AbortFlag) - { - break; - } - - size_t RangeCount = 1; - size_t RangesLeft = PartialBlocks.BlockRanges.size() - BlockRangeIndex; - const ChunkBlockAnalyser::BlockRangeDescriptor& CurrentBlockRange = PartialBlocks.BlockRanges[BlockRangeIndex]; - while (RangeCount < RangesLeft && - CurrentBlockRange.BlockIndex == PartialBlocks.BlockRanges[BlockRangeIndex + RangeCount].BlockIndex) - { - RangeCount++; - } - - Context.Work.ScheduleWork( - m_NetworkPool, - [this, &Context, &PartialBlocks, BlockRangeStartIndex = BlockRangeIndex, RangeCount = RangeCount](std::atomic&) { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Async_GetPartialBlockRanges"); - - Context.FilteredDownloadedBytesPerSecond.Start(); - - DownloadPartialBlock( - PartialBlocks.BlockRanges, - BlockRangeStartIndex, - RangeCount, - Context.ExistsResult, - Context.TotalRequestCount, - Context.FilteredDownloadedBytesPerSecond, - [this, &Context, &PartialBlocks](IoBuffer&& InMemoryBuffer, - const std::filesystem::path& OnDiskPath, - size_t BlockRangeStartIndex, - std::span> OffsetAndLengths) { - if (!m_AbortFlag) - { - Context.Work.ScheduleWork( - m_IOWorkerPool, - [this, - &Context, - &PartialBlocks, - BlockRangeStartIndex, - BlockChunkPath = std::filesystem::path(OnDiskPath), - BlockPartialBuffer = std::move(InMemoryBuffer), - OffsetAndLengths = - std::vector>(OffsetAndLengths.begin(), OffsetAndLengths.end())]( - std::atomic&) mutable { - if (!m_AbortFlag) - { - WritePartialBlockToCache(Context, - BlockRangeStartIndex, - std::move(BlockPartialBuffer), - BlockChunkPath, - OffsetAndLengths, - PartialBlocks); - } - }, - OnDiskPath.empty() ? WorkerThreadPool::EMode::DisableBacklog : WorkerThreadPool::EMode::EnableBacklog); - } - }); - } - }); - BlockRangeIndex += RangeCount; - } -} - -void -BuildsOperationUpdateFolder::WritePartialBlockToCache(WriteChunksContext& Context, - size_t BlockRangeStartIndex, - IoBuffer BlockPartialBuffer, - const std::filesystem::path& BlockChunkPath, - std::span> OffsetAndLengths, - const ChunkBlockAnalyser::BlockResult& PartialBlocks) -{ - ZEN_TRACE_CPU("Async_WritePartialBlock"); - - const uint32_t BlockIndex = PartialBlocks.BlockRanges[BlockRangeStartIndex].BlockIndex; - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; - - if (BlockChunkPath.empty()) - { - ZEN_ASSERT(BlockPartialBuffer); - } - else - { - ZEN_ASSERT(!BlockPartialBuffer); - BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockPartialBuffer) - { - throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}", BlockDescription.BlockHash, BlockChunkPath)); - } - } - - Context.FilteredWrittenBytesPerSecond.Start(); - - const size_t RangeCount = OffsetAndLengths.size(); - - for (size_t PartialRangeIndex = 0; PartialRangeIndex < RangeCount; PartialRangeIndex++) - { - const std::pair& OffsetAndLength = OffsetAndLengths[PartialRangeIndex]; - IoBuffer BlockRangeBuffer(BlockPartialBuffer, OffsetAndLength.first, OffsetAndLength.second); - - const ChunkBlockAnalyser::BlockRangeDescriptor& RangeDescriptor = - PartialBlocks.BlockRanges[BlockRangeStartIndex + PartialRangeIndex]; - - if (!WritePartialBlockChunksToCache(BlockDescription, - Context.SequenceIndexChunksLeftToWriteCounters, - Context.Work, - CompositeBuffer(std::move(BlockRangeBuffer)), - RangeDescriptor.ChunkBlockIndexStart, - RangeDescriptor.ChunkBlockIndexStart + RangeDescriptor.ChunkBlockIndexCount - 1, - Context.RemoteChunkIndexNeedsCopyFromSourceFlags, - Context.WriteCache)) - { - std::error_code DummyEc; - RemoveFile(BlockChunkPath, DummyEc); - throw std::runtime_error(fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); - } - - if (Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount) - { - Context.FilteredWrittenBytesPerSecond.Stop(); - } - } - std::error_code Ec = TryRemoveFile(BlockChunkPath); - if (Ec) - { - ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", BlockChunkPath, Ec.value(), Ec.message()); - } -} - -void -BuildsOperationUpdateFolder::ScheduleFullBlockDownloads(WriteChunksContext& Context, std::span FullBlockIndexes) -{ - for (uint32_t BlockIndex : FullBlockIndexes) - { - if (m_AbortFlag) - { - break; - } - - Context.Work.ScheduleWork(m_NetworkPool, [this, &Context, BlockIndex](std::atomic&) { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Async_GetFullBlock"); - - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; - - Context.FilteredDownloadedBytesPerSecond.Start(); - - IoBuffer BlockBuffer; - const bool ExistsInCache = - m_Storage.CacheStorage && Context.ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); - if (ExistsInCache) - { - BlockBuffer = m_Storage.CacheStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash); - } - if (!BlockBuffer) - { - try - { - BlockBuffer = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash); - } - catch (const std::exception&) - { - // Silence http errors due to abort - if (!m_AbortFlag) - { - throw; - } - } - } - if (!m_AbortFlag) - { - if (!BlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); - } - - uint64_t BlockSize = BlockBuffer.GetSize(); - m_DownloadStats.DownloadedBlockCount++; - m_DownloadStats.DownloadedBlockByteCount += BlockSize; - if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == Context.TotalRequestCount) - { - Context.FilteredDownloadedBytesPerSecond.Stop(); - } - - const bool PutInCache = !ExistsInCache && m_Storage.CacheStorage && m_Options.PopulateCache; - - std::filesystem::path BlockChunkPath = - TryMoveDownloadedChunk(BlockBuffer, - m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString(), - /* ForceDiskBased */ PutInCache || (BlockSize > m_Options.MaximumInMemoryPayloadSize)); - - if (PutInCache) - { - ZEN_ASSERT(!BlockChunkPath.empty()); - IoBuffer CacheBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (CacheBuffer) - { - m_Storage.CacheStorage->PutBuildBlob(m_BuildId, - BlockDescription.BlockHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(SharedBuffer(CacheBuffer))); - } - } - - if (!m_AbortFlag) - { - Context.Work.ScheduleWork( - m_IOWorkerPool, - [this, &Context, BlockIndex, BlockChunkPath, BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { - if (!m_AbortFlag) - { - WriteFullBlockToCache(Context, BlockIndex, std::move(BlockBuffer), BlockChunkPath); - } - }, - BlockChunkPath.empty() ? WorkerThreadPool::EMode::DisableBacklog : WorkerThreadPool::EMode::EnableBacklog); - } - } - } - }); - } -} - -void -BuildsOperationUpdateFolder::WriteFullBlockToCache(WriteChunksContext& Context, - uint32_t BlockIndex, - IoBuffer BlockBuffer, - const std::filesystem::path& BlockChunkPath) -{ - ZEN_TRACE_CPU("Async_WriteFullBlock"); - - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; - - if (BlockChunkPath.empty()) - { - ZEN_ASSERT(BlockBuffer); - } - else - { - ZEN_ASSERT(!BlockBuffer); - BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockBuffer) - { - throw std::runtime_error(fmt::format("Could not open dowloaded block {} from {}", BlockDescription.BlockHash, BlockChunkPath)); - } - } - - Context.FilteredWrittenBytesPerSecond.Start(); - if (!WriteChunksBlockToCache(BlockDescription, - Context.SequenceIndexChunksLeftToWriteCounters, - Context.Work, - CompositeBuffer(std::move(BlockBuffer)), - Context.RemoteChunkIndexNeedsCopyFromSourceFlags, - Context.WriteCache)) - { - std::error_code DummyEc; - RemoveFile(BlockChunkPath, DummyEc); - throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); - } - - if (!BlockChunkPath.empty()) - { - std::error_code Ec = TryRemoveFile(BlockChunkPath); - if (Ec) - { - ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", BlockChunkPath, Ec.value(), Ec.message()); - } - } - - if (Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount) - { - Context.FilteredWrittenBytesPerSecond.Stop(); - } -} - -void -BuildsOperationUpdateFolder::ScheduleLocalFileRemovals(ParallelWork& Work, - std::span RemoveLocalPathIndexes, - std::atomic& DeletedCount) -{ - for (uint32_t LocalPathIndex : RemoveLocalPathIndexes) - { - if (m_AbortFlag) - { - break; - } - Work.ScheduleWork(m_IOWorkerPool, [this, &DeletedCount, LocalPathIndex](std::atomic&) { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Async_RemoveFile"); - - const std::filesystem::path LocalFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred(); - SetFileReadOnlyWithRetry(LocalFilePath, false); - RemoveFileWithRetry(LocalFilePath); - DeletedCount++; - } - }); - } -} - -void -BuildsOperationUpdateFolder::ScheduleTargetFinalization( - ParallelWork& Work, - std::span Targets, - const tsl::robin_map& SequenceHashToLocalPathIndex, - const tsl::robin_map& RemotePathIndexToLocalPathIndex, - FolderContent& OutLocalFolderState, - std::atomic& TargetsComplete) -{ - size_t TargetOffset = 0; - while (TargetOffset < Targets.size()) - { - if (m_AbortFlag) - { - break; - } - - size_t TargetCount = 1; - while ((TargetOffset + TargetCount) < Targets.size() && - (Targets[TargetOffset + TargetCount].RawHash == Targets[TargetOffset].RawHash)) - { - TargetCount++; - } - - Work.ScheduleWork(m_IOWorkerPool, - [this, - &SequenceHashToLocalPathIndex, - Targets, - &RemotePathIndexToLocalPathIndex, - &OutLocalFolderState, - BaseTargetOffset = TargetOffset, - TargetCount, - &TargetsComplete](std::atomic&) { - if (!m_AbortFlag) - { - FinalizeTargetGroup(BaseTargetOffset, - TargetCount, - Targets, - SequenceHashToLocalPathIndex, - RemotePathIndexToLocalPathIndex, - OutLocalFolderState, - TargetsComplete); - } - }); - - TargetOffset += TargetCount; - } -} - -void -BuildsOperationUpdateFolder::FinalizeTargetGroup(size_t BaseOffset, - size_t Count, - std::span Targets, - const tsl::robin_map& SequenceHashToLocalPathIndex, - const tsl::robin_map& RemotePathIndexToLocalPathIndex, - FolderContent& OutLocalFolderState, - std::atomic& TargetsComplete) -{ - ZEN_TRACE_CPU("Async_FinalizeChunkSequence"); - - size_t TargetOffset = BaseOffset; - const IoHash& RawHash = Targets[TargetOffset].RawHash; - - if (RawHash == IoHash::Zero) - { - ZEN_TRACE_CPU("CreateEmptyFiles"); - while (TargetOffset < (BaseOffset + Count)) - { - const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; - ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); - const std::filesystem::path& TargetPath = m_RemoteContent.Paths[RemotePathIndex]; - std::filesystem::path TargetFilePath = (m_Path / TargetPath).make_preferred(); - auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); - if (InPlaceIt == RemotePathIndexToLocalPathIndex.end() || InPlaceIt->second == 0) - { - if (IsFileWithRetry(TargetFilePath)) - { - SetFileReadOnlyWithRetry(TargetFilePath, false); - } - else - { - CreateDirectories(TargetFilePath.parent_path()); - } - BasicFile OutputFile; - OutputFile.Open(TargetFilePath, BasicFile::Mode::kTruncate); - } - OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; - OutLocalFolderState.RawSizes[RemotePathIndex] = m_RemoteContent.RawSizes[RemotePathIndex]; - - OutLocalFolderState.Attributes[RemotePathIndex] = - m_RemoteContent.Attributes.empty() - ? GetNativeFileAttributes(TargetFilePath) - : SetNativeFileAttributes(TargetFilePath, m_RemoteContent.Platform, m_RemoteContent.Attributes[RemotePathIndex]); - OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); - - TargetOffset++; - TargetsComplete++; - } - } - else - { - ZEN_TRACE_CPU("FinalizeFile"); - ZEN_ASSERT(m_RemoteLookup.RawHashToSequenceIndex.contains(RawHash)); - const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex; - const std::filesystem::path& FirstTargetPath = m_RemoteContent.Paths[FirstRemotePathIndex]; - std::filesystem::path FirstTargetFilePath = (m_Path / FirstTargetPath).make_preferred(); - - if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end()) - { - ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); - } - else - { - if (IsFileWithRetry(FirstTargetFilePath)) - { - SetFileReadOnlyWithRetry(FirstTargetFilePath, false); - } - else - { - CreateDirectories(FirstTargetFilePath.parent_path()); - } - - if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash); InplaceIt != SequenceHashToLocalPathIndex.end()) - { - ZEN_TRACE_CPU("Copy"); - const uint32_t LocalPathIndex = InplaceIt->second; - const std::filesystem::path& SourcePath = m_LocalContent.Paths[LocalPathIndex]; - std::filesystem::path SourceFilePath = (m_Path / SourcePath).make_preferred(); - ZEN_ASSERT_SLOW(IsFileWithRetry(SourceFilePath)); - - ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath); - const uint64_t RawSize = m_LocalContent.RawSizes[LocalPathIndex]; - FastCopyFile(m_Options.AllowFileClone, - m_Options.UseSparseFiles, - SourceFilePath, - FirstTargetFilePath, - RawSize, - m_DiskStats.WriteCount, - m_DiskStats.WriteByteCount, - m_DiskStats.CloneCount, - m_DiskStats.CloneByteCount); - - m_RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; - } - else - { - ZEN_TRACE_CPU("Rename"); - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(IsFileWithRetry(CacheFilePath)); - - std::error_code Ec = RenameFileWithRetry(CacheFilePath, FirstTargetFilePath); - if (Ec) - { - ZEN_WARN("Failed to move file from '{}' to '{}', reason: ({}) {}, retrying...", - CacheFilePath, - FirstTargetFilePath, - Ec.value(), - Ec.message()); - Ec = RenameFileWithRetry(CacheFilePath, FirstTargetFilePath); - if (Ec) - { - throw std::system_error(std::error_code(Ec.value(), std::system_category()), - fmt::format("Failed to move file from '{}' to '{}', reason: ({}) {}", - CacheFilePath, - FirstTargetFilePath, - Ec.value(), - Ec.message())); - } - } - - m_RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; - } - } - - OutLocalFolderState.Paths[FirstRemotePathIndex] = FirstTargetPath; - OutLocalFolderState.RawSizes[FirstRemotePathIndex] = m_RemoteContent.RawSizes[FirstRemotePathIndex]; - - OutLocalFolderState.Attributes[FirstRemotePathIndex] = - m_RemoteContent.Attributes.empty() - ? GetNativeFileAttributes(FirstTargetFilePath) - : SetNativeFileAttributes(FirstTargetFilePath, m_RemoteContent.Platform, m_RemoteContent.Attributes[FirstRemotePathIndex]); - OutLocalFolderState.ModificationTicks[FirstRemotePathIndex] = GetModificationTickFromPath(FirstTargetFilePath); - - TargetOffset++; - TargetsComplete++; - - while (TargetOffset < (BaseOffset + Count)) - { - const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; - ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); - const std::filesystem::path& TargetPath = m_RemoteContent.Paths[RemotePathIndex]; - std::filesystem::path TargetFilePath = (m_Path / TargetPath).make_preferred(); - - if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end()) - { - ZEN_ASSERT_SLOW(IsFileWithRetry(TargetFilePath)); - } - else - { - ZEN_TRACE_CPU("Copy"); - if (IsFileWithRetry(TargetFilePath)) - { - SetFileReadOnlyWithRetry(TargetFilePath, false); - } - else - { - CreateDirectories(TargetFilePath.parent_path()); - } - - ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); - ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath); - const uint64_t RawSize = m_RemoteContent.RawSizes[RemotePathIndex]; - FastCopyFile(m_Options.AllowFileClone, - m_Options.UseSparseFiles, - FirstTargetFilePath, - TargetFilePath, - RawSize, - m_DiskStats.WriteCount, - m_DiskStats.WriteByteCount, - m_DiskStats.CloneCount, - m_DiskStats.CloneByteCount); - - m_RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; - } - - OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; - OutLocalFolderState.RawSizes[RemotePathIndex] = m_RemoteContent.RawSizes[RemotePathIndex]; - - OutLocalFolderState.Attributes[RemotePathIndex] = - m_RemoteContent.Attributes.empty() - ? GetNativeFileAttributes(TargetFilePath) - : SetNativeFileAttributes(TargetFilePath, m_RemoteContent.Platform, m_RemoteContent.Attributes[RemotePathIndex]); - OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); - - TargetOffset++; - TargetsComplete++; - } - } -} - -std::vector -BuildsOperationUpdateFolder::FindScavengeSources() -{ - ZEN_TRACE_CPU("FindScavengeSources"); - - const bool TargetPathExists = IsDir(m_Path); - - std::vector StatePaths = GetDownloadedStatePaths(m_Options.SystemRootDir); - - std::vector Result; - for (const std::filesystem::path& EntryPath : StatePaths) - { - if (IsFile(EntryPath)) - { - bool DeleteEntry = false; - - try - { - BuildsDownloadInfo Info = ReadDownloadedInfoFile(EntryPath); - const bool LocalPathExists = !Info.LocalPath.empty() && IsDir(Info.LocalPath); - const bool LocalStateFileExists = IsFile(Info.StateFilePath); - if (LocalPathExists && LocalStateFileExists) - { - if (TargetPathExists && std::filesystem::equivalent(Info.LocalPath, m_Path)) - { - DeleteEntry = true; - } - else - { - Result.push_back({.StateFilePath = std::move(Info.StateFilePath), .Path = std::move(Info.LocalPath)}); - } - } - else - { - DeleteEntry = true; - } - } - catch (const std::exception& Ex) - { - ZEN_WARN("{}", Ex.what()); - DeleteEntry = true; - } - - if (DeleteEntry) - { - std::error_code DummyEc; - std::filesystem::remove(EntryPath, DummyEc); - } - } - } - return Result; -} - -std::vector -BuildsOperationUpdateFolder::ScanTargetFolder(const tsl::robin_map& CachedChunkHashesFound, - const tsl::robin_map& CachedSequenceHashesFound) -{ - ZEN_TRACE_CPU("ScanTargetFolder"); - - Stopwatch LocalTimer; - - std::vector MissingSequenceIndexes; - - for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < m_RemoteContent.ChunkedContent.SequenceRawHashes.size(); - RemoteSequenceIndex++) - { - const IoHash& RemoteSequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(m_RemoteLookup, RemoteSequenceIndex); - const uint64_t RemoteRawSize = m_RemoteContent.RawSizes[RemotePathIndex]; - if (auto CacheSequenceIt = CachedSequenceHashesFound.find(RemoteSequenceRawHash); - CacheSequenceIt != CachedSequenceHashesFound.end()) - { - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash); - ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); - if (m_Options.IsVerbose) - { - ZEN_INFO("Found sequence {} at {} ({})", RemoteSequenceRawHash, CacheFilePath, NiceBytes(RemoteRawSize)); - } - } - else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash); CacheChunkIt != CachedChunkHashesFound.end()) - { - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash); - ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); - if (m_Options.IsVerbose) - { - ZEN_INFO("Found chunk {} at {} ({})", RemoteSequenceRawHash, CacheFilePath, NiceBytes(RemoteRawSize)); - } - } - else if (auto It = m_LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash); - It != m_LocalLookup.RawHashToSequenceIndex.end()) - { - const uint32_t LocalSequenceIndex = It->second; - const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(m_LocalLookup, LocalSequenceIndex); - const std::filesystem::path LocalFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred(); - ZEN_ASSERT_SLOW(IsFile(LocalFilePath)); - m_CacheMappingStats.LocalPathsMatchingSequencesCount++; - m_CacheMappingStats.LocalPathsMatchingSequencesByteCount += RemoteRawSize; - if (m_Options.IsVerbose) - { - ZEN_INFO("Found sequence {} at {} ({})", RemoteSequenceRawHash, LocalFilePath, NiceBytes(RemoteRawSize)); - } - } - else - { - MissingSequenceIndexes.push_back(RemoteSequenceIndex); - } - } - - m_CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs(); - return MissingSequenceIndexes; -} - -bool -BuildsOperationUpdateFolder::FindScavengeContent(const ScavengeSource& Source, - ChunkedFolderContent& OutScavengedLocalContent, - ChunkedContentLookup& OutScavengedLookup) -{ - ZEN_TRACE_CPU("FindScavengeContent"); - - FolderContent LocalFolderState; - try - { - BuildSaveState SavedState = ReadBuildSaveStateFile(Source.StateFilePath); - if (SavedState.Version == BuildSaveState::NoVersion) - { - ZEN_DEBUG("Skipping old build state at '{}', state files before version {} can not be trusted during scavenge", - Source.StateFilePath, - BuildSaveState::kVersion1); - return false; - } - OutScavengedLocalContent = std::move(SavedState.State.ChunkedContent); - LocalFolderState = std::move(SavedState.FolderState); - } - catch (const std::exception& Ex) - { - ZEN_DEBUG("Skipping invalid build state at '{}', reason: {}", Source.StateFilePath, Ex.what()); - return false; - } - - tsl::robin_set PathIndexesToScavenge; - PathIndexesToScavenge.reserve(OutScavengedLocalContent.Paths.size()); - std::vector ChunkOrderOffsets = BuildChunkOrderOffset(OutScavengedLocalContent.ChunkedContent.ChunkCounts); - - { - tsl::robin_map RawHashToPathIndex; - - RawHashToPathIndex.reserve(OutScavengedLocalContent.Paths.size()); - for (uint32_t ScavengedPathIndex = 0; ScavengedPathIndex < OutScavengedLocalContent.RawHashes.size(); ScavengedPathIndex++) - { - if (!RawHashToPathIndex.contains(OutScavengedLocalContent.RawHashes[ScavengedPathIndex])) - { - RawHashToPathIndex.insert_or_assign(OutScavengedLocalContent.RawHashes[ScavengedPathIndex], ScavengedPathIndex); - } - } - - for (uint32_t ScavengeSequenceIndex = 0; ScavengeSequenceIndex < OutScavengedLocalContent.ChunkedContent.SequenceRawHashes.size(); - ScavengeSequenceIndex++) - { - const IoHash& SequenceHash = OutScavengedLocalContent.ChunkedContent.SequenceRawHashes[ScavengeSequenceIndex]; - if (auto It = RawHashToPathIndex.find(SequenceHash); It != RawHashToPathIndex.end()) - { - uint32_t PathIndex = It->second; - if (!PathIndexesToScavenge.contains(PathIndex)) - { - if (m_RemoteLookup.RawHashToSequenceIndex.contains(SequenceHash)) - { - PathIndexesToScavenge.insert(PathIndex); - } - else - { - uint32_t ChunkOrderIndexStart = ChunkOrderOffsets[ScavengeSequenceIndex]; - const uint32_t ChunkCount = OutScavengedLocalContent.ChunkedContent.ChunkCounts[ScavengeSequenceIndex]; - for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < ChunkCount; ChunkOrderIndex++) - { - const uint32_t ChunkIndex = - OutScavengedLocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndexStart + ChunkOrderIndex]; - const IoHash& ChunkHash = OutScavengedLocalContent.ChunkedContent.ChunkHashes[ChunkIndex]; - if (m_RemoteLookup.ChunkHashToChunkIndex.contains(ChunkHash)) - { - PathIndexesToScavenge.insert(PathIndex); - break; - } - } - } - } - } - else - { - ZEN_WARN("Scavenged state file at '{}' for '{}' is invalid, skipping scavenging for sequence {}", - Source.StateFilePath, - Source.Path, - SequenceHash); - } - } - } - - if (PathIndexesToScavenge.empty()) - { - OutScavengedLocalContent = {}; - return false; - } - - std::vector PathsToScavenge; - PathsToScavenge.reserve(PathIndexesToScavenge.size()); - for (uint32_t ScavengedStatePathIndex : PathIndexesToScavenge) - { - PathsToScavenge.push_back(OutScavengedLocalContent.Paths[ScavengedStatePathIndex]); - } - - FolderContent ValidFolderContent = - GetValidFolderContent(m_IOWorkerPool, m_ScavengedFolderScanStats, Source.Path, PathsToScavenge, {}, 0, m_AbortFlag, m_PauseFlag); - - if (!LocalFolderState.AreKnownFilesEqual(ValidFolderContent)) - { - std::vector DeletedPaths; - FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, ValidFolderContent, DeletedPaths); - - // If the files are modified since the state was saved we ignore the files since we don't - // want to incur the cost of scanning/hashing scavenged files - DeletedPaths.insert(DeletedPaths.end(), UpdatedContent.Paths.begin(), UpdatedContent.Paths.end()); - if (!DeletedPaths.empty()) - { - OutScavengedLocalContent = - DeletePathsFromChunkedContent(OutScavengedLocalContent, - BuildHashLookup(OutScavengedLocalContent.ChunkedContent.SequenceRawHashes), - ChunkOrderOffsets, - DeletedPaths); - } - } - - if (OutScavengedLocalContent.Paths.empty()) - { - OutScavengedLocalContent = {}; - return false; - } - - OutScavengedLookup = BuildChunkedContentLookup(OutScavengedLocalContent); - - return true; -} - -void -BuildsOperationUpdateFolder::ScavengeSourceForChunks(uint32_t& InOutRemainingChunkCount, - std::vector& InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags, - tsl::robin_map& InOutRawHashToCopyChunkDataIndex, - const std::vector>& SequenceIndexChunksLeftToWriteCounters, - const ChunkedFolderContent& ScavengedContent, - const ChunkedContentLookup& ScavengedLookup, - std::vector& InOutCopyChunkDatas, - uint32_t ScavengedContentIndex, - uint64_t& InOutChunkMatchingRemoteCount, - uint64_t& InOutChunkMatchingRemoteByteCount) -{ - for (uint32_t RemoteChunkIndex = 0; - RemoteChunkIndex < m_RemoteContent.ChunkedContent.ChunkHashes.size() && (InOutRemainingChunkCount > 0); - RemoteChunkIndex++) - { - if (!InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) - { - const IoHash& RemoteChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - if (auto It = ScavengedLookup.ChunkHashToChunkIndex.find(RemoteChunkHash); It != ScavengedLookup.ChunkHashToChunkIndex.end()) - { - std::vector ChunkTargetPtrs = - GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex); - - if (!ChunkTargetPtrs.empty()) - { - const uint32_t ScavengedChunkIndex = It->second; - const uint64_t ScavengedChunkRawSize = ScavengedContent.ChunkedContent.ChunkRawSizes[ScavengedChunkIndex]; - const size_t ChunkSequenceLocationOffset = ScavengedLookup.ChunkSequenceLocationOffset[ScavengedChunkIndex]; - const ChunkedContentLookup::ChunkSequenceLocation& ScavengeLocation = - ScavengedLookup.ChunkSequenceLocations[ChunkSequenceLocationOffset]; - const IoHash& ScavengedSequenceRawHash = - ScavengedContent.ChunkedContent.SequenceRawHashes[ScavengeLocation.SequenceIndex]; - - CopyChunkData::ChunkTarget Target = {.TargetChunkLocationCount = gsl::narrow(ChunkTargetPtrs.size()), - .RemoteChunkIndex = RemoteChunkIndex, - .CacheFileOffset = ScavengeLocation.Offset}; - if (auto CopySourceIt = InOutRawHashToCopyChunkDataIndex.find(ScavengedSequenceRawHash); - CopySourceIt != InOutRawHashToCopyChunkDataIndex.end()) - { - CopyChunkData& Data = InOutCopyChunkDatas[CopySourceIt->second]; - if (Data.TargetChunkLocationPtrs.size() > 1024) - { - InOutRawHashToCopyChunkDataIndex.insert_or_assign(ScavengedSequenceRawHash, InOutCopyChunkDatas.size()); - InOutCopyChunkDatas.push_back(CopyChunkData{.ScavengeSourceIndex = ScavengedContentIndex, - .SourceSequenceIndex = ScavengeLocation.SequenceIndex, - .TargetChunkLocationPtrs = ChunkTargetPtrs, - .ChunkTargets = std::vector{Target}}); - } - else - { - Data.TargetChunkLocationPtrs.insert(Data.TargetChunkLocationPtrs.end(), - ChunkTargetPtrs.begin(), - ChunkTargetPtrs.end()); - Data.ChunkTargets.push_back(Target); - } - } - else - { - InOutRawHashToCopyChunkDataIndex.insert_or_assign(ScavengedSequenceRawHash, InOutCopyChunkDatas.size()); - InOutCopyChunkDatas.push_back(CopyChunkData{.ScavengeSourceIndex = ScavengedContentIndex, - .SourceSequenceIndex = ScavengeLocation.SequenceIndex, - .TargetChunkLocationPtrs = ChunkTargetPtrs, - .ChunkTargets = std::vector{Target}}); - } - InOutChunkMatchingRemoteCount++; - InOutChunkMatchingRemoteByteCount += ScavengedChunkRawSize; - InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true; - InOutRemainingChunkCount--; - } - } - } - } -} - -std::filesystem::path -BuildsOperationUpdateFolder::FindDownloadedChunk(const IoHash& ChunkHash) -{ - ZEN_TRACE_CPU("FindDownloadedChunk"); - - std::filesystem::path CompressedChunkPath = m_TempDownloadFolderPath / ChunkHash.ToHexString(); - if (IsFile(CompressedChunkPath)) - { - IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); - if (ExistingCompressedPart) - { - IoHash RawHash; - uint64_t RawSize; - if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart, - RawHash, - RawSize, - /*OutOptionalTotalCompressedSize*/ nullptr)) - { - return CompressedChunkPath; - } - else - { - std::error_code DummyEc; - RemoveFile(CompressedChunkPath, DummyEc); - } - } - } - return {}; -} - -std::vector -BuildsOperationUpdateFolder::GetRemainingChunkTargets(std::span> SequenceIndexChunksLeftToWriteCounters, - uint32_t ChunkIndex) -{ - ZEN_TRACE_CPU("GetRemainingChunkTargets"); - - std::span ChunkSources = GetChunkSequenceLocations(m_RemoteLookup, ChunkIndex); - std::vector ChunkTargetPtrs; - if (!ChunkSources.empty()) - { - ChunkTargetPtrs.reserve(ChunkSources.size()); - for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources) - { - if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0) - { - ChunkTargetPtrs.push_back(&Source); - } - } - } - return ChunkTargetPtrs; -}; - -uint64_t -BuildsOperationUpdateFolder::GetChunkWriteCount(std::span> SequenceIndexChunksLeftToWriteCounters, - uint32_t ChunkIndex) -{ - ZEN_TRACE_CPU("GetChunkWriteCount"); - - uint64_t WriteCount = 0; - std::span ChunkSources = GetChunkSequenceLocations(m_RemoteLookup, ChunkIndex); - for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources) - { - if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0) - { - WriteCount++; - } - } - return WriteCount; -}; - -void -BuildsOperationUpdateFolder::CheckRequiredDiskSpace(const tsl::robin_map& RemotePathToRemoteIndex) -{ - tsl::robin_set ExistingRemotePaths; - - if (m_Options.EnableTargetFolderScavenging) - { - for (uint32_t LocalPathIndex = 0; LocalPathIndex < m_LocalContent.Paths.size(); LocalPathIndex++) - { - const IoHash& RawHash = m_LocalContent.RawHashes[LocalPathIndex]; - const std::filesystem::path& LocalPath = m_LocalContent.Paths[LocalPathIndex]; - - if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string()); RemotePathIt != RemotePathToRemoteIndex.end()) - { - const uint32_t RemotePathIndex = RemotePathIt->second; - if (m_RemoteContent.RawHashes[RemotePathIndex] == RawHash) - { - ExistingRemotePaths.insert(RemotePathIndex); - } - } - } - } - - uint64_t RequiredSpace = 0; - for (uint32_t RemotePathIndex = 0; RemotePathIndex < m_RemoteContent.Paths.size(); RemotePathIndex++) - { - if (!ExistingRemotePaths.contains(RemotePathIndex)) - { - RequiredSpace += m_RemoteContent.RawSizes[RemotePathIndex]; - } - } - - std::error_code Ec; - DiskSpace Space = DiskSpaceInfo(m_Path, Ec); - if (Ec) - { - throw std::runtime_error(fmt::format("Get free disk space for target path '{}' FAILED, reason: {}", m_Path, Ec.message())); - } - if (Space.Free < (RequiredSpace + 16u * 1024u * 1024u)) - { - throw std::runtime_error( - fmt::format("Not enough free space for target path '{}', {} of free space is needed but only {} is available", - m_Path, - NiceBytes(RequiredSpace), - NiceBytes(Space.Free))); - } -} - -void -BuildsOperationUpdateFolder::WriteScavengedSequenceToCache(const std::filesystem::path& ScavengeRootPath, - const ChunkedFolderContent& ScavengedContent, - const ScavengedSequenceCopyOperation& ScavengeOp) -{ - ZEN_TRACE_CPU("WriteScavengedSequenceToCache"); - - const std::filesystem::path ScavengedPath = ScavengedContent.Paths[ScavengeOp.ScavengedPathIndex]; - const std::filesystem::path ScavengedFilePath = (ScavengeRootPath / ScavengedPath).make_preferred(); - ZEN_ASSERT_SLOW(FileSizeFromPath(ScavengedFilePath) == ScavengeOp.RawSize); - - const IoHash& RemoteSequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[ScavengeOp.RemoteSequenceIndex]; - const std::filesystem::path TempFilePath = GetTempChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash); - - const uint64_t RawSize = ScavengedContent.RawSizes[ScavengeOp.ScavengedPathIndex]; - FastCopyFile(m_Options.AllowFileClone, - m_Options.UseSparseFiles, - ScavengedFilePath, - TempFilePath, - RawSize, - m_DiskStats.WriteCount, - m_DiskStats.WriteByteCount, - m_DiskStats.CloneCount, - m_DiskStats.CloneByteCount); - - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash); - RenameFile(TempFilePath, CacheFilePath); - - m_WrittenChunkByteCount += RawSize; - if (m_Options.ValidateCompletedSequences) - { - m_ValidatedChunkByteCount += RawSize; - } -} - -void -BuildsOperationUpdateFolder::WriteLooseChunk(const uint32_t RemoteChunkIndex, - const BlobsExistsResult& ExistsResult, - std::span> SequenceIndexChunksLeftToWriteCounters, - std::atomic& WritePartsComplete, - std::vector&& ChunkTargetPtrs, - BufferedWriteFileCache& WriteCache, - ParallelWork& Work, - uint64_t TotalRequestCount, - uint64_t TotalPartWriteCount, - FilteredRate& FilteredDownloadedBytesPerSecond, - FilteredRate& FilteredWrittenBytesPerSecond) -{ - const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - std::filesystem::path ExistingCompressedChunkPath = FindDownloadedChunk(ChunkHash); - if (!ExistingCompressedChunkPath.empty()) - { - if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - } - if (!m_AbortFlag) - { - if (!ExistingCompressedChunkPath.empty()) - { - Work.ScheduleWork( - m_IOWorkerPool, - [this, - SequenceIndexChunksLeftToWriteCounters, - &WriteCache, - &Work, - &WritePartsComplete, - TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - RemoteChunkIndex, - ChunkTargetPtrs = std::move(ChunkTargetPtrs), - CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic& AbortFlag) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("Async_WritePreDownloadedChunk"); - - FilteredWrittenBytesPerSecond.Start(); - - const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - - IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); - if (!CompressedPart) - { - throw std::runtime_error( - fmt::format("Could not open dowloaded compressed chunk {} from {}", ChunkHash, CompressedChunkPath)); - } - - bool NeedHashVerify = - WriteCompressedChunkToCache(ChunkHash, ChunkTargetPtrs, WriteCache, std::move(CompressedPart)); - bool WritePartsDone = WritePartsComplete.fetch_add(1) + 1 == TotalPartWriteCount; - - if (!AbortFlag) - { - if (WritePartsDone) - { - FilteredWrittenBytesPerSecond.Stop(); - } - - std::error_code Ec = TryRemoveFile(CompressedChunkPath); - if (Ec) - { - ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", CompressedChunkPath, Ec.value(), Ec.message()); - } - - std::vector CompletedSequences = - CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); - WriteCache.Close(CompletedSequences); - if (NeedHashVerify) - { - VerifyAndCompleteChunkSequencesAsync(CompletedSequences, Work); - } - else - { - FinalizeChunkSequences(CompletedSequences); - } - } - } - }); - } - else - { - Work.ScheduleWork(m_NetworkPool, - [this, - &ExistsResult, - SequenceIndexChunksLeftToWriteCounters, - &WriteCache, - &Work, - &WritePartsComplete, - TotalPartWriteCount, - TotalRequestCount, - &FilteredDownloadedBytesPerSecond, - &FilteredWrittenBytesPerSecond, - RemoteChunkIndex, - ChunkTargetPtrs = std::vector( - std::move(ChunkTargetPtrs))](std::atomic&) mutable { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Async_DownloadChunk"); - - FilteredDownloadedBytesPerSecond.Start(); - DownloadBuildBlob(RemoteChunkIndex, - ExistsResult, - Work, - TotalRequestCount, - FilteredDownloadedBytesPerSecond, - [this, - &ExistsResult, - SequenceIndexChunksLeftToWriteCounters, - &WriteCache, - &Work, - &WritePartsComplete, - TotalPartWriteCount, - RemoteChunkIndex, - &FilteredWrittenBytesPerSecond, - ChunkTargetPtrs = std::move(ChunkTargetPtrs)](IoBuffer&& Payload) mutable { - AsyncWriteDownloadedChunk(RemoteChunkIndex, - ExistsResult, - std::move(ChunkTargetPtrs), - WriteCache, - Work, - std::move(Payload), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond); - }); - } - }); - } - } -} - -void -BuildsOperationUpdateFolder::DownloadBuildBlob(uint32_t RemoteChunkIndex, - const BlobsExistsResult& ExistsResult, - ParallelWork& Work, - uint64_t TotalRequestCount, - FilteredRate& FilteredDownloadedBytesPerSecond, - std::function&& OnDownloaded) -{ - const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - // FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BuildBlob; - const bool ExistsInCache = m_Storage.CacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash); - if (ExistsInCache) - { - BuildBlob = m_Storage.CacheStorage->GetBuildBlob(m_BuildId, ChunkHash); - } - if (BuildBlob) - { - uint64_t BlobSize = BuildBlob.GetSize(); - m_DownloadStats.DownloadedChunkCount++; - m_DownloadStats.DownloadedChunkByteCount += BlobSize; - if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - OnDownloaded(std::move(BuildBlob)); - } - else - { - if (m_RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= m_Options.LargeAttachmentSize) - { - DownloadLargeBlob( - *m_Storage.BuildStorage, - m_TempDownloadFolderPath, - m_BuildId, - ChunkHash, - m_Options.PreferredMultipartChunkSize, - Work, - m_NetworkPool, - m_DownloadStats.DownloadedChunkByteCount, - m_DownloadStats.MultipartAttachmentCount, - [this, &FilteredDownloadedBytesPerSecond, TotalRequestCount, OnDownloaded = std::move(OnDownloaded)](IoBuffer&& Payload) { - m_DownloadStats.DownloadedChunkCount++; - if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - - OnDownloaded(std::move(Payload)); - }); - } - else - { - try - { - BuildBlob = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, ChunkHash); - } - catch (const std::exception&) - { - // Silence http errors due to abort - if (!m_AbortFlag) - { - throw; - } - } - if (!m_AbortFlag) - { - if (!BuildBlob) - { - throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); - } - - if (!m_AbortFlag) - { - uint64_t BlobSize = BuildBlob.GetSize(); - m_DownloadStats.DownloadedChunkCount++; - m_DownloadStats.DownloadedChunkByteCount += BlobSize; - if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - - OnDownloaded(std::move(BuildBlob)); - } - } - } - } -} - -void -BuildsOperationUpdateFolder::DownloadPartialBlock( - std::span BlockRanges, - size_t BlockRangeStartIndex, - size_t BlockRangeCount, - const BlobsExistsResult& ExistsResult, - uint64_t TotalRequestCount, - FilteredRate& FilteredDownloadedBytesPerSecond, - std::function> OffsetAndLengths)>&& OnDownloaded) -{ - const uint32_t BlockIndex = BlockRanges[BlockRangeStartIndex].BlockIndex; - - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; - - auto ProcessDownload = [this]( - const ChunkBlockDescription& BlockDescription, - IoBuffer&& BlockRangeBuffer, - size_t BlockRangeStartIndex, - std::span> BlockOffsetAndLengths, - uint64_t TotalRequestCount, - FilteredRate& FilteredDownloadedBytesPerSecond, - const std::function> OffsetAndLengths)>& OnDownloaded) { - uint64_t BlockRangeBufferSize = BlockRangeBuffer.GetSize(); - m_DownloadStats.DownloadedBlockCount++; - m_DownloadStats.DownloadedBlockByteCount += BlockRangeBufferSize; - if (m_DownloadStats.RequestsCompleteCount.fetch_add(BlockOffsetAndLengths.size()) + BlockOffsetAndLengths.size() == - TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - - IoHashStream RangeId; - for (const std::pair& Range : BlockOffsetAndLengths) - { - RangeId.Append(&Range.first, sizeof(uint64_t)); - RangeId.Append(&Range.second, sizeof(uint64_t)); - } - std::filesystem::path BlockChunkPath = - TryMoveDownloadedChunk(BlockRangeBuffer, - m_TempBlockFolderPath / fmt::format("{}_{}", BlockDescription.BlockHash, RangeId.GetHash()), - /* ForceDiskBased */ BlockRangeBufferSize > m_Options.MaximumInMemoryPayloadSize); - - if (!m_AbortFlag) - { - OnDownloaded(std::move(BlockRangeBuffer), std::move(BlockChunkPath), BlockRangeStartIndex, BlockOffsetAndLengths); - } - }; - - std::vector> Ranges; - Ranges.reserve(BlockRangeCount); - for (size_t BlockRangeIndex = BlockRangeStartIndex; BlockRangeIndex < BlockRangeStartIndex + BlockRangeCount; BlockRangeIndex++) - { - const ChunkBlockAnalyser::BlockRangeDescriptor& BlockRange = BlockRanges[BlockRangeIndex]; - Ranges.push_back(std::make_pair(BlockRange.RangeStart, BlockRange.RangeLength)); - } - - const bool ExistsInCache = m_Storage.CacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); - - size_t SubBlockRangeCount = BlockRangeCount; - size_t SubRangeCountComplete = 0; - std::span> RangesSpan(Ranges); - while (SubRangeCountComplete < SubBlockRangeCount) - { - if (m_AbortFlag) - { - break; - } - - // First try to get subrange from cache. - // If not successful, try to get the ranges from the build store and adapt SubRangeCount... - - size_t SubRangeStartIndex = BlockRangeStartIndex + SubRangeCountComplete; - if (ExistsInCache) - { - size_t SubRangeCount = Min(BlockRangeCount - SubRangeCountComplete, m_Storage.CacheHost.Caps.MaxRangeCountPerRequest); - - if (SubRangeCount == 1) - { - // Legacy single-range path, prefer that for max compatibility - - const std::pair SubRange = RangesSpan[SubRangeCountComplete]; - IoBuffer PayloadBuffer = - m_Storage.CacheStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash, SubRange.first, SubRange.second); - if (m_AbortFlag) - { - break; - } - if (PayloadBuffer) - { - ProcessDownload(BlockDescription, - std::move(PayloadBuffer), - SubRangeStartIndex, - std::vector>{std::make_pair(0u, SubRange.second)}, - TotalRequestCount, - FilteredDownloadedBytesPerSecond, - OnDownloaded); - SubRangeCountComplete += SubRangeCount; - continue; - } - } - else - { - auto SubRanges = RangesSpan.subspan(SubRangeCountComplete, SubRangeCount); - - BuildStorageCache::BuildBlobRanges RangeBuffers = - m_Storage.CacheStorage->GetBuildBlobRanges(m_BuildId, BlockDescription.BlockHash, SubRanges); - if (m_AbortFlag) - { - break; - } - if (RangeBuffers.PayloadBuffer) - { - if (RangeBuffers.Ranges.empty()) - { - SubRangeCount = Ranges.size() - SubRangeCountComplete; - ProcessDownload(BlockDescription, - std::move(RangeBuffers.PayloadBuffer), - SubRangeStartIndex, - RangesSpan.subspan(SubRangeCountComplete, SubRangeCount), - TotalRequestCount, - FilteredDownloadedBytesPerSecond, - OnDownloaded); - SubRangeCountComplete += SubRangeCount; - continue; - } - else if (RangeBuffers.Ranges.size() == SubRangeCount) - { - ProcessDownload(BlockDescription, - std::move(RangeBuffers.PayloadBuffer), - SubRangeStartIndex, - RangeBuffers.Ranges, - TotalRequestCount, - FilteredDownloadedBytesPerSecond, - OnDownloaded); - SubRangeCountComplete += SubRangeCount; - continue; - } - } - } - } - - size_t SubRangeCount = Min(BlockRangeCount - SubRangeCountComplete, m_Storage.BuildStorageHost.Caps.MaxRangeCountPerRequest); - - auto SubRanges = RangesSpan.subspan(SubRangeCountComplete, SubRangeCount); - - BuildStorageBase::BuildBlobRanges RangeBuffers; - - try - { - RangeBuffers = m_Storage.BuildStorage->GetBuildBlobRanges(m_BuildId, BlockDescription.BlockHash, SubRanges); - } - catch (const std::exception&) - { - // Silence http errors due to abort - if (!m_AbortFlag) - { - throw; - } - } - - if (!m_AbortFlag) - { - if (RangeBuffers.PayloadBuffer) - { - if (RangeBuffers.Ranges.empty()) - { - // Jupiter will ignore the ranges and send the whole payload if it fetches the payload from S3 - // Upload to cache (if enabled) and use the whole payload for the remaining ranges - - const uint64_t Size = RangeBuffers.PayloadBuffer.GetSize(); - - const bool PopulateCache = !ExistsInCache && m_Storage.CacheStorage && m_Options.PopulateCache; - - std::filesystem::path BlockPath = - TryMoveDownloadedChunk(RangeBuffers.PayloadBuffer, - m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString(), - /* ForceDiskBased */ PopulateCache || Size > m_Options.MaximumInMemoryPayloadSize); - if (!BlockPath.empty()) - { - RangeBuffers.PayloadBuffer = IoBufferBuilder::MakeFromFile(BlockPath); - if (!RangeBuffers.PayloadBuffer) - { - throw std::runtime_error( - fmt::format("Failed to read block {} from temporary path '{}'", BlockDescription.BlockHash, BlockPath)); - } - RangeBuffers.PayloadBuffer.SetDeleteOnClose(true); - } - - if (PopulateCache) - { - m_Storage.CacheStorage->PutBuildBlob(m_BuildId, - BlockDescription.BlockHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(SharedBuffer(RangeBuffers.PayloadBuffer))); - } - - if (m_AbortFlag) - { - break; - } - - SubRangeCount = Ranges.size() - SubRangeCountComplete; - ProcessDownload(BlockDescription, - std::move(RangeBuffers.PayloadBuffer), - SubRangeStartIndex, - RangesSpan.subspan(SubRangeCountComplete, SubRangeCount), - TotalRequestCount, - FilteredDownloadedBytesPerSecond, - OnDownloaded); - } - else - { - if (RangeBuffers.Ranges.size() != SubRanges.size()) - { - throw std::runtime_error(fmt::format("Fetching {} ranges from {} resulted in {} ranges", - SubRanges.size(), - BlockDescription.BlockHash, - RangeBuffers.Ranges.size())); - } - ProcessDownload(BlockDescription, - std::move(RangeBuffers.PayloadBuffer), - SubRangeStartIndex, - RangeBuffers.Ranges, - TotalRequestCount, - FilteredDownloadedBytesPerSecond, - OnDownloaded); - } - } - else - { - throw std::runtime_error( - fmt::format("Block {} is missing when fetching {} ranges", BlockDescription.BlockHash, SubRangeCount)); - } - - SubRangeCountComplete += SubRangeCount; - } - } -} - -std::vector -BuildsOperationUpdateFolder::WriteLocalChunkToCache(CloneQueryInterface* CloneQuery, - const CopyChunkData& CopyData, - const std::vector& ScavengedContents, - const std::vector& ScavengedLookups, - const std::vector& ScavengedPaths, - BufferedWriteFileCache& WriteCache) -{ - ZEN_TRACE_CPU("WriteLocalChunkToCache"); - - std::filesystem::path SourceFilePath; - - if (CopyData.ScavengeSourceIndex == (uint32_t)-1) - { - const uint32_t LocalPathIndex = m_LocalLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; - SourceFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred(); - } - else - { - const ChunkedFolderContent& ScavengedContent = ScavengedContents[CopyData.ScavengeSourceIndex]; - const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[CopyData.ScavengeSourceIndex]; - const std::filesystem::path ScavengedPath = ScavengedPaths[CopyData.ScavengeSourceIndex]; - const uint32_t ScavengedPathIndex = ScavengedLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; - SourceFilePath = (ScavengedPath / ScavengedContent.Paths[ScavengedPathIndex]).make_preferred(); - } - ZEN_ASSERT_SLOW(IsFile(SourceFilePath)); - ZEN_ASSERT(!CopyData.TargetChunkLocationPtrs.empty()); - - uint64_t CacheLocalFileBytesRead = 0; - - size_t TargetStart = 0; - const std::span AllTargets(CopyData.TargetChunkLocationPtrs); - - struct WriteOp - { - const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr; - uint64_t CacheFileOffset = (uint64_t)-1; - uint32_t ChunkIndex = (uint32_t)-1; - }; - - std::vector WriteOps; - - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Sort"); - WriteOps.reserve(AllTargets.size()); - for (const CopyChunkData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) - { - std::span TargetRange = - AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); - for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange) - { - WriteOps.push_back( - WriteOp{.Target = Target, .CacheFileOffset = ChunkTarget.CacheFileOffset, .ChunkIndex = ChunkTarget.RemoteChunkIndex}); - } - TargetStart += ChunkTarget.TargetChunkLocationCount; - } - - std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { - if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) - { - return true; - } - else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) - { - return false; - } - if (Lhs.Target->Offset < Rhs.Target->Offset) - { - return true; - } - return false; - }); - } - - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Write"); - - tsl::robin_set ChunkIndexesWritten; - - BufferedOpenFile SourceFile(SourceFilePath, - m_DiskStats.OpenReadCount, - m_DiskStats.CurrentOpenFileCount, - m_DiskStats.ReadCount, - m_DiskStats.ReadByteCount); - - bool CanCloneSource = CloneQuery && CloneQuery->CanClone(SourceFile.Handle()); - - BufferedWriteFileCache::Local LocalWriter(WriteCache); - - for (size_t WriteOpIndex = 0; WriteOpIndex < WriteOps.size();) - { - if (m_AbortFlag) - { - break; - } - const WriteOp& Op = WriteOps[WriteOpIndex]; - - const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; - const uint32_t RemotePathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; - const uint64_t TargetSize = m_RemoteContent.RawSizes[RemotePathIndex]; - const uint64_t ChunkSize = m_RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex]; - - uint64_t ReadLength = ChunkSize; - size_t WriteCount = 1; - uint64_t OpSourceEnd = Op.CacheFileOffset + ChunkSize; - uint64_t OpTargetEnd = Op.Target->Offset + ChunkSize; - while ((WriteOpIndex + WriteCount) < WriteOps.size()) - { - const WriteOp& NextOp = WriteOps[WriteOpIndex + WriteCount]; - if (NextOp.Target->SequenceIndex != Op.Target->SequenceIndex) - { - break; - } - if (NextOp.Target->Offset != OpTargetEnd) - { - break; - } - if (NextOp.CacheFileOffset != OpSourceEnd) - { - break; - } - const uint64_t NextChunkLength = m_RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex]; - if (ReadLength + NextChunkLength > BufferedOpenFile::BlockSize) - { - break; - } - ReadLength += NextChunkLength; - OpSourceEnd += NextChunkLength; - OpTargetEnd += NextChunkLength; - WriteCount++; - } - - { - bool DidClone = false; - - if (CanCloneSource) - { - uint64_t PreBytes = 0; - uint64_t PostBytes = 0; - uint64_t ClonableBytes = - CloneQuery->GetClonableRange(Op.CacheFileOffset, Op.Target->Offset, ReadLength, PreBytes, PostBytes); - if (ClonableBytes > 0) - { - // We need to open the file... - BufferedWriteFileCache::Local::Writer* Writer = LocalWriter.GetWriter(RemoteSequenceIndex); - if (!Writer) - { - Writer = LocalWriter.PutWriter(RemoteSequenceIndex, std::make_unique()); - - Writer->File = std::make_unique(); - - const std::filesystem::path FileName = - GetTempChunkedSequenceFileName(m_CacheFolderPath, - m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]); - Writer->File->Open(FileName, BasicFile::Mode::kWrite); - if (m_Options.UseSparseFiles) - { - PrepareFileForScatteredWrite(Writer->File->Handle(), TargetSize); - } - } - DidClone = CloneQuery->TryClone(SourceFile.Handle(), - Writer->File->Handle(), - Op.CacheFileOffset + PreBytes, - Op.Target->Offset + PreBytes, - ClonableBytes, - TargetSize); - if (DidClone) - { - m_DiskStats.WriteCount++; - m_DiskStats.WriteByteCount += ClonableBytes; - - m_DiskStats.CloneCount++; - m_DiskStats.CloneByteCount += ClonableBytes; - - m_WrittenChunkByteCount += ClonableBytes; - - if (PreBytes > 0) - { - CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, PreBytes); - const uint64_t FileOffset = Op.Target->Offset; - - WriteSequenceChunkToCache(LocalWriter, ChunkSource, RemoteSequenceIndex, FileOffset, RemotePathIndex); - } - if (PostBytes > 0) - { - CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset + ReadLength - PostBytes, PostBytes); - const uint64_t FileOffset = Op.Target->Offset + ReadLength - PostBytes; - - WriteSequenceChunkToCache(LocalWriter, ChunkSource, RemoteSequenceIndex, FileOffset, RemotePathIndex); - } - } - } - } - - if (!DidClone) - { - CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ReadLength); - - const uint64_t FileOffset = Op.Target->Offset; - - WriteSequenceChunkToCache(LocalWriter, ChunkSource, RemoteSequenceIndex, FileOffset, RemotePathIndex); - } - } - - CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes? - - WriteOpIndex += WriteCount; - } - } - - if (m_Options.IsVerbose) - { - ZEN_INFO("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), SourceFilePath); - } - - std::vector Result; - Result.reserve(WriteOps.size()); - - for (const WriteOp& Op : WriteOps) - { - Result.push_back(Op.Target->SequenceIndex); - } - return Result; -} - -bool -BuildsOperationUpdateFolder::WriteCompressedChunkToCache( - const IoHash& ChunkHash, - const std::vector& ChunkTargetPtrs, - BufferedWriteFileCache& WriteCache, - IoBuffer&& CompressedPart) -{ - ZEN_TRACE_CPU("WriteCompressedChunkToCache"); - - auto ChunkHashToChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); - ZEN_ASSERT(ChunkHashToChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end()); - if (IsSingleFileChunk(m_RemoteContent, ChunkTargetPtrs)) - { - const std::uint32_t SequenceIndex = ChunkTargetPtrs.front()->SequenceIndex; - const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]; - StreamDecompress(SequenceRawHash, CompositeBuffer(std::move(CompressedPart))); - return false; - } - else - { - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompositeBuffer(std::move(CompressedPart)), RawHash, RawSize); - if (!Compressed) - { - throw std::runtime_error(fmt::format("Failed to parse header of compressed large blob {}", ChunkHash)); - } - if (RawHash != ChunkHash) - { - throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, ChunkHash)); - } - - BufferedWriteFileCache::Local LocalWriter(WriteCache); - - IoHashStream Hash; - bool CouldDecompress = Compressed.DecompressToStream( - 0, - (uint64_t)-1, - [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { - ZEN_UNUSED(SourceOffset); - ZEN_TRACE_CPU("Async_StreamDecompress_Write"); - m_DiskStats.ReadByteCount += SourceSize; - if (!m_AbortFlag) - { - for (const ChunkedContentLookup::ChunkSequenceLocation* TargetPtr : ChunkTargetPtrs) - { - const auto& Target = *TargetPtr; - const uint64_t FileOffset = Target.Offset + Offset; - const uint32_t SequenceIndex = Target.SequenceIndex; - const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; - - WriteSequenceChunkToCache(LocalWriter, RangeBuffer, SequenceIndex, FileOffset, PathIndex); - } - - return true; - } - return false; - }); - - if (m_AbortFlag) - { - return false; - } - - if (!CouldDecompress) - { - throw std::runtime_error(fmt::format("Failed to decompress large chunk {}", ChunkHash)); - } - - return true; - } -} - -void -BuildsOperationUpdateFolder::StreamDecompress(const IoHash& SequenceRawHash, CompositeBuffer&& CompressedPart) -{ - ZEN_TRACE_CPU("StreamDecompress"); - const std::filesystem::path TempChunkSequenceFileName = GetTempChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash); - TemporaryFile DecompressedTemp; - std::error_code Ec; - DecompressedTemp.CreateTemporary(TempChunkSequenceFileName.parent_path(), Ec); - if (Ec) - { - throw std::runtime_error(fmt::format("Failed creating temporary file for decompressing large blob {}, reason: ({}) {}", - SequenceRawHash, - Ec.value(), - Ec.message())); - } - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompressedPart, RawHash, RawSize); - if (!Compressed) - { - throw std::runtime_error(fmt::format("Failed to parse header of compressed large blob {}", SequenceRawHash)); - } - if (RawHash != SequenceRawHash) - { - throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, SequenceRawHash)); - } - PrepareFileForScatteredWrite(DecompressedTemp.Handle(), RawSize); - - IoHashStream Hash; - bool CouldDecompress = - Compressed.DecompressToStream(0, - (uint64_t)-1, - [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { - ZEN_UNUSED(SourceOffset); - ZEN_TRACE_CPU("StreamDecompress_Write"); - m_DiskStats.ReadCount++; - m_DiskStats.ReadByteCount += SourceSize; - if (!m_AbortFlag) - { - for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) - { - if (m_Options.ValidateCompletedSequences) - { - Hash.Append(Segment.GetView()); - m_ValidatedChunkByteCount += Segment.GetSize(); - } - DecompressedTemp.Write(Segment, Offset); - Offset += Segment.GetSize(); - m_DiskStats.WriteByteCount += Segment.GetSize(); - m_DiskStats.WriteCount++; - m_WrittenChunkByteCount += Segment.GetSize(); - } - return true; - } - return false; - }); - - if (m_AbortFlag) - { - return; - } - - if (!CouldDecompress) - { - throw std::runtime_error(fmt::format("Failed to decompress large blob {}", SequenceRawHash)); - } - if (m_Options.ValidateCompletedSequences) - { - const IoHash VerifyHash = Hash.GetHash(); - if (VerifyHash != SequenceRawHash) - { - throw std::runtime_error( - fmt::format("Decompressed blob payload hash {} does not match expected hash {}", VerifyHash, SequenceRawHash)); - } - } - DecompressedTemp.MoveTemporaryIntoPlace(TempChunkSequenceFileName, Ec); - if (Ec) - { - throw std::runtime_error(fmt::format("Failed moving temporary file for decompressing large blob {}, reason: ({}) {}", - SequenceRawHash, - Ec.value(), - Ec.message())); - } - // WriteChunkStats.ChunkCountWritten++; -} - -void -BuildsOperationUpdateFolder::WriteSequenceChunkToCache(BufferedWriteFileCache::Local& LocalWriter, - const CompositeBuffer& Chunk, - const uint32_t SequenceIndex, - const uint64_t FileOffset, - const uint32_t PathIndex) -{ - ZEN_TRACE_CPU("WriteSequenceChunkToCache"); - - const uint64_t SequenceSize = m_RemoteContent.RawSizes[PathIndex]; - - auto OpenFile = [&](BasicFile& File) { - const std::filesystem::path FileName = - GetTempChunkedSequenceFileName(m_CacheFolderPath, m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); - File.Open(FileName, BasicFile::Mode::kWrite); - if (m_Options.UseSparseFiles) - { - PrepareFileForScatteredWrite(File.Handle(), SequenceSize); - } - }; - - const uint64_t ChunkSize = Chunk.GetSize(); - ZEN_ASSERT(FileOffset + ChunkSize <= SequenceSize); - if (ChunkSize == SequenceSize) - { - BasicFile SingleChunkFile; - OpenFile(SingleChunkFile); - - m_DiskStats.CurrentOpenFileCount++; - auto _ = MakeGuard([this]() { m_DiskStats.CurrentOpenFileCount--; }); - SingleChunkFile.Write(Chunk, FileOffset); - } - else - { - const uint64_t MaxWriterBufferSize = 256u * 1025u; - - BufferedWriteFileCache::Local::Writer* Writer = LocalWriter.GetWriter(SequenceIndex); - if (Writer) - { - if ((!Writer->Writer) && (ChunkSize < MaxWriterBufferSize)) - { - Writer->Writer = std::make_unique(*Writer->File, Min(SequenceSize, MaxWriterBufferSize)); - } - Writer->Write(Chunk, FileOffset); - } - else - { - Writer = LocalWriter.PutWriter(SequenceIndex, std::make_unique()); - - Writer->File = std::make_unique(); - OpenFile(*Writer->File); - if (ChunkSize < MaxWriterBufferSize) - { - Writer->Writer = std::make_unique(*Writer->File, Min(SequenceSize, MaxWriterBufferSize)); - } - Writer->Write(Chunk, FileOffset); - } - } - m_DiskStats.WriteCount++; - m_DiskStats.WriteByteCount += ChunkSize; - m_WrittenChunkByteCount += ChunkSize; -} - -bool -BuildsOperationUpdateFolder::GetBlockWriteOps(const IoHash& BlockRawHash, - std::span ChunkRawHashes, - std::span ChunkCompressedLengths, - std::span> SequenceIndexChunksLeftToWriteCounters, - std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - const MemoryView BlockView, - uint32_t FirstIncludedBlockChunkIndex, - uint32_t LastIncludedBlockChunkIndex, - BlockWriteOps& OutOps) -{ - ZEN_TRACE_CPU("GetBlockWriteOps"); - - uint32_t OffsetInBlock = 0; - for (uint32_t ChunkBlockIndex = FirstIncludedBlockChunkIndex; ChunkBlockIndex <= LastIncludedBlockChunkIndex; ChunkBlockIndex++) - { - const uint32_t ChunkCompressedSize = ChunkCompressedLengths[ChunkBlockIndex]; - const IoHash& ChunkHash = ChunkRawHashes[ChunkBlockIndex]; - if (auto It = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != m_RemoteLookup.ChunkHashToChunkIndex.end()) - { - const uint32_t ChunkIndex = It->second; - std::vector ChunkTargetPtrs = - GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, ChunkIndex); - - if (!ChunkTargetPtrs.empty()) - { - bool NeedsWrite = true; - if (RemoteChunkIndexNeedsCopyFromSourceFlags[ChunkIndex].compare_exchange_strong(NeedsWrite, false)) - { - MemoryView ChunkMemoryView = BlockView.Mid(OffsetInBlock, ChunkCompressedSize); - IoHash VerifyChunkHash; - uint64_t VerifyChunkSize; - CompressedBuffer CompressedChunk = - CompressedBuffer::FromCompressed(SharedBuffer::MakeView(ChunkMemoryView), VerifyChunkHash, VerifyChunkSize); - if (!CompressedChunk) - { - throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} is not a valid compressed buffer", - ChunkHash, - OffsetInBlock, - ChunkCompressedSize, - BlockRawHash)); - } - if (VerifyChunkHash != ChunkHash) - { - throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} has a mismatching content hash {}", - ChunkHash, - OffsetInBlock, - ChunkCompressedSize, - BlockRawHash, - VerifyChunkHash)); - } - if (VerifyChunkSize != m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]) - { - throw std::runtime_error( - fmt::format("Chunk {} at {}, size {} in block {} has a mismatching raw size {}, expected {}", - ChunkHash, - OffsetInBlock, - ChunkCompressedSize, - BlockRawHash, - VerifyChunkSize, - m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex])); - } - - OodleCompressor ChunkCompressor; - OodleCompressionLevel ChunkCompressionLevel; - uint64_t ChunkBlockSize; - - bool GetCompressParametersSuccess = - CompressedChunk.TryGetCompressParameters(ChunkCompressor, ChunkCompressionLevel, ChunkBlockSize); - ZEN_ASSERT(GetCompressParametersSuccess); - - IoBuffer Decompressed; - if (ChunkCompressionLevel == OodleCompressionLevel::None) - { - MemoryView ChunkDecompressedMemoryView = ChunkMemoryView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder()); - Decompressed = - IoBuffer(IoBuffer::Wrap, ChunkDecompressedMemoryView.GetData(), ChunkDecompressedMemoryView.GetSize()); - } - else - { - Decompressed = CompressedChunk.Decompress().AsIoBuffer(); - } - - if (Decompressed.GetSize() != m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]) - { - throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} decompressed to size {}, expected {}", - ChunkHash, - OffsetInBlock, - ChunkCompressedSize, - BlockRawHash, - Decompressed.GetSize(), - m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex])); - } - - ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed)); - for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs) - { - OutOps.WriteOps.push_back( - BlockWriteOps::WriteOpData{.Target = Target, .ChunkBufferIndex = OutOps.ChunkBuffers.size()}); - } - OutOps.ChunkBuffers.emplace_back(std::move(Decompressed)); - } - } - } - - OffsetInBlock += ChunkCompressedSize; - } - { - ZEN_TRACE_CPU("Sort"); - std::sort(OutOps.WriteOps.begin(), - OutOps.WriteOps.end(), - [](const BlockWriteOps::WriteOpData& Lhs, const BlockWriteOps::WriteOpData& Rhs) { - if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) - { - return true; - } - if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) - { - return false; - } - return Lhs.Target->Offset < Rhs.Target->Offset; - }); - } - return true; -} - -void -BuildsOperationUpdateFolder::WriteBlockChunkOpsToCache(std::span> SequenceIndexChunksLeftToWriteCounters, - const BlockWriteOps& Ops, - BufferedWriteFileCache& WriteCache, - ParallelWork& Work) -{ - ZEN_TRACE_CPU("WriteBlockChunkOpsToCache"); - - { - BufferedWriteFileCache::Local LocalWriter(WriteCache); - for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps) - { - if (Work.IsAborted()) - { - break; - } - const CompositeBuffer& Chunk = Ops.ChunkBuffers[WriteOp.ChunkBufferIndex]; - const uint32_t SequenceIndex = WriteOp.Target->SequenceIndex; - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() <= - m_RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]); - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() > 0); - const uint64_t FileOffset = WriteOp.Target->Offset; - const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; - - WriteSequenceChunkToCache(LocalWriter, Chunk, SequenceIndex, FileOffset, PathIndex); - } - } - if (!Work.IsAborted()) - { - // Write tracking, updating this must be done without any files open (BufferedWriteFileCache::Local) - std::vector CompletedChunkSequences; - for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps) - { - const uint32_t RemoteSequenceIndex = WriteOp.Target->SequenceIndex; - if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters)) - { - CompletedChunkSequences.push_back(RemoteSequenceIndex); - } - } - WriteCache.Close(CompletedChunkSequences); - VerifyAndCompleteChunkSequencesAsync(CompletedChunkSequences, Work); - } -} - -bool -BuildsOperationUpdateFolder::WriteChunksBlockToCache(const ChunkBlockDescription& BlockDescription, - std::span> SequenceIndexChunksLeftToWriteCounters, - ParallelWork& Work, - CompositeBuffer&& BlockBuffer, - std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - BufferedWriteFileCache& WriteCache) -{ - ZEN_TRACE_CPU("WriteChunksBlockToCache"); - - IoBuffer BlockMemoryBuffer = MakeBufferMemoryBased(BlockBuffer); - const MemoryView BlockView = BlockMemoryBuffer.GetView(); - - BlockWriteOps Ops; - if ((BlockDescription.HeaderSize == 0) || BlockDescription.ChunkCompressedLengths.empty()) - { - ZEN_TRACE_CPU("WriteChunksBlockToCache_Legacy"); - - uint64_t HeaderSize; - const std::vector ChunkCompressedLengths = - ReadChunkBlockHeader(BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder()), HeaderSize); - - if (GetBlockWriteOps(BlockDescription.BlockHash, - BlockDescription.ChunkRawHashes, - ChunkCompressedLengths, - SequenceIndexChunksLeftToWriteCounters, - RemoteChunkIndexNeedsCopyFromSourceFlags, - BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + HeaderSize), - 0, - gsl::narrow(BlockDescription.ChunkRawHashes.size() - 1), - Ops)) - { - WriteBlockChunkOpsToCache(SequenceIndexChunksLeftToWriteCounters, Ops, WriteCache, Work); - return true; - } - return false; - } - - if (GetBlockWriteOps(BlockDescription.BlockHash, - BlockDescription.ChunkRawHashes, - BlockDescription.ChunkCompressedLengths, - SequenceIndexChunksLeftToWriteCounters, - RemoteChunkIndexNeedsCopyFromSourceFlags, - BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize), - 0, - gsl::narrow(BlockDescription.ChunkRawHashes.size() - 1), - Ops)) - { - WriteBlockChunkOpsToCache(SequenceIndexChunksLeftToWriteCounters, Ops, WriteCache, Work); - return true; - } - return false; -} - -bool -BuildsOperationUpdateFolder::WritePartialBlockChunksToCache(const ChunkBlockDescription& BlockDescription, - std::span> SequenceIndexChunksLeftToWriteCounters, - ParallelWork& Work, - CompositeBuffer&& PartialBlockBuffer, - uint32_t FirstIncludedBlockChunkIndex, - uint32_t LastIncludedBlockChunkIndex, - std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - BufferedWriteFileCache& WriteCache) -{ - ZEN_TRACE_CPU("WritePartialBlockChunksToCache"); - - IoBuffer BlockMemoryBuffer = MakeBufferMemoryBased(PartialBlockBuffer); - const MemoryView BlockView = BlockMemoryBuffer.GetView(); - - BlockWriteOps Ops; - if (GetBlockWriteOps(BlockDescription.BlockHash, - BlockDescription.ChunkRawHashes, - BlockDescription.ChunkCompressedLengths, - SequenceIndexChunksLeftToWriteCounters, - RemoteChunkIndexNeedsCopyFromSourceFlags, - BlockView, - FirstIncludedBlockChunkIndex, - LastIncludedBlockChunkIndex, - Ops)) - { - WriteBlockChunkOpsToCache(SequenceIndexChunksLeftToWriteCounters, Ops, WriteCache, Work); - return true; - } - else - { - return false; - } -} - -void -BuildsOperationUpdateFolder::AsyncWriteDownloadedChunk(uint32_t RemoteChunkIndex, - const BlobsExistsResult& ExistsResult, - std::vector&& ChunkTargetPtrs, - BufferedWriteFileCache& WriteCache, - ParallelWork& Work, - IoBuffer&& Payload, - std::span> SequenceIndexChunksLeftToWriteCounters, - std::atomic& WritePartsComplete, - const uint64_t TotalPartWriteCount, - FilteredRate& FilteredWrittenBytesPerSecond) -{ - ZEN_TRACE_CPU("AsyncWriteDownloadedChunk"); - - const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - - const uint64_t Size = Payload.GetSize(); - - const bool ExistsInCache = m_Storage.CacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash); - - const bool PopulateCache = !ExistsInCache && m_Storage.CacheStorage && m_Options.PopulateCache; - - std::filesystem::path CompressedChunkPath = - TryMoveDownloadedChunk(Payload, - m_TempDownloadFolderPath / ChunkHash.ToHexString(), - /* ForceDiskBased */ PopulateCache || Size > m_Options.MaximumInMemoryPayloadSize); - if (PopulateCache) - { - IoBuffer CacheBlob = IoBufferBuilder::MakeFromFile(CompressedChunkPath); - if (CacheBlob) - { - m_Storage.CacheStorage->PutBuildBlob(m_BuildId, - ChunkHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(SharedBuffer(CacheBlob))); - } - } - - IoBufferFileReference FileRef; - bool EnableBacklog = !CompressedChunkPath.empty() || Payload.GetFileReference(FileRef); - - Work.ScheduleWork( - m_IOWorkerPool, - [this, - SequenceIndexChunksLeftToWriteCounters, - &Work, - CompressedChunkPath, - RemoteChunkIndex, - TotalPartWriteCount, - &WriteCache, - &WritePartsComplete, - &FilteredWrittenBytesPerSecond, - ChunkTargetPtrs = std::move(ChunkTargetPtrs), - CompressedPart = IoBuffer(std::move(Payload))](std::atomic&) mutable { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Async_WriteChunk"); - - FilteredWrittenBytesPerSecond.Start(); - - const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - if (CompressedChunkPath.empty()) - { - ZEN_ASSERT(CompressedPart); - } - else - { - ZEN_ASSERT(!CompressedPart); - CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); - if (!CompressedPart) - { - throw std::runtime_error( - fmt::format("Could not open dowloaded compressed chunk {} from {}", ChunkHash, CompressedChunkPath)); - } - } - - bool NeedHashVerify = WriteCompressedChunkToCache(ChunkHash, ChunkTargetPtrs, WriteCache, std::move(CompressedPart)); - if (!m_AbortFlag) - { - if (WritePartsComplete.fetch_add(1) + 1 == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } - - if (!CompressedChunkPath.empty()) - { - std::error_code Ec = TryRemoveFile(CompressedChunkPath); - if (Ec) - { - ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", CompressedChunkPath, Ec.value(), Ec.message()); - } - } - - std::vector CompletedSequences = - CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); - WriteCache.Close(CompletedSequences); - if (NeedHashVerify) - { - VerifyAndCompleteChunkSequencesAsync(CompletedSequences, Work); - } - else - { - FinalizeChunkSequences(CompletedSequences); - } - } - } - }, - EnableBacklog ? WorkerThreadPool::EMode::EnableBacklog : WorkerThreadPool::EMode::DisableBacklog); -} - -void -BuildsOperationUpdateFolder::VerifyAndCompleteChunkSequencesAsync(std::span RemoteSequenceIndexes, ParallelWork& Work) -{ - if (RemoteSequenceIndexes.empty()) - { - return; - } - ZEN_TRACE_CPU("VerifyAndCompleteChunkSequence"); - if (m_Options.ValidateCompletedSequences) - { - for (uint32_t RemoteSequenceIndexOffset = 1; RemoteSequenceIndexOffset < RemoteSequenceIndexes.size(); RemoteSequenceIndexOffset++) - { - const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[RemoteSequenceIndexOffset]; - Work.ScheduleWork(m_IOWorkerPool, [this, RemoteSequenceIndex](std::atomic&) { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Async_VerifyAndFinalizeSequence"); - - VerifySequence(RemoteSequenceIndex); - if (!m_AbortFlag) - { - const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - FinalizeChunkSequence(SequenceRawHash); - } - } - }); - } - const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[0]; - - VerifySequence(RemoteSequenceIndex); - const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - FinalizeChunkSequence(SequenceRawHash); - } - else - { - for (uint32_t RemoteSequenceIndexOffset = 0; RemoteSequenceIndexOffset < RemoteSequenceIndexes.size(); RemoteSequenceIndexOffset++) - { - const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[RemoteSequenceIndexOffset]; - const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - FinalizeChunkSequence(SequenceRawHash); - } - } -} - -bool -BuildsOperationUpdateFolder::CompleteSequenceChunk(uint32_t RemoteSequenceIndex, - std::span> SequenceIndexChunksLeftToWriteCounters) -{ - uint32_t PreviousValue = SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1); - ZEN_ASSERT(PreviousValue >= 1); - ZEN_ASSERT(PreviousValue != (uint32_t)-1); - return PreviousValue == 1; -} - -std::vector -BuildsOperationUpdateFolder::CompleteChunkTargets(const std::vector& ChunkTargetPtrs, - std::span> SequenceIndexChunksLeftToWriteCounters) -{ - ZEN_TRACE_CPU("CompleteChunkTargets"); - - std::vector CompletedSequenceIndexes; - for (const ChunkedContentLookup::ChunkSequenceLocation* Location : ChunkTargetPtrs) - { - const uint32_t RemoteSequenceIndex = Location->SequenceIndex; - if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters)) - { - CompletedSequenceIndexes.push_back(RemoteSequenceIndex); - } - } - return CompletedSequenceIndexes; -} - -void -BuildsOperationUpdateFolder::FinalizeChunkSequence(const IoHash& SequenceRawHash) -{ - ZEN_TRACE_CPU("FinalizeChunkSequence"); - - ZEN_ASSERT_SLOW(!IsFile(GetFinalChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash))); - std::error_code Ec; - RenameFile(GetTempChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash), - GetFinalChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash), - Ec); - if (Ec) - { - throw std::system_error(Ec); - } -} - -void -BuildsOperationUpdateFolder::FinalizeChunkSequences(std::span RemoteSequenceIndexes) -{ - ZEN_TRACE_CPU("FinalizeChunkSequences"); - - for (uint32_t SequenceIndex : RemoteSequenceIndexes) - { - FinalizeChunkSequence(m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); - } -} - -void -BuildsOperationUpdateFolder::VerifySequence(uint32_t RemoteSequenceIndex) -{ - ZEN_TRACE_CPU("VerifySequence"); - - ZEN_ASSERT(m_Options.ValidateCompletedSequences); - - const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - { - ZEN_TRACE_CPU("HashSequence"); - const std::uint32_t RemotePathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; - const uint64_t ExpectedSize = m_RemoteContent.RawSizes[RemotePathIndex]; - IoBuffer VerifyBuffer = IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash)); - const uint64_t VerifySize = VerifyBuffer.GetSize(); - if (VerifySize != ExpectedSize) - { - throw std::runtime_error(fmt::format("Written chunk sequence {} size {} does not match expected size {}", - SequenceRawHash, - VerifySize, - ExpectedSize)); - } - - const IoHash VerifyChunkHash = IoHash::HashBuffer(std::move(VerifyBuffer), &m_ValidatedChunkByteCount); - if (VerifyChunkHash != SequenceRawHash) - { - throw std::runtime_error( - fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash)); - } - } -} - -////////////////////// BuildsOperationUploadFolder - -BuildsOperationUploadFolder::BuildsOperationUploadFolder(LoggerRef Log, - ProgressBase& Progress, - StorageInstance& Storage, - std::atomic& AbortFlag, - std::atomic& PauseFlag, - WorkerThreadPool& IOWorkerPool, - WorkerThreadPool& NetworkPool, - const Oid& BuildId, - const std::filesystem::path& Path, - bool CreateBuild, - const CbObject& MetaData, - const Options& Options) -: m_Log(Log) -, m_Progress(Progress) -, m_Storage(Storage) -, m_AbortFlag(AbortFlag) -, m_PauseFlag(PauseFlag) -, m_IOWorkerPool(IOWorkerPool) -, m_NetworkPool(NetworkPool) -, m_BuildId(BuildId) -, m_Path(Path) -, m_CreateBuild(CreateBuild) -, m_MetaData(MetaData) -, m_Options(Options) -{ - m_NonCompressableExtensionHashes.reserve(Options.NonCompressableExtensions.size()); - for (const std::string& Extension : Options.NonCompressableExtensions) - { - m_NonCompressableExtensionHashes.insert(HashStringAsLowerDjb2(Extension)); - } -} - -BuildsOperationUploadFolder::PrepareBuildResult -BuildsOperationUploadFolder::PrepareBuild() -{ - ZEN_TRACE_CPU("PrepareBuild"); - - PrepareBuildResult Result; - Result.PreferredMultipartChunkSize = m_Options.PreferredMultipartChunkSize; - Stopwatch Timer; - if (m_CreateBuild) - { - ZEN_TRACE_CPU("CreateBuild"); - - Stopwatch PutBuildTimer; - CbObject PutBuildResult = m_Storage.BuildStorage->PutBuild(m_BuildId, m_MetaData); - Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs(); - if (auto ChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(); ChunkSize != 0) - { - Result.PreferredMultipartChunkSize = ChunkSize; - } - Result.PayloadSize = m_MetaData.GetSize(); - } - else - { - ZEN_TRACE_CPU("PutBuild"); - Stopwatch GetBuildTimer; - CbObject Build = m_Storage.BuildStorage->GetBuild(m_BuildId); - Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs(); - Result.PayloadSize = Build.GetSize(); - if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) - { - Result.PreferredMultipartChunkSize = ChunkSize; - } - else if (m_Options.AllowMultiparts) - { - ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", NiceBytes(Result.PreferredMultipartChunkSize)); - } - } - - if (!m_Options.IgnoreExistingBlocks) - { - ZEN_TRACE_CPU("FindBlocks"); - Stopwatch KnownBlocksTimer; - CbObject BlockDescriptionList = m_Storage.BuildStorage->FindBlocks(m_BuildId, m_Options.FindBlockMaxCount); - if (BlockDescriptionList) - { - Result.KnownBlocks = ParseChunkBlockDescriptionList(BlockDescriptionList); - } - Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); - } - Result.ElapsedTimeMs = Timer.GetElapsedTimeMs(); - return Result; -} - -std::vector -BuildsOperationUploadFolder::ReadFolder() -{ - std::vector UploadParts; - std::filesystem::path ExcludeManifestPath = m_Path / m_Options.ZenExcludeManifestName; - tsl::robin_set ExcludeAssetPaths; - if (IsFile(ExcludeManifestPath)) - { - std::filesystem::path AbsoluteExcludeManifestPath = - MakeSafeAbsolutePath(ExcludeManifestPath.is_absolute() ? ExcludeManifestPath : m_Path / ExcludeManifestPath); - BuildManifest Manifest = ParseBuildManifest(AbsoluteExcludeManifestPath); - const std::vector& AssetPaths = Manifest.Parts.front().Files; - ExcludeAssetPaths.reserve(AssetPaths.size()); - for (const std::filesystem::path& AssetPath : AssetPaths) - { - ExcludeAssetPaths.insert(AssetPath.generic_string()); - } - } - - UploadParts.resize(1); - - UploadPart& Part = UploadParts.front(); - GetFolderContentStatistics& LocalFolderScanStats = Part.LocalFolderScanStats; - - Part.Content = GetFolderContent( - Part.LocalFolderScanStats, - m_Path, - [this](const std::string_view& RelativePath) { return IsAcceptedFolder(RelativePath); }, - [this, &ExcludeAssetPaths](const std::string_view& RelativePath, uint64_t Size, uint32_t Attributes) -> bool { - ZEN_UNUSED(Size, Attributes); - if (!IsAcceptedFile(RelativePath)) - { - return false; - } - if (ExcludeAssetPaths.contains(std::filesystem::path(RelativePath).generic_string())) - { - return false; - } - return true; - }, - m_IOWorkerPool, - m_Progress.GetProgressUpdateDelayMS(), - [&](bool, std::ptrdiff_t) { ZEN_INFO("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), m_Path); }, - m_AbortFlag); - Part.TotalRawSize = std::accumulate(Part.Content.RawSizes.begin(), Part.Content.RawSizes.end(), std::uint64_t(0)); - - return UploadParts; -} - -std::vector -BuildsOperationUploadFolder::ReadManifestParts(const std::filesystem::path& ManifestPath) -{ - std::vector UploadParts; - Stopwatch ManifestParseTimer; - std::filesystem::path AbsoluteManifestPath = MakeSafeAbsolutePath(ManifestPath.is_absolute() ? ManifestPath : m_Path / ManifestPath); - BuildManifest Manifest = ParseBuildManifest(AbsoluteManifestPath); - if (Manifest.Parts.empty()) - { - throw std::runtime_error(fmt::format("Manifest file at '{}' is invalid", ManifestPath)); - } - - UploadParts.resize(Manifest.Parts.size()); - for (size_t PartIndex = 0; PartIndex < Manifest.Parts.size(); PartIndex++) - { - BuildManifest::Part& PartManifest = Manifest.Parts[PartIndex]; - if (ManifestPath.is_relative()) - { - PartManifest.Files.push_back(ManifestPath); - } - - UploadPart& Part = UploadParts[PartIndex]; - FolderContent& Content = Part.Content; - - GetFolderContentStatistics& LocalFolderScanStats = Part.LocalFolderScanStats; - - const std::vector& AssetPaths = PartManifest.Files; - Content = GetValidFolderContent( - m_IOWorkerPool, - LocalFolderScanStats, - m_Path, - AssetPaths, - [](uint64_t PathCount, uint64_t CompletedPathCount) { ZEN_UNUSED(PathCount, CompletedPathCount); }, - 1000, - m_AbortFlag, - m_PauseFlag); - - if (Content.Paths.size() != AssetPaths.size()) - { - const tsl::robin_set FoundPaths(Content.Paths.begin(), Content.Paths.end()); - ExtendableStringBuilder<1024> SB; - for (const std::filesystem::path& AssetPath : AssetPaths) - { - if (!FoundPaths.contains(AssetPath)) - { - SB << "\n " << AssetPath.generic_string(); - } - } - throw std::runtime_error( - fmt::format("Manifest file at '{}' references files that does not exist{}", ManifestPath, SB.ToView())); - } - - Part.PartId = PartManifest.PartId; - Part.PartName = PartManifest.PartName; - Part.TotalRawSize = std::accumulate(Part.Content.RawSizes.begin(), Part.Content.RawSizes.end(), std::uint64_t(0)); - } - - return UploadParts; -} - -std::vector> -BuildsOperationUploadFolder::Execute(const Oid& BuildPartId, - const std::string_view BuildPartName, - const std::filesystem::path& ManifestPath, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache) -{ - ZEN_TRACE_CPU("BuildsOperationUploadFolder::Execute"); - try - { - Stopwatch ReadPartsTimer; - std::vector UploadParts = ManifestPath.empty() ? ReadFolder() : ReadManifestParts(ManifestPath); - - for (UploadPart& Part : UploadParts) - { - if (Part.PartId == Oid::Zero) - { - if (UploadParts.size() != 1) - { - throw std::runtime_error(fmt::format("Multi part upload manifest '{}' must contains build part id", ManifestPath)); - } - - if (BuildPartId == Oid::Zero) - { - Part.PartId = Oid::NewOid(); - } - else - { - Part.PartId = BuildPartId; - } - } - if (Part.PartName.empty()) - { - if (UploadParts.size() != 1) - { - throw std::runtime_error(fmt::format("Multi part upload manifest '{}' must contains build part name", ManifestPath)); - } - if (BuildPartName.empty()) - { - throw std::runtime_error("Build part name must be set"); - } - Part.PartName = std::string(BuildPartName); - } - } - - if (!m_Options.IsQuiet) - { - ZEN_INFO("Reading {} parts took {}", UploadParts.size(), NiceTimeSpanMs(ReadPartsTimer.GetElapsedTimeMs())); - } - - const uint32_t PartsUploadStepCount = gsl::narrow(uint32_t(PartTaskSteps::StepCount) * UploadParts.size()); - - const uint32_t PrepareBuildStep = 0; - const uint32_t UploadPartsStep = 1; - const uint32_t FinalizeBuildStep = UploadPartsStep + PartsUploadStepCount; - const uint32_t CleanupStep = FinalizeBuildStep + 1; - const uint32_t StepCount = CleanupStep + 1; - - auto EndProgress = MakeGuard([&]() { m_Progress.SetLogOperationProgress(StepCount, StepCount); }); - - Stopwatch ProcessTimer; - - CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir); - CreateDirectories(m_Options.TempDir); - auto _ = MakeGuard([&]() { CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir); }); - - m_Progress.SetLogOperationProgress(PrepareBuildStep, StepCount); - - m_PrepBuildResultFuture = m_NetworkPool.EnqueueTask(std::packaged_task{[this] { return PrepareBuild(); }}, - WorkerThreadPool::EMode::EnableBacklog); - - for (uint32_t PartIndex = 0; PartIndex < UploadParts.size(); PartIndex++) - { - const uint32_t PartStepOffset = UploadPartsStep + (PartIndex * uint32_t(PartTaskSteps::StepCount)); - - const UploadPart& Part = UploadParts[PartIndex]; - UploadBuildPart(ChunkController, ChunkCache, PartIndex, Part, PartStepOffset, StepCount); - if (m_AbortFlag) - { - return {}; - } - } - - m_Progress.SetLogOperationProgress(FinalizeBuildStep, StepCount); - - if (m_CreateBuild && !m_AbortFlag) - { - Stopwatch FinalizeBuildTimer; - m_Storage.BuildStorage->FinalizeBuild(m_BuildId); - if (!m_Options.IsQuiet) - { - ZEN_INFO("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs())); - } - } - - m_Progress.SetLogOperationProgress(CleanupStep, StepCount); - - std::vector> Result; - Result.reserve(UploadParts.size()); - for (UploadPart& Part : UploadParts) - { - Result.push_back(std::make_pair(Part.PartId, Part.PartName)); - } - return Result; - } - catch (const std::exception&) - { - m_AbortFlag = true; - throw; - } -} - -bool -BuildsOperationUploadFolder::IsAcceptedFolder(const std::string_view& RelativePath) const -{ - for (const std::string& ExcludeFolder : m_Options.ExcludeFolders) - { - if (RelativePath.starts_with(ExcludeFolder)) - { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } - } - } - return true; -} - -bool -BuildsOperationUploadFolder::IsAcceptedFile(const std::string_view& RelativePath) const -{ - if (RelativePath == m_Options.ZenExcludeManifestName) - { - return false; - } - for (const std::string& ExcludeExtension : m_Options.ExcludeExtensions) - { - if (RelativePath.ends_with(ExcludeExtension)) - { - return false; - } - } - return true; -} - -void -BuildsOperationUploadFolder::ArrangeChunksIntoBlocks(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - std::vector& ChunkIndexes, - std::vector>& OutBlocks) -{ - ZEN_TRACE_CPU("ArrangeChunksIntoBlocks"); - std::sort(ChunkIndexes.begin(), ChunkIndexes.end(), [&Content, &Lookup](uint32_t Lhs, uint32_t Rhs) { - const ChunkedContentLookup::ChunkSequenceLocation& LhsLocation = GetChunkSequenceLocations(Lookup, Lhs)[0]; - const ChunkedContentLookup::ChunkSequenceLocation& RhsLocation = GetChunkSequenceLocations(Lookup, Rhs)[0]; - if (LhsLocation.SequenceIndex < RhsLocation.SequenceIndex) - { - return true; - } - else if (LhsLocation.SequenceIndex > RhsLocation.SequenceIndex) - { - return false; - } - return LhsLocation.Offset < RhsLocation.Offset; - }); - - uint64_t MaxBlockSizeLowThreshold = m_Options.BlockParameters.MaxBlockSize - (m_Options.BlockParameters.MaxBlockSize / 16); - - uint64_t BlockSize = 0; - - uint32_t ChunkIndexStart = 0; - for (uint32_t ChunkIndexOffset = 0; ChunkIndexOffset < ChunkIndexes.size();) - { - const uint32_t ChunkIndex = ChunkIndexes[ChunkIndexOffset]; - const uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; - - if (((BlockSize + ChunkSize) > m_Options.BlockParameters.MaxBlockSize) || - (ChunkIndexOffset - ChunkIndexStart) > m_Options.BlockParameters.MaxChunksPerBlock) - { - // Within the span of MaxBlockSizeLowThreshold and MaxBlockSize, see if there is a break - // between source paths for chunks. Break the block at the last such break if any. - ZEN_ASSERT(ChunkIndexOffset > ChunkIndexStart); - - const uint32_t ChunkSequenceIndex = Lookup.ChunkSequenceLocations[Lookup.ChunkSequenceLocationOffset[ChunkIndex]].SequenceIndex; - - uint64_t ScanBlockSize = BlockSize; - - uint32_t ScanChunkIndexOffset = ChunkIndexOffset - 1; - while (ScanChunkIndexOffset > (ChunkIndexStart + 2)) - { - const uint32_t TestChunkIndex = ChunkIndexes[ScanChunkIndexOffset]; - const uint64_t TestChunkSize = Content.ChunkedContent.ChunkRawSizes[TestChunkIndex]; - if ((ScanBlockSize - TestChunkSize) < MaxBlockSizeLowThreshold) - { - break; - } - - const uint32_t TestSequenceIndex = - Lookup.ChunkSequenceLocations[Lookup.ChunkSequenceLocationOffset[TestChunkIndex]].SequenceIndex; - if (ChunkSequenceIndex != TestSequenceIndex) - { - ChunkIndexOffset = ScanChunkIndexOffset + 1; - break; - } - - ScanBlockSize -= TestChunkSize; - ScanChunkIndexOffset--; - } - - std::vector ChunksInBlock; - ChunksInBlock.reserve(ChunkIndexOffset - ChunkIndexStart); - for (uint32_t AddIndexOffset = ChunkIndexStart; AddIndexOffset < ChunkIndexOffset; AddIndexOffset++) - { - const uint32_t AddChunkIndex = ChunkIndexes[AddIndexOffset]; - ChunksInBlock.push_back(AddChunkIndex); - } - OutBlocks.emplace_back(std::move(ChunksInBlock)); - BlockSize = 0; - ChunkIndexStart = ChunkIndexOffset; - } - else - { - ChunkIndexOffset++; - BlockSize += ChunkSize; - } - } - if (ChunkIndexStart < ChunkIndexes.size()) - { - std::vector ChunksInBlock; - ChunksInBlock.reserve(ChunkIndexes.size() - ChunkIndexStart); - for (uint32_t AddIndexOffset = ChunkIndexStart; AddIndexOffset < ChunkIndexes.size(); AddIndexOffset++) - { - const uint32_t AddChunkIndex = ChunkIndexes[AddIndexOffset]; - ChunksInBlock.push_back(AddChunkIndex); - } - OutBlocks.emplace_back(std::move(ChunksInBlock)); - } -} - -void -BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const std::vector>& NewBlockChunks, - GeneratedBlocks& OutBlocks, - GenerateBlocksStatistics& GenerateBlocksStats, - UploadStatistics& UploadStats) -{ - ZEN_TRACE_CPU("GenerateBuildBlocks"); - const std::size_t NewBlockCount = NewBlockChunks.size(); - if (NewBlockCount == 0) - { - return; - } - - std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Generate Blocks"); - - OutBlocks.BlockDescriptions.resize(NewBlockCount); - OutBlocks.BlockSizes.resize(NewBlockCount); - OutBlocks.BlockMetaDatas.resize(NewBlockCount); - OutBlocks.BlockHeaders.resize(NewBlockCount); - OutBlocks.MetaDataHasBeenUploaded.resize(NewBlockCount, 0); - OutBlocks.BlockHashToBlockIndex.reserve(NewBlockCount); - - RwLock Lock; - FilteredRate FilteredGeneratedBytesPerSecond; - FilteredRate FilteredUploadedBytesPerSecond; - ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - std::atomic QueuedPendingBlocksForUpload = 0; - - GenerateBuildBlocksContext Context{.Work = Work, - .GenerateBlobsPool = m_IOWorkerPool, - .UploadBlocksPool = m_NetworkPool, - .FilteredGeneratedBytesPerSecond = FilteredGeneratedBytesPerSecond, - .FilteredUploadedBytesPerSecond = FilteredUploadedBytesPerSecond, - .QueuedPendingBlocksForUpload = QueuedPendingBlocksForUpload, - .Lock = Lock, - .OutBlocks = OutBlocks, - .GenerateBlocksStats = GenerateBlocksStats, - .UploadStats = UploadStats, - .NewBlockCount = NewBlockCount}; - - ScheduleBlockGeneration(Context, Content, Lookup, NewBlockChunks); - - Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(PendingWork); - - FilteredGeneratedBytesPerSecond.Update(GenerateBlocksStats.GeneratedBlockByteCount.load()); - FilteredUploadedBytesPerSecond.Update(UploadStats.BlocksBytes.load()); - - std::string Details = fmt::format("Generated {}/{} ({}, {}B/s). Uploaded {}/{} ({}, {}bits/s)", - GenerateBlocksStats.GeneratedBlockCount.load(), - NewBlockCount, - NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()), - NiceNum(FilteredGeneratedBytesPerSecond.GetCurrent()), - UploadStats.BlockCount.load(), - NewBlockCount, - NiceBytes(UploadStats.BlocksBytes.load()), - NiceNum(FilteredUploadedBytesPerSecond.GetCurrent() * 8)); - - ProgressBar->UpdateState({.Task = "Generating blocks", - .Details = Details, - .TotalCount = gsl::narrow(NewBlockCount), - .RemainingCount = gsl::narrow(NewBlockCount - GenerateBlocksStats.GeneratedBlockCount.load()), - .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }); - - ZEN_ASSERT(m_AbortFlag || QueuedPendingBlocksForUpload.load() == 0); - - ProgressBar->Finish(); - - GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGeneratedBytesPerSecond.GetElapsedTimeUS(); - UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTimeUS(); -} - -void -BuildsOperationUploadFolder::ScheduleBlockGeneration(GenerateBuildBlocksContext& Context, - const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const std::vector>& NewBlockChunks) -{ - for (size_t BlockIndex = 0; BlockIndex < Context.NewBlockCount; BlockIndex++) - { - if (Context.Work.IsAborted()) - { - break; - } - const std::vector& ChunksInBlock = NewBlockChunks[BlockIndex]; - Context.Work.ScheduleWork( - Context.GenerateBlobsPool, - [this, &Context, &Content, &Lookup, ChunksInBlock, BlockIndex](std::atomic&) { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("GenerateBuildBlocks_Generate"); - - Context.FilteredGeneratedBytesPerSecond.Start(); - - Stopwatch GenerateTimer; - CompressedBuffer CompressedBlock = - GenerateBlock(Content, Lookup, ChunksInBlock, Context.OutBlocks.BlockDescriptions[BlockIndex]); - if (m_Options.IsVerbose) - { - ZEN_INFO("Generated block {} ({}) containing {} chunks in {}", - Context.OutBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(CompressedBlock.GetCompressedSize()), - Context.OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(), - NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs())); - } - - Context.OutBlocks.BlockSizes[BlockIndex] = CompressedBlock.GetCompressedSize(); - { - CbObjectWriter Writer; - Writer.AddString("createdBy", "zen"); - Context.OutBlocks.BlockMetaDatas[BlockIndex] = Writer.Save(); - } - Context.GenerateBlocksStats.GeneratedBlockByteCount += Context.OutBlocks.BlockSizes[BlockIndex]; - Context.GenerateBlocksStats.GeneratedBlockCount++; - - Context.Lock.WithExclusiveLock([&]() { - Context.OutBlocks.BlockHashToBlockIndex.insert_or_assign(Context.OutBlocks.BlockDescriptions[BlockIndex].BlockHash, - BlockIndex); - }); - - { - std::span Segments = CompressedBlock.GetCompressed().GetSegments(); - ZEN_ASSERT(Segments.size() >= 2); - Context.OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); - } - - if (Context.GenerateBlocksStats.GeneratedBlockCount == Context.NewBlockCount) - { - Context.FilteredGeneratedBytesPerSecond.Stop(); - } - - if (Context.QueuedPendingBlocksForUpload.load() > 16) - { - std::span Segments = CompressedBlock.GetCompressed().GetSegments(); - ZEN_ASSERT(Segments.size() >= 2); - Context.OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); - } - else - { - if (!m_AbortFlag) - { - Context.QueuedPendingBlocksForUpload++; - Context.Work.ScheduleWork( - Context.UploadBlocksPool, - [this, &Context, BlockIndex, Payload = std::move(CompressedBlock)](std::atomic&) mutable { - UploadGeneratedBlock(Context, BlockIndex, std::move(Payload)); - }); - } - } - } - }); - } -} - -void -BuildsOperationUploadFolder::UploadGeneratedBlock(GenerateBuildBlocksContext& Context, size_t BlockIndex, CompressedBuffer Payload) -{ - auto _ = MakeGuard([&Context] { Context.QueuedPendingBlocksForUpload--; }); - if (m_AbortFlag) - { - return; - } - - if (Context.GenerateBlocksStats.GeneratedBlockCount == Context.NewBlockCount) - { - ZEN_TRACE_CPU("GenerateBuildBlocks_Save"); - - Context.FilteredUploadedBytesPerSecond.Stop(); - std::span Segments = Payload.GetCompressed().GetSegments(); - ZEN_ASSERT(Segments.size() >= 2); - Context.OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); - return; - } - - ZEN_TRACE_CPU("GenerateBuildBlocks_Upload"); - - Context.FilteredUploadedBytesPerSecond.Start(); - - const CbObject BlockMetaData = - BuildChunkBlockDescription(Context.OutBlocks.BlockDescriptions[BlockIndex], Context.OutBlocks.BlockMetaDatas[BlockIndex]); - - const IoHash& BlockHash = Context.OutBlocks.BlockDescriptions[BlockIndex].BlockHash; - const uint64_t CompressedBlockSize = Payload.GetCompressedSize(); - - if (m_Storage.CacheStorage && m_Options.PopulateCache) - { - m_Storage.CacheStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload.GetCompressed()); - } - - try - { - m_Storage.BuildStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, std::move(Payload).GetCompressed()); - } - catch (const std::exception&) - { - // Silence http errors due to abort - if (!m_AbortFlag) - { - throw; - } - } - - if (m_AbortFlag) - { - return; - } - - Context.UploadStats.BlocksBytes += CompressedBlockSize; - - if (m_Options.IsVerbose) - { - ZEN_INFO("Uploaded block {} ({}) containing {} chunks", - BlockHash, - NiceBytes(CompressedBlockSize), - Context.OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); - } - - if (m_Storage.CacheStorage && m_Options.PopulateCache) - { - m_Storage.CacheStorage->PutBlobMetadatas(m_BuildId, std::vector({BlockHash}), std::vector({BlockMetaData})); - } - - bool MetadataSucceeded = false; - try - { - MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData); - } - catch (const std::exception&) - { - // Silence http errors due to abort - if (!m_AbortFlag) - { - throw; - } - } - - if (m_AbortFlag) - { - return; - } - - if (MetadataSucceeded) - { - if (m_Options.IsVerbose) - { - ZEN_INFO("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize())); - } - - Context.OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; - Context.UploadStats.BlocksBytes += BlockMetaData.GetSize(); - } - - Context.UploadStats.BlockCount++; - if (Context.UploadStats.BlockCount == Context.NewBlockCount) - { - Context.FilteredUploadedBytesPerSecond.Stop(); - } -} - -std::vector -BuildsOperationUploadFolder::CalculateAbsoluteChunkOrders( - const std::span LocalChunkHashes, - const std::span LocalChunkOrder, - const tsl::robin_map& ChunkHashToLocalChunkIndex, - const std::span& LooseChunkIndexes, - const std::span& BlockDescriptions) -{ - ZEN_TRACE_CPU("CalculateAbsoluteChunkOrders"); - - std::vector TmpAbsoluteChunkHashes; - if (m_Options.DoExtraContentValidation) - { - TmpAbsoluteChunkHashes.reserve(LocalChunkHashes.size()); - } - std::vector LocalChunkIndexToAbsoluteChunkIndex; - LocalChunkIndexToAbsoluteChunkIndex.resize(LocalChunkHashes.size(), (uint32_t)-1); - std::uint32_t AbsoluteChunkCount = 0; - for (uint32_t ChunkIndex : LooseChunkIndexes) - { - LocalChunkIndexToAbsoluteChunkIndex[ChunkIndex] = AbsoluteChunkCount; - if (m_Options.DoExtraContentValidation) - { - TmpAbsoluteChunkHashes.push_back(LocalChunkHashes[ChunkIndex]); - } - AbsoluteChunkCount++; - } - for (const ChunkBlockDescription& Block : BlockDescriptions) - { - for (const IoHash& ChunkHash : Block.ChunkRawHashes) - { - if (auto It = ChunkHashToLocalChunkIndex.find(ChunkHash); It != ChunkHashToLocalChunkIndex.end()) - { - const uint32_t LocalChunkIndex = It->second; - ZEN_ASSERT_SLOW(LocalChunkHashes[LocalChunkIndex] == ChunkHash); - LocalChunkIndexToAbsoluteChunkIndex[LocalChunkIndex] = AbsoluteChunkCount; - } - if (m_Options.DoExtraContentValidation) - { - TmpAbsoluteChunkHashes.push_back(ChunkHash); - } - AbsoluteChunkCount++; - } - } - std::vector AbsoluteChunkOrder; - AbsoluteChunkOrder.reserve(LocalChunkHashes.size()); - for (const uint32_t LocalChunkIndex : LocalChunkOrder) - { - const uint32_t AbsoluteChunkIndex = LocalChunkIndexToAbsoluteChunkIndex[LocalChunkIndex]; - if (m_Options.DoExtraContentValidation) - { - ZEN_ASSERT(LocalChunkHashes[LocalChunkIndex] == TmpAbsoluteChunkHashes[AbsoluteChunkIndex]); - } - AbsoluteChunkOrder.push_back(AbsoluteChunkIndex); - } - if (m_Options.DoExtraContentValidation) - { - uint32_t OrderIndex = 0; - while (OrderIndex < LocalChunkOrder.size()) - { - const uint32_t LocalChunkIndex = LocalChunkOrder[OrderIndex]; - const IoHash& LocalChunkHash = LocalChunkHashes[LocalChunkIndex]; - const uint32_t AbsoluteChunkIndex = AbsoluteChunkOrder[OrderIndex]; - const IoHash& AbsoluteChunkHash = TmpAbsoluteChunkHashes[AbsoluteChunkIndex]; - ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash); - OrderIndex++; - } - } - return AbsoluteChunkOrder; -} - -CompositeBuffer -BuildsOperationUploadFolder::FetchChunk(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const IoHash& ChunkHash, - ReadFileCache& OpenFileCache) -{ - ZEN_TRACE_CPU("FetchChunk"); - auto It = Lookup.ChunkHashToChunkIndex.find(ChunkHash); - ZEN_ASSERT(It != Lookup.ChunkHashToChunkIndex.end()); - uint32_t ChunkIndex = It->second; - std::span ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex); - ZEN_ASSERT(!ChunkLocations.empty()); - CompositeBuffer Chunk = - OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, ChunkLocations[0].Offset, Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); - if (!Chunk) - { - throw std::runtime_error(fmt::format("Unable to read chunk at {}, size {} from '{}'", - ChunkLocations[0].Offset, - Content.ChunkedContent.ChunkRawSizes[ChunkIndex], - Content.Paths[Lookup.SequenceIndexFirstPathIndex[ChunkLocations[0].SequenceIndex]])); - } - ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == ChunkHash); - return Chunk; -}; - -CompressedBuffer -BuildsOperationUploadFolder::GenerateBlock(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const std::vector& ChunksInBlock, - ChunkBlockDescription& OutBlockDescription) -{ - ZEN_TRACE_CPU("GenerateBlock"); - ReadFileCache OpenFileCache(m_DiskStats.OpenReadCount, - m_DiskStats.CurrentOpenFileCount, - m_DiskStats.ReadCount, - m_DiskStats.ReadByteCount, - m_Path, - Content, - Lookup, - 4); - - std::vector> BlockContent; - BlockContent.reserve(ChunksInBlock.size()); - for (uint32_t ChunkIndex : ChunksInBlock) - { - BlockContent.emplace_back(std::make_pair( - Content.ChunkedContent.ChunkHashes[ChunkIndex], - [this, &Content, &Lookup, &OpenFileCache, ChunkIndex](const IoHash& ChunkHash) -> std::pair { - CompositeBuffer Chunk = FetchChunk(Content, Lookup, ChunkHash, OpenFileCache); - ZEN_ASSERT(Chunk); - uint64_t RawSize = Chunk.GetSize(); - - const bool ShouldCompressChunk = RawSize >= m_Options.MinimumSizeForCompressInBlock && - IsChunkCompressable(m_NonCompressableExtensionHashes, Lookup, ChunkIndex); - - const OodleCompressionLevel CompressionLevel = - ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None; - return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, CompressionLevel).GetCompressed()}; - })); - } - - return GenerateChunkBlock(std::move(BlockContent), OutBlockDescription); -}; - -CompressedBuffer -BuildsOperationUploadFolder::RebuildBlock(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - CompositeBuffer&& HeaderBuffer, - const std::vector& ChunksInBlock) -{ - ZEN_TRACE_CPU("RebuildBlock"); - ReadFileCache OpenFileCache(m_DiskStats.OpenReadCount, - m_DiskStats.CurrentOpenFileCount, - m_DiskStats.ReadCount, - m_DiskStats.ReadByteCount, - m_Path, - Content, - Lookup, - 4); - - std::vector ResultBuffers; - ResultBuffers.reserve(HeaderBuffer.GetSegments().size() + ChunksInBlock.size()); - ResultBuffers.insert(ResultBuffers.end(), HeaderBuffer.GetSegments().begin(), HeaderBuffer.GetSegments().end()); - for (uint32_t ChunkIndex : ChunksInBlock) - { - std::span ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex); - ZEN_ASSERT(!ChunkLocations.empty()); - CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, - ChunkLocations[0].Offset, - Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); - ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == Content.ChunkedContent.ChunkHashes[ChunkIndex]); - - const uint64_t RawSize = Chunk.GetSize(); - const bool ShouldCompressChunk = - RawSize >= m_Options.MinimumSizeForCompressInBlock && IsChunkCompressable(m_NonCompressableExtensionHashes, Lookup, ChunkIndex); - - const OodleCompressionLevel CompressionLevel = ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None; - - CompositeBuffer CompressedChunk = - CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, CompressionLevel).GetCompressed(); - ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); - } - return CompressedBuffer::FromCompressedNoValidate(CompositeBuffer(std::move(ResultBuffers))); -}; - -void -BuildsOperationUploadFolder::UploadBuildPart(ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - uint32_t PartIndex, - const UploadPart& Part, - uint32_t PartStepOffset, - uint32_t StepCount) -{ - Stopwatch UploadTimer; - - ChunkingStatistics ChunkingStats; - FindBlocksStatistics FindBlocksStats; - ReuseBlocksStatistics ReuseBlocksStats; - UploadStatistics UploadStats; - GenerateBlocksStatistics GenerateBlocksStats; - LooseChunksStatistics LooseChunksStats; - - m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::ChunkPartContent, StepCount); - - ChunkedFolderContent LocalContent = ScanPartContent(Part, ChunkController, ChunkCache, ChunkingStats); - if (m_AbortFlag) - { - return; - } - - const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent); - - if (PartIndex == 0) - { - ConsumePrepareBuildResult(); - } - - ZEN_ASSERT(m_PreferredMultipartChunkSize != 0); - ZEN_ASSERT(m_LargeAttachmentSize != 0); - - m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::CalculateDelta, StepCount); - - Stopwatch BlockArrangeTimer; - - std::vector LooseChunkIndexes; - std::vector NewBlockChunkIndexes; - std::vector ReuseBlockIndexes; - ClassifyChunksByBlockEligibility(LocalContent, - LooseChunkIndexes, - NewBlockChunkIndexes, - ReuseBlockIndexes, - LooseChunksStats, - FindBlocksStats, - ReuseBlocksStats); - - std::vector> NewBlockChunks; - ArrangeChunksIntoBlocks(LocalContent, LocalLookup, NewBlockChunkIndexes, NewBlockChunks); - - FindBlocksStats.NewBlocksCount += NewBlockChunks.size(); - for (uint32_t ChunkIndex : NewBlockChunkIndexes) - { - FindBlocksStats.NewBlocksChunkByteCount += LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - } - FindBlocksStats.NewBlocksChunkCount += NewBlockChunkIndexes.size(); - - const double AcceptedByteCountPercent = FindBlocksStats.PotentialChunkByteCount > 0 - ? (100.0 * ReuseBlocksStats.AcceptedRawByteCount / FindBlocksStats.PotentialChunkByteCount) - : 0.0; - - const double AcceptedReduntantByteCountPercent = - ReuseBlocksStats.AcceptedByteCount > 0 ? (100.0 * ReuseBlocksStats.AcceptedReduntantByteCount) / - (ReuseBlocksStats.AcceptedByteCount + ReuseBlocksStats.AcceptedReduntantByteCount) - : 0.0; - if (!m_Options.IsQuiet) - { - ZEN_INFO( - "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n" - " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n" - " Accepting {} ({}) redundant chunks ({:.1f}%)\n" - " Rejected {} ({}) chunks in {} blocks\n" - " Arranged {} ({}) chunks in {} new blocks\n" - " Keeping {} ({}) chunks as loose chunks\n" - " Discovery completed in {}", - FindBlocksStats.FoundBlockChunkCount, - FindBlocksStats.FoundBlockCount, - NiceBytes(FindBlocksStats.FoundBlockByteCount), - NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS), - - ReuseBlocksStats.AcceptedChunkCount, - NiceBytes(ReuseBlocksStats.AcceptedRawByteCount), - FindBlocksStats.AcceptedBlockCount, - AcceptedByteCountPercent, - - ReuseBlocksStats.AcceptedReduntantChunkCount, - NiceBytes(ReuseBlocksStats.AcceptedReduntantByteCount), - AcceptedReduntantByteCountPercent, - - ReuseBlocksStats.RejectedChunkCount, - NiceBytes(ReuseBlocksStats.RejectedByteCount), - ReuseBlocksStats.RejectedBlockCount, - - FindBlocksStats.NewBlocksChunkCount, - NiceBytes(FindBlocksStats.NewBlocksChunkByteCount), - FindBlocksStats.NewBlocksCount, - - LooseChunksStats.ChunkCount, - NiceBytes(LooseChunksStats.ChunkByteCount), - - NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs())); - } - - m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::GenerateBlocks, StepCount); - GeneratedBlocks NewBlocks; - - if (!NewBlockChunks.empty()) - { - Stopwatch GenerateBuildBlocksTimer; - auto __ = MakeGuard([&]() { - uint64_t BlockGenerateTimeUs = GenerateBuildBlocksTimer.GetElapsedTimeUs(); - if (!m_Options.IsQuiet) - { - ZEN_INFO("Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.", - GenerateBlocksStats.GeneratedBlockCount.load(), - NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount), - UploadStats.BlockCount.load(), - NiceBytes(UploadStats.BlocksBytes.load()), - NiceTimeSpanMs(BlockGenerateTimeUs / 1000), - NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, - GenerateBlocksStats.GeneratedBlockByteCount)), - NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8))); - } - }); - GenerateBuildBlocks(LocalContent, LocalLookup, NewBlockChunks, NewBlocks, GenerateBlocksStats, UploadStats); - } - - m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::BuildPartManifest, StepCount); - - BuiltPartManifest Manifest = - BuildPartManifestObject(LocalContent, LocalLookup, ChunkController, ReuseBlockIndexes, NewBlocks, LooseChunkIndexes); - - m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::UploadBuildPart, StepCount); - - Stopwatch PutBuildPartResultTimer; - std::pair> PutBuildPartResult = - m_Storage.BuildStorage->PutBuildPart(m_BuildId, Part.PartId, Part.PartName, Manifest.PartManifest); - if (!m_Options.IsQuiet) - { - ZEN_INFO("PutBuildPart took {}, payload size {}. {} attachments are needed.", - NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()), - NiceBytes(Manifest.PartManifest.GetSize()), - PutBuildPartResult.second.size()); - } - IoHash PartHash = PutBuildPartResult.first; - - m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::UploadAttachments, StepCount); - - std::vector UnknownChunks; - if (m_Options.IgnoreExistingBlocks) - { - if (m_Options.IsVerbose) - { - ZEN_INFO("PutBuildPart uploading all attachments, needs are: {}", FormatArray(PutBuildPartResult.second, "\n "sv)); - } - - std::vector ForceUploadChunkHashes; - ForceUploadChunkHashes.reserve(LooseChunkIndexes.size()); - - for (uint32_t ChunkIndex : LooseChunkIndexes) - { - ForceUploadChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); - } - - for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockHeaders.size(); BlockIndex++) - { - if (NewBlocks.BlockHeaders[BlockIndex]) - { - // Block was not uploaded during generation - ForceUploadChunkHashes.push_back(NewBlocks.BlockDescriptions[BlockIndex].BlockHash); - } - } - UploadAttachmentBatch(ForceUploadChunkHashes, - UnknownChunks, - LocalContent, - LocalLookup, - NewBlockChunks, - NewBlocks, - LooseChunkIndexes, - UploadStats, - LooseChunksStats); - } - else if (!PutBuildPartResult.second.empty()) - { - if (m_Options.IsVerbose) - { - ZEN_INFO("PutBuildPart needs attachments: {}", FormatArray(PutBuildPartResult.second, "\n "sv)); - } - UploadAttachmentBatch(PutBuildPartResult.second, - UnknownChunks, - LocalContent, - LocalLookup, - NewBlockChunks, - NewBlocks, - LooseChunkIndexes, - UploadStats, - LooseChunksStats); - } - - FinalizeBuildPartWithRetries(Part, - PartHash, - UnknownChunks, - LocalContent, - LocalLookup, - NewBlockChunks, - NewBlocks, - LooseChunkIndexes, - UploadStats, - LooseChunksStats); - - if (!NewBlocks.BlockDescriptions.empty() && !m_AbortFlag) - { - UploadMissingBlockMetadata(NewBlocks, UploadStats); - // The newly generated blocks are now known blocks so the next part upload can use those blocks as well - m_KnownBlocks.insert(m_KnownBlocks.end(), NewBlocks.BlockDescriptions.begin(), NewBlocks.BlockDescriptions.end()); - } - - m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::PutBuildPartStats, StepCount); - - m_Storage.BuildStorage->PutBuildPartStats( - m_BuildId, - Part.PartId, - {{"totalSize", double(Part.LocalFolderScanStats.FoundFileByteCount.load())}, - {"reusedRatio", AcceptedByteCountPercent / 100.0}, - {"reusedBlockCount", double(FindBlocksStats.AcceptedBlockCount)}, - {"reusedBlockByteCount", double(ReuseBlocksStats.AcceptedRawByteCount)}, - {"newBlockCount", double(FindBlocksStats.NewBlocksCount)}, - {"newBlockByteCount", double(FindBlocksStats.NewBlocksChunkByteCount)}, - {"uploadedCount", double(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load())}, - {"uploadedByteCount", double(UploadStats.BlocksBytes.load() + UploadStats.ChunksBytes.load())}, - {"uploadedBytesPerSec", - double(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.ChunksBytes + UploadStats.BlocksBytes))}, - {"elapsedTimeSec", double(UploadTimer.GetElapsedTimeMs() / 1000.0)}}); - - m_LocalFolderScanStats += Part.LocalFolderScanStats; - m_ChunkingStats += ChunkingStats; - m_FindBlocksStats += FindBlocksStats; - m_ReuseBlocksStats += ReuseBlocksStats; - m_UploadStats += UploadStats; - m_GenerateBlocksStats += GenerateBlocksStats; - m_LooseChunksStats += LooseChunksStats; -} - -ChunkedFolderContent -BuildsOperationUploadFolder::ScanPartContent(const UploadPart& Part, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - ChunkingStatistics& ChunkingStats) -{ - Stopwatch ScanTimer; - - std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Scan Folder"); - - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkedFolderContent LocalContent = ChunkFolderContent( - ChunkingStats, - m_IOWorkerPool, - m_Path, - Part.Content, - ChunkController, - ChunkCache, - m_Progress.GetProgressUpdateDelayMS(), - [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { - FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - ChunkingStats.FilesProcessed.load(), - Part.Content.Paths.size(), - NiceBytes(ChunkingStats.BytesHashed.load()), - NiceBytes(Part.TotalRawSize), - NiceNum(FilteredBytesHashed.GetCurrent()), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load())); - ProgressBar->UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = Part.TotalRawSize, - .RemainingCount = Part.TotalRawSize - ChunkingStats.BytesHashed.load(), - .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }, - m_AbortFlag, - m_PauseFlag); - FilteredBytesHashed.Stop(); - ProgressBar->Finish(); - if (m_AbortFlag) - { - return LocalContent; - } - - if (!m_Options.IsQuiet) - { - ZEN_INFO("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", - Part.Content.Paths.size(), - NiceBytes(Part.TotalRawSize), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load()), - m_Path, - NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()), - NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); - } - - return LocalContent; -} - -void -BuildsOperationUploadFolder::ConsumePrepareBuildResult() -{ - const PrepareBuildResult PrepBuildResult = m_PrepBuildResultFuture.get(); - - m_FindBlocksStats.FindBlockTimeMS = PrepBuildResult.ElapsedTimeMs; - m_FindBlocksStats.FoundBlockCount = PrepBuildResult.KnownBlocks.size(); - - if (!m_Options.IsQuiet) - { - ZEN_INFO("Build prepare took {}. {} took {}, payload size {}{}", - NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs), - m_CreateBuild ? "PutBuild" : "GetBuild", - NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs), - NiceBytes(PrepBuildResult.PayloadSize), - m_Options.IgnoreExistingBlocks ? "" - : fmt::format(". Found {} blocks in {}", - PrepBuildResult.KnownBlocks.size(), - NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs))); - } - - m_PreferredMultipartChunkSize = PrepBuildResult.PreferredMultipartChunkSize; - m_LargeAttachmentSize = m_Options.AllowMultiparts ? m_PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; - m_KnownBlocks = std::move(PrepBuildResult.KnownBlocks); -} - -void -BuildsOperationUploadFolder::ClassifyChunksByBlockEligibility(const ChunkedFolderContent& LocalContent, - std::vector& OutLooseChunkIndexes, - std::vector& OutNewBlockChunkIndexes, - std::vector& OutReuseBlockIndexes, - LooseChunksStatistics& LooseChunksStats, - FindBlocksStatistics& FindBlocksStats, - ReuseBlocksStatistics& ReuseBlocksStats) -{ - const bool EnableBlocks = true; - std::vector BlockChunkIndexes; - for (uint32_t ChunkIndex = 0; ChunkIndex < LocalContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++) - { - const uint64_t ChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - if (!EnableBlocks || ChunkRawSize == 0 || ChunkRawSize > m_Options.BlockParameters.MaxChunkEmbedSize) - { - OutLooseChunkIndexes.push_back(ChunkIndex); - LooseChunksStats.ChunkByteCount += ChunkRawSize; - } - else - { - BlockChunkIndexes.push_back(ChunkIndex); - FindBlocksStats.PotentialChunkByteCount += ChunkRawSize; - } - } - FindBlocksStats.PotentialChunkCount += BlockChunkIndexes.size(); - LooseChunksStats.ChunkCount = OutLooseChunkIndexes.size(); - - if (m_Options.IgnoreExistingBlocks) - { - if (!m_Options.IsQuiet) - { - ZEN_INFO("Ignoring any existing blocks in store"); - } - OutNewBlockChunkIndexes = std::move(BlockChunkIndexes); - return; - } - - OutReuseBlockIndexes = FindReuseBlocks(Log(), - m_Options.BlockReuseMinPercentLimit, - m_Options.IsVerbose, - ReuseBlocksStats, - m_KnownBlocks, - LocalContent.ChunkedContent.ChunkHashes, - BlockChunkIndexes, - OutNewBlockChunkIndexes); - FindBlocksStats.AcceptedBlockCount += OutReuseBlockIndexes.size(); - - for (const ChunkBlockDescription& Description : m_KnownBlocks) - { - for (uint32_t ChunkRawLength : Description.ChunkRawLengths) - { - FindBlocksStats.FoundBlockByteCount += ChunkRawLength; - } - FindBlocksStats.FoundBlockChunkCount += Description.ChunkRawHashes.size(); - } -} - -BuildsOperationUploadFolder::BuiltPartManifest -BuildsOperationUploadFolder::BuildPartManifestObject(const ChunkedFolderContent& LocalContent, - const ChunkedContentLookup& LocalLookup, - ChunkingController& ChunkController, - std::span ReuseBlockIndexes, - const GeneratedBlocks& NewBlocks, - std::span LooseChunkIndexes) -{ - BuiltPartManifest Result; - - CbObjectWriter PartManifestWriter; - Stopwatch ManifestGenerationTimer; - auto __ = MakeGuard([&]() { - if (!m_Options.IsQuiet) - { - ZEN_INFO("Generated build part manifest in {} ({})", - NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()), - NiceBytes(PartManifestWriter.GetSaveSize())); - } - }); - - PartManifestWriter.BeginObject("chunker"sv); - { - PartManifestWriter.AddString("name"sv, ChunkController.GetName()); - PartManifestWriter.AddObject("parameters"sv, ChunkController.GetParameters()); - } - PartManifestWriter.EndObject(); // chunker - - Result.AllChunkBlockHashes.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size()); - Result.AllChunkBlockDescriptions.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size()); - for (size_t ReuseBlockIndex : ReuseBlockIndexes) - { - Result.AllChunkBlockDescriptions.push_back(m_KnownBlocks[ReuseBlockIndex]); - Result.AllChunkBlockHashes.push_back(m_KnownBlocks[ReuseBlockIndex].BlockHash); - } - Result.AllChunkBlockDescriptions.insert(Result.AllChunkBlockDescriptions.end(), - NewBlocks.BlockDescriptions.begin(), - NewBlocks.BlockDescriptions.end()); - for (const ChunkBlockDescription& BlockDescription : NewBlocks.BlockDescriptions) - { - Result.AllChunkBlockHashes.push_back(BlockDescription.BlockHash); - } - - std::vector AbsoluteChunkHashes; - if (m_Options.DoExtraContentValidation) - { - tsl::robin_map ChunkHashToAbsoluteChunkIndex; - AbsoluteChunkHashes.reserve(LocalContent.ChunkedContent.ChunkHashes.size()); - for (uint32_t ChunkIndex : LooseChunkIndexes) - { - ChunkHashToAbsoluteChunkIndex.insert({LocalContent.ChunkedContent.ChunkHashes[ChunkIndex], AbsoluteChunkHashes.size()}); - AbsoluteChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); - } - for (const ChunkBlockDescription& Block : Result.AllChunkBlockDescriptions) - { - for (const IoHash& ChunkHash : Block.ChunkRawHashes) - { - ChunkHashToAbsoluteChunkIndex.insert({ChunkHash, AbsoluteChunkHashes.size()}); - AbsoluteChunkHashes.push_back(ChunkHash); - } - } - for (const IoHash& ChunkHash : LocalContent.ChunkedContent.ChunkHashes) - { - ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(ChunkHash)] == ChunkHash); - ZEN_ASSERT(LocalContent.ChunkedContent.ChunkHashes[LocalLookup.ChunkHashToChunkIndex.at(ChunkHash)] == ChunkHash); - } - for (const uint32_t ChunkIndex : LocalContent.ChunkedContent.ChunkOrders) - { - ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex])] == - LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); - ZEN_ASSERT(LocalLookup.ChunkHashToChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]) == ChunkIndex); - } - } - - std::vector AbsoluteChunkOrders = CalculateAbsoluteChunkOrders(LocalContent.ChunkedContent.ChunkHashes, - LocalContent.ChunkedContent.ChunkOrders, - LocalLookup.ChunkHashToChunkIndex, - LooseChunkIndexes, - Result.AllChunkBlockDescriptions); - - if (m_Options.DoExtraContentValidation) - { - for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); ChunkOrderIndex++) - { - uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndex]; - uint32_t AbsoluteChunkIndex = AbsoluteChunkOrders[ChunkOrderIndex]; - const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; - const IoHash& AbsoluteChunkHash = AbsoluteChunkHashes[AbsoluteChunkIndex]; - ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash); - } - } - - WriteBuildContentToCompactBinary(PartManifestWriter, - LocalContent.Platform, - LocalContent.Paths, - LocalContent.RawHashes, - LocalContent.RawSizes, - LocalContent.Attributes, - LocalContent.ChunkedContent.SequenceRawHashes, - LocalContent.ChunkedContent.ChunkCounts, - LocalContent.ChunkedContent.ChunkHashes, - LocalContent.ChunkedContent.ChunkRawSizes, - AbsoluteChunkOrders, - LooseChunkIndexes, - Result.AllChunkBlockHashes); - - if (m_Options.DoExtraContentValidation) - { - ChunkedFolderContent VerifyFolderContent; - - std::vector OutAbsoluteChunkOrders; - std::vector OutLooseChunkHashes; - std::vector OutLooseChunkRawSizes; - std::vector OutBlockRawHashes; - ReadBuildContentFromCompactBinary(PartManifestWriter.Save(), - VerifyFolderContent.Platform, - VerifyFolderContent.Paths, - VerifyFolderContent.RawHashes, - VerifyFolderContent.RawSizes, - VerifyFolderContent.Attributes, - VerifyFolderContent.ChunkedContent.SequenceRawHashes, - VerifyFolderContent.ChunkedContent.ChunkCounts, - OutAbsoluteChunkOrders, - OutLooseChunkHashes, - OutLooseChunkRawSizes, - OutBlockRawHashes); - ZEN_ASSERT(OutBlockRawHashes == Result.AllChunkBlockHashes); - - for (uint32_t OrderIndex = 0; OrderIndex < OutAbsoluteChunkOrders.size(); OrderIndex++) - { - uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex]; - const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; - - uint32_t VerifyChunkIndex = OutAbsoluteChunkOrders[OrderIndex]; - const IoHash VerifyChunkHash = AbsoluteChunkHashes[VerifyChunkIndex]; - - ZEN_ASSERT(LocalChunkHash == VerifyChunkHash); - } - - CalculateLocalChunkOrders(OutAbsoluteChunkOrders, - OutLooseChunkHashes, - OutLooseChunkRawSizes, - Result.AllChunkBlockDescriptions, - VerifyFolderContent.ChunkedContent.ChunkHashes, - VerifyFolderContent.ChunkedContent.ChunkRawSizes, - VerifyFolderContent.ChunkedContent.ChunkOrders, - m_Options.DoExtraContentValidation); - - ZEN_ASSERT(LocalContent.Paths == VerifyFolderContent.Paths); - ZEN_ASSERT(LocalContent.RawHashes == VerifyFolderContent.RawHashes); - ZEN_ASSERT(LocalContent.RawSizes == VerifyFolderContent.RawSizes); - ZEN_ASSERT(LocalContent.Attributes == VerifyFolderContent.Attributes); - ZEN_ASSERT(LocalContent.ChunkedContent.SequenceRawHashes == VerifyFolderContent.ChunkedContent.SequenceRawHashes); - ZEN_ASSERT(LocalContent.ChunkedContent.ChunkCounts == VerifyFolderContent.ChunkedContent.ChunkCounts); - - for (uint32_t OrderIndex = 0; OrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); OrderIndex++) - { - uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex]; - const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; - uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex]; - - uint32_t VerifyChunkIndex = VerifyFolderContent.ChunkedContent.ChunkOrders[OrderIndex]; - const IoHash VerifyChunkHash = VerifyFolderContent.ChunkedContent.ChunkHashes[VerifyChunkIndex]; - uint64_t VerifyChunkRawSize = VerifyFolderContent.ChunkedContent.ChunkRawSizes[VerifyChunkIndex]; - - ZEN_ASSERT(LocalChunkHash == VerifyChunkHash); - ZEN_ASSERT(LocalChunkRawSize == VerifyChunkRawSize); - } - } - - Result.PartManifest = PartManifestWriter.Save(); - return Result; -} - -void -BuildsOperationUploadFolder::UploadAttachmentBatch(std::span RawHashes, - std::vector& OutUnknownChunks, - const ChunkedFolderContent& LocalContent, - const ChunkedContentLookup& LocalLookup, - const std::vector>& NewBlockChunks, - GeneratedBlocks& NewBlocks, - std::span LooseChunkIndexes, - UploadStatistics& UploadStats, - LooseChunksStatistics& LooseChunksStats) -{ - if (m_AbortFlag) - { - return; - } - - UploadStatistics TempUploadStats; - LooseChunksStatistics TempLooseChunksStats; - - Stopwatch TempUploadTimer; - auto __ = MakeGuard([&]() { - if (!m_Options.IsQuiet) - { - uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs(); - ZEN_INFO( - "Uploaded {} ({}) blocks. " - "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. " - "Transferred {} ({}bits/s) in {}", - TempUploadStats.BlockCount.load(), - NiceBytes(TempUploadStats.BlocksBytes), - - TempLooseChunksStats.CompressedChunkCount.load(), - NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()), - NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, TempLooseChunksStats.ChunkByteCount)), - TempUploadStats.ChunkCount.load(), - NiceBytes(TempUploadStats.ChunksBytes), - - NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes), - NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)), - NiceTimeSpanMs(TempChunkUploadTimeUs / 1000)); - } - }); - UploadPartBlobs(LocalContent, - LocalLookup, - RawHashes, - NewBlockChunks, - NewBlocks, - LooseChunkIndexes, - m_LargeAttachmentSize, - TempUploadStats, - TempLooseChunksStats, - OutUnknownChunks); - UploadStats += TempUploadStats; - LooseChunksStats += TempLooseChunksStats; -} - -void -BuildsOperationUploadFolder::FinalizeBuildPartWithRetries(const UploadPart& Part, - const IoHash& PartHash, - std::vector& InOutUnknownChunks, - const ChunkedFolderContent& LocalContent, - const ChunkedContentLookup& LocalLookup, - const std::vector>& NewBlockChunks, - GeneratedBlocks& NewBlocks, - std::span LooseChunkIndexes, - UploadStatistics& UploadStats, - LooseChunksStatistics& LooseChunksStats) -{ - auto BuildUnkownChunksResponse = [](const std::vector& UnknownChunks, bool WillRetry) { - return fmt::format( - "The following build blobs was reported as needed for upload but was reported as existing at the start of the " - "operation.{}{}", - WillRetry ? " Treating this as a transient inconsistency issue and will attempt to retry finalization."sv : ""sv, - FormatArray(UnknownChunks, "\n "sv)); - }; - - if (!InOutUnknownChunks.empty()) - { - ZEN_WARN("{}", BuildUnkownChunksResponse(InOutUnknownChunks, /*WillRetry*/ true)); - } - - uint32_t FinalizeBuildPartRetryCount = 5; - while (!m_AbortFlag && (FinalizeBuildPartRetryCount--) > 0) - { - Stopwatch FinalizeBuildPartTimer; - std::vector Needs = m_Storage.BuildStorage->FinalizeBuildPart(m_BuildId, Part.PartId, PartHash); - if (!m_Options.IsQuiet) - { - ZEN_INFO("FinalizeBuildPart took {}. {} attachments are missing.", - NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()), - Needs.size()); - } - if (Needs.empty()) - { - break; - } - if (m_Options.IsVerbose) - { - ZEN_INFO("FinalizeBuildPart needs attachments: {}", FormatArray(Needs, "\n "sv)); - } - - std::vector RetryUnknownChunks; - UploadAttachmentBatch(Needs, - RetryUnknownChunks, - LocalContent, - LocalLookup, - NewBlockChunks, - NewBlocks, - LooseChunkIndexes, - UploadStats, - LooseChunksStats); - if (RetryUnknownChunks == InOutUnknownChunks) - { - if (FinalizeBuildPartRetryCount > 0) - { - // Back off a bit - Sleep(1000); - } - } - else - { - InOutUnknownChunks = RetryUnknownChunks; - ZEN_WARN("{}", BuildUnkownChunksResponse(InOutUnknownChunks, /*WillRetry*/ FinalizeBuildPartRetryCount != 0)); - } - } - - if (!InOutUnknownChunks.empty()) - { - throw std::runtime_error(BuildUnkownChunksResponse(InOutUnknownChunks, /*WillRetry*/ false)); - } -} - -void -BuildsOperationUploadFolder::UploadMissingBlockMetadata(GeneratedBlocks& NewBlocks, UploadStatistics& UploadStats) -{ - uint64_t UploadBlockMetadataCount = 0; - Stopwatch UploadBlockMetadataTimer; - - uint32_t FailedMetadataUploadCount = 1; - int32_t MetadataUploadRetryCount = 3; - while ((MetadataUploadRetryCount-- > 0) && (FailedMetadataUploadCount > 0)) - { - FailedMetadataUploadCount = 0; - for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++) - { - if (m_AbortFlag) - { - break; - } - const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; - if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex]) - { - const CbObject BlockMetaData = - BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); - if (m_Storage.CacheStorage && m_Options.PopulateCache) - { - m_Storage.CacheStorage->PutBlobMetadatas(m_BuildId, - std::vector({BlockHash}), - std::vector({BlockMetaData})); - } - bool MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData); - if (MetadataSucceeded) - { - UploadStats.BlocksBytes += BlockMetaData.GetSize(); - NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; - UploadBlockMetadataCount++; - } - else - { - FailedMetadataUploadCount++; - } - } - } - } - if (UploadBlockMetadataCount > 0) - { - uint64_t ElapsedUS = UploadBlockMetadataTimer.GetElapsedTimeUs(); - UploadStats.ElapsedWallTimeUS += ElapsedUS; - if (!m_Options.IsQuiet) - { - ZEN_INFO("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000)); - } - } -} - -void -BuildsOperationUploadFolder::UploadPartBlobs(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - std::span RawHashes, - const std::vector>& NewBlockChunks, - GeneratedBlocks& NewBlocks, - std::span LooseChunkIndexes, - const std::uint64_t LargeAttachmentSize, - UploadStatistics& TempUploadStats, - LooseChunksStatistics& TempLooseChunksStats, - std::vector& OutUnknownChunks) -{ - ZEN_TRACE_CPU("UploadPartBlobs"); - - UploadPartClassification Classification = - ClassifyUploadRawHashes(RawHashes, Content, Lookup, NewBlocks, LooseChunkIndexes, OutUnknownChunks); - - if (Classification.BlockIndexes.empty() && Classification.LooseChunkOrderIndexes.empty()) - { - return; - } - - std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Upload Blobs"); - - FilteredRate FilteredGenerateBlockBytesPerSecond; - FilteredRate FilteredCompressedBytesPerSecond; - FilteredRate FilteredUploadedBytesPerSecond; - - ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - - std::atomic UploadedBlockSize = 0; - std::atomic UploadedBlockCount = 0; - std::atomic UploadedRawChunkSize = 0; - std::atomic UploadedCompressedChunkSize = 0; - std::atomic UploadedChunkCount = 0; - std::atomic GeneratedBlockCount = 0; - std::atomic GeneratedBlockByteCount = 0; - std::atomic QueuedPendingInMemoryBlocksForUpload = 0; - - const size_t UploadBlockCount = Classification.BlockIndexes.size(); - const uint32_t UploadChunkCount = gsl::narrow(Classification.LooseChunkOrderIndexes.size()); - const uint64_t TotalRawSize = Classification.TotalLooseChunksSize + Classification.TotalBlocksSize; - - UploadPartBlobsContext Context{.Work = Work, - .ReadChunkPool = m_IOWorkerPool, - .UploadChunkPool = m_NetworkPool, - .FilteredGenerateBlockBytesPerSecond = FilteredGenerateBlockBytesPerSecond, - .FilteredCompressedBytesPerSecond = FilteredCompressedBytesPerSecond, - .FilteredUploadedBytesPerSecond = FilteredUploadedBytesPerSecond, - .UploadedBlockSize = UploadedBlockSize, - .UploadedBlockCount = UploadedBlockCount, - .UploadedRawChunkSize = UploadedRawChunkSize, - .UploadedCompressedChunkSize = UploadedCompressedChunkSize, - .UploadedChunkCount = UploadedChunkCount, - .GeneratedBlockCount = GeneratedBlockCount, - .GeneratedBlockByteCount = GeneratedBlockByteCount, - .QueuedPendingInMemoryBlocksForUpload = QueuedPendingInMemoryBlocksForUpload, - .UploadBlockCount = UploadBlockCount, - .UploadChunkCount = UploadChunkCount, - .LargeAttachmentSize = LargeAttachmentSize, - .NewBlocks = NewBlocks, - .Content = Content, - .Lookup = Lookup, - .NewBlockChunks = NewBlockChunks, - .LooseChunkIndexes = LooseChunkIndexes, - .TempUploadStats = TempUploadStats, - .TempLooseChunksStats = TempLooseChunksStats}; - - ScheduleBlockGenerationAndUpload(Context, Classification.BlockIndexes); - ScheduleLooseChunkCompressionAndUpload(Context, Classification.LooseChunkOrderIndexes); - - Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(PendingWork); - FilteredCompressedBytesPerSecond.Update(TempLooseChunksStats.CompressedChunkRawBytes.load()); - FilteredGenerateBlockBytesPerSecond.Update(GeneratedBlockByteCount.load()); - FilteredUploadedBytesPerSecond.Update(UploadedCompressedChunkSize.load() + UploadedBlockSize.load()); - uint64_t UploadedRawSize = UploadedRawChunkSize.load() + UploadedBlockSize.load(); - uint64_t UploadedCompressedSize = UploadedCompressedChunkSize.load() + UploadedBlockSize.load(); - - std::string Details = fmt::format( - "Compressed {}/{} ({}/{}{}) chunks. " - "Uploaded {}/{} ({}/{}) blobs " - "({}{})", - TempLooseChunksStats.CompressedChunkCount.load(), - Classification.LooseChunkOrderIndexes.size(), - NiceBytes(TempLooseChunksStats.CompressedChunkRawBytes), - NiceBytes(Classification.TotalLooseChunksSize), - (TempLooseChunksStats.CompressedChunkCount == Classification.LooseChunkOrderIndexes.size()) - ? "" - : fmt::format(" {}B/s", NiceNum(FilteredCompressedBytesPerSecond.GetCurrent())), - - UploadedBlockCount.load() + UploadedChunkCount.load(), - UploadBlockCount + UploadChunkCount, - NiceBytes(UploadedRawSize), - NiceBytes(TotalRawSize), - - NiceBytes(UploadedCompressedSize), - (UploadedBlockCount == UploadBlockCount && UploadedChunkCount == UploadChunkCount) - ? "" - : fmt::format(" {}bits/s", NiceNum(FilteredUploadedBytesPerSecond.GetCurrent()))); - - ProgressBar->UpdateState({.Task = "Uploading blobs ", - .Details = Details, - .TotalCount = gsl::narrow(TotalRawSize), - .RemainingCount = gsl::narrow(TotalRawSize - UploadedRawSize), - .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }); - - ZEN_ASSERT(m_AbortFlag || QueuedPendingInMemoryBlocksForUpload.load() == 0); - - ProgressBar->Finish(); - - TempUploadStats.ElapsedWallTimeUS += FilteredUploadedBytesPerSecond.GetElapsedTimeUS(); - TempLooseChunksStats.CompressChunksElapsedWallTimeUS += FilteredCompressedBytesPerSecond.GetElapsedTimeUS(); -} - -BuildsOperationUploadFolder::UploadPartClassification -BuildsOperationUploadFolder::ClassifyUploadRawHashes(std::span RawHashes, - const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const GeneratedBlocks& NewBlocks, - std::span LooseChunkIndexes, - std::vector& OutUnknownChunks) -{ - UploadPartClassification Result; - - tsl::robin_map ChunkIndexToLooseChunkOrderIndex; - ChunkIndexToLooseChunkOrderIndex.reserve(LooseChunkIndexes.size()); - for (uint32_t OrderIndex = 0; OrderIndex < LooseChunkIndexes.size(); OrderIndex++) - { - ChunkIndexToLooseChunkOrderIndex.insert_or_assign(LooseChunkIndexes[OrderIndex], OrderIndex); - } - - for (const IoHash& RawHash : RawHashes) - { - if (auto It = NewBlocks.BlockHashToBlockIndex.find(RawHash); It != NewBlocks.BlockHashToBlockIndex.end()) - { - Result.BlockIndexes.push_back(It->second); - Result.TotalBlocksSize += NewBlocks.BlockSizes[It->second]; - } - else if (auto ChunkIndexIt = Lookup.ChunkHashToChunkIndex.find(RawHash); ChunkIndexIt != Lookup.ChunkHashToChunkIndex.end()) - { - const uint32_t ChunkIndex = ChunkIndexIt->second; - if (auto LooseOrderIndexIt = ChunkIndexToLooseChunkOrderIndex.find(ChunkIndex); - LooseOrderIndexIt != ChunkIndexToLooseChunkOrderIndex.end()) - { - Result.LooseChunkOrderIndexes.push_back(LooseOrderIndexIt->second); - Result.TotalLooseChunksSize += Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; - } - } - else - { - OutUnknownChunks.push_back(RawHash); - } - } - return Result; -} - -void -BuildsOperationUploadFolder::ScheduleBlockGenerationAndUpload(UploadPartBlobsContext& Context, std::span BlockIndexes) -{ - for (const size_t BlockIndex : BlockIndexes) - { - const IoHash& BlockHash = Context.NewBlocks.BlockDescriptions[BlockIndex].BlockHash; - if (m_AbortFlag) - { - break; - } - Context.Work.ScheduleWork( - Context.ReadChunkPool, - [this, &Context, BlockHash = IoHash(BlockHash), BlockIndex, GenerateBlockCount = BlockIndexes.size()](std::atomic&) { - if (m_AbortFlag) - { - return; - } - ZEN_TRACE_CPU("UploadPartBlobs_GenerateBlock"); - - Context.FilteredGenerateBlockBytesPerSecond.Start(); - - Stopwatch GenerateTimer; - CompositeBuffer Payload; - if (Context.NewBlocks.BlockHeaders[BlockIndex]) - { - Payload = RebuildBlock(Context.Content, - Context.Lookup, - std::move(Context.NewBlocks.BlockHeaders[BlockIndex]), - Context.NewBlockChunks[BlockIndex]) - .GetCompressed(); - } - else - { - ChunkBlockDescription BlockDescription; - CompressedBuffer CompressedBlock = - GenerateBlock(Context.Content, Context.Lookup, Context.NewBlockChunks[BlockIndex], BlockDescription); - if (!CompressedBlock) - { - throw std::runtime_error(fmt::format("Failed generating block {}", BlockHash)); - } - ZEN_ASSERT(BlockDescription.BlockHash == BlockHash); - Payload = std::move(CompressedBlock).GetCompressed(); - } - - Context.GeneratedBlockByteCount += Context.NewBlocks.BlockSizes[BlockIndex]; - if (Context.GeneratedBlockCount.fetch_add(1) + 1 == GenerateBlockCount) - { - Context.FilteredGenerateBlockBytesPerSecond.Stop(); - } - if (m_Options.IsVerbose) - { - ZEN_INFO("{} block {} ({}) containing {} chunks in {}", - Context.NewBlocks.BlockHeaders[BlockIndex] ? "Regenerated" : "Generated", - Context.NewBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(Context.NewBlocks.BlockSizes[BlockIndex]), - Context.NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(), - NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs())); - } - if (!m_AbortFlag) - { - UploadBlockPayload(Context, BlockIndex, BlockHash, std::move(Payload)); - } - }); - } -} - -void -BuildsOperationUploadFolder::UploadBlockPayload(UploadPartBlobsContext& Context, - size_t BlockIndex, - const IoHash& BlockHash, - CompositeBuffer Payload) -{ - bool IsInMemoryBlock = true; - if (Context.QueuedPendingInMemoryBlocksForUpload.load() > 16) - { - ZEN_TRACE_CPU("AsyncUploadBlock_WriteTempBlock"); - std::filesystem::path TempFilePath = m_Options.TempDir / (BlockHash.ToHexString()); - Payload = CompositeBuffer(WriteToTempFile(std::move(Payload), TempFilePath)); - IsInMemoryBlock = false; - } - else - { - Context.QueuedPendingInMemoryBlocksForUpload++; - } - - Context.Work.ScheduleWork( - Context.UploadChunkPool, - [this, &Context, IsInMemoryBlock, BlockIndex, BlockHash = IoHash(BlockHash), Payload = CompositeBuffer(std::move(Payload))]( - std::atomic&) { - auto _ = MakeGuard([IsInMemoryBlock, &Context] { - if (IsInMemoryBlock) - { - Context.QueuedPendingInMemoryBlocksForUpload--; - } - }); - if (m_AbortFlag) - { - return; - } - ZEN_TRACE_CPU("AsyncUploadBlock"); - - const uint64_t PayloadSize = Payload.GetSize(); - - Context.FilteredUploadedBytesPerSecond.Start(); - const CbObject BlockMetaData = - BuildChunkBlockDescription(Context.NewBlocks.BlockDescriptions[BlockIndex], Context.NewBlocks.BlockMetaDatas[BlockIndex]); - - if (m_Storage.CacheStorage && m_Options.PopulateCache) - { - m_Storage.CacheStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); - } - - try - { - m_Storage.BuildStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); - } - catch (const std::exception&) - { - // Silence http errors due to abort - if (!m_AbortFlag) - { - throw; - } - } - - if (m_AbortFlag) - { - return; - } - if (m_Options.IsVerbose) - { - ZEN_INFO("Uploaded block {} ({}) containing {} chunks", - BlockHash, - NiceBytes(PayloadSize), - Context.NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); - } - Context.UploadedBlockSize += PayloadSize; - Context.TempUploadStats.BlocksBytes += PayloadSize; - - if (m_Storage.CacheStorage && m_Options.PopulateCache) - { - m_Storage.CacheStorage->PutBlobMetadatas(m_BuildId, - std::vector({BlockHash}), - std::vector({BlockMetaData})); - } - - bool MetadataSucceeded = false; - try - { - MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData); - } - catch (const std::exception&) - { - // Silence http errors due to abort - if (!m_AbortFlag) - { - throw; - } - } - if (m_AbortFlag) - { - return; - } - if (MetadataSucceeded) - { - if (m_Options.IsVerbose) - { - ZEN_INFO("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize())); - } - Context.NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; - Context.TempUploadStats.BlocksBytes += BlockMetaData.GetSize(); - } - - Context.TempUploadStats.BlockCount++; - - if (Context.UploadedBlockCount.fetch_add(1) + 1 == Context.UploadBlockCount && - Context.UploadedChunkCount == Context.UploadChunkCount) - { - Context.FilteredUploadedBytesPerSecond.Stop(); - } - }); -} - -void -BuildsOperationUploadFolder::ScheduleLooseChunkCompressionAndUpload(UploadPartBlobsContext& Context, - std::span LooseChunkOrderIndexes) -{ - for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes) - { - const uint32_t ChunkIndex = Context.LooseChunkIndexes[LooseChunkOrderIndex]; - Context.Work.ScheduleWork(Context.ReadChunkPool, - [this, &Context, LooseChunkOrderCount = LooseChunkOrderIndexes.size(), ChunkIndex](std::atomic&) { - if (m_AbortFlag) - { - return; - } - ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk"); - - Context.FilteredCompressedBytesPerSecond.Start(); - Stopwatch CompressTimer; - CompositeBuffer Payload = - CompressChunk(Context.Content, Context.Lookup, ChunkIndex, Context.TempLooseChunksStats); - if (m_Options.IsVerbose) - { - ZEN_INFO("Compressed chunk {} ({} -> {}) in {}", - Context.Content.ChunkedContent.ChunkHashes[ChunkIndex], - NiceBytes(Context.Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), - NiceBytes(Payload.GetSize()), - NiceTimeSpanMs(CompressTimer.GetElapsedTimeMs())); - } - const uint64_t ChunkRawSize = Context.Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; - Context.TempUploadStats.ReadFromDiskBytes += ChunkRawSize; - if (Context.TempLooseChunksStats.CompressedChunkCount == LooseChunkOrderCount) - { - Context.FilteredCompressedBytesPerSecond.Stop(); - } - if (!m_AbortFlag) - { - UploadLooseChunkPayload(Context, - Context.Content.ChunkedContent.ChunkHashes[ChunkIndex], - ChunkRawSize, - std::move(Payload)); - } - }); - } -} - -void -BuildsOperationUploadFolder::UploadLooseChunkPayload(UploadPartBlobsContext& Context, - const IoHash& RawHash, - uint64_t RawSize, - CompositeBuffer Payload) -{ - Context.Work.ScheduleWork( - Context.UploadChunkPool, - [this, &Context, RawHash = IoHash(RawHash), RawSize, Payload = CompositeBuffer(std::move(Payload))](std::atomic&) mutable { - if (m_AbortFlag) - { - return; - } - ZEN_TRACE_CPU("AsyncUploadLooseChunk"); - - const uint64_t PayloadSize = Payload.GetSize(); - - if (m_Storage.CacheStorage && m_Options.PopulateCache) - { - m_Storage.CacheStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); - } - - if (PayloadSize >= Context.LargeAttachmentSize) - { - ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart"); - Context.TempUploadStats.MultipartAttachmentCount++; - try - { - std::vector> MultipartWork = m_Storage.BuildStorage->PutLargeBuildBlob( - m_BuildId, - RawHash, - ZenContentType::kCompressedBinary, - PayloadSize, - [Payload = std::move(Payload), &Context](uint64_t Offset, uint64_t Size) -> IoBuffer { - Context.FilteredUploadedBytesPerSecond.Start(); - - IoBuffer PartPayload = Payload.Mid(Offset, Size).Flatten().AsIoBuffer(); - PartPayload.SetContentType(ZenContentType::kBinary); - return PartPayload; - }, - [&Context, RawSize](uint64_t SentBytes, bool IsComplete) { - Context.TempUploadStats.ChunksBytes += SentBytes; - Context.UploadedCompressedChunkSize += SentBytes; - if (IsComplete) - { - Context.TempUploadStats.ChunkCount++; - if (Context.UploadedChunkCount.fetch_add(1) + 1 == Context.UploadChunkCount && - Context.UploadedBlockCount == Context.UploadBlockCount) - { - Context.FilteredUploadedBytesPerSecond.Stop(); - } - Context.UploadedRawChunkSize += RawSize; - } - }); - for (auto& WorkPart : MultipartWork) - { - Context.Work.ScheduleWork(Context.UploadChunkPool, [Work = std::move(WorkPart)](std::atomic& AbortFlag) { - ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart_Work"); - if (!AbortFlag) - { - Work(); - } - }); - } - if (m_Options.IsVerbose) - { - ZEN_INFO("Uploaded multipart chunk {} ({})", RawHash, NiceBytes(PayloadSize)); - } - } - catch (const std::exception&) - { - // Silence http errors due to abort - if (!m_AbortFlag) - { - throw; - } - } - return; - } - - ZEN_TRACE_CPU("AsyncUploadLooseChunk_Singlepart"); - try - { - m_Storage.BuildStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); - } - catch (const std::exception&) - { - // Silence http errors due to abort - if (!m_AbortFlag) - { - throw; - } - } - if (m_AbortFlag) - { - return; - } - if (m_Options.IsVerbose) - { - ZEN_INFO("Uploaded chunk {} ({})", RawHash, NiceBytes(PayloadSize)); - } - Context.TempUploadStats.ChunksBytes += Payload.GetSize(); - Context.TempUploadStats.ChunkCount++; - Context.UploadedCompressedChunkSize += Payload.GetSize(); - Context.UploadedRawChunkSize += RawSize; - if (Context.UploadedChunkCount.fetch_add(1) + 1 == Context.UploadChunkCount && - Context.UploadedBlockCount == Context.UploadBlockCount) - { - Context.FilteredUploadedBytesPerSecond.Stop(); - } - }); -} - -CompositeBuffer -BuildsOperationUploadFolder::CompressChunk(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - uint32_t ChunkIndex, - LooseChunksStatistics& TempLooseChunksStats) -{ - ZEN_TRACE_CPU("CompressChunk"); - ZEN_ASSERT(!m_Options.TempDir.empty()); - const IoHash& ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; - const uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; - - const ChunkedContentLookup::ChunkSequenceLocation& Source = GetChunkSequenceLocations(Lookup, ChunkIndex)[0]; - const std::uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[Source.SequenceIndex]; - IoBuffer RawSource = IoBufferBuilder::MakeFromFile((m_Path / Content.Paths[PathIndex]).make_preferred(), Source.Offset, ChunkSize); - if (!RawSource) - { - throw std::runtime_error(fmt::format("Failed fetching chunk {}", ChunkHash)); - } - if (RawSource.GetSize() != ChunkSize) - { - throw std::runtime_error(fmt::format("Fetched chunk {} has invalid size", ChunkHash)); - } - - const bool ShouldCompressChunk = IsChunkCompressable(m_NonCompressableExtensionHashes, Lookup, ChunkIndex); - const OodleCompressionLevel CompressionLevel = ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None; - - if (ShouldCompressChunk) - { - std::filesystem::path TempFilePath = m_Options.TempDir / ChunkHash.ToHexString(); - - BasicFile CompressedFile; - std::error_code Ec; - CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncateDelete, Ec); - if (Ec) - { - throw std::runtime_error(fmt::format("Failed creating temporary file for compressing blob {}, reason: ({}) {}", - ChunkHash, - Ec.value(), - Ec.message())); - } - - uint64_t StreamRawBytes = 0; - uint64_t StreamCompressedBytes = 0; - - bool CouldCompress = CompressedBuffer::CompressToStream( - CompositeBuffer(SharedBuffer(RawSource)), - [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { - ZEN_UNUSED(SourceOffset); - TempLooseChunksStats.CompressedChunkRawBytes += SourceSize; - CompressedFile.Write(RangeBuffer, Offset); - TempLooseChunksStats.CompressedChunkBytes += RangeBuffer.GetSize(); - StreamRawBytes += SourceSize; - StreamCompressedBytes += RangeBuffer.GetSize(); - }, - OodleCompressor::Mermaid, - CompressionLevel); - if (CouldCompress) - { - uint64_t CompressedSize = CompressedFile.FileSize(); - void* FileHandle = CompressedFile.Detach(); - IoBuffer TempPayload = IoBuffer(IoBuffer::File, - FileHandle, - 0, - CompressedSize, - /*IsWholeFile*/ true); - ZEN_ASSERT(TempPayload); - TempPayload.SetDeleteOnClose(true); - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(TempPayload), RawHash, RawSize); - ZEN_ASSERT(Compressed); - ZEN_ASSERT(RawHash == ChunkHash); - ZEN_ASSERT(RawSize == ChunkSize); - - TempLooseChunksStats.CompressedChunkCount++; - - return Compressed.GetCompressed(); - } - else - { - TempLooseChunksStats.CompressedChunkRawBytes -= StreamRawBytes; - TempLooseChunksStats.CompressedChunkBytes -= StreamCompressedBytes; - } - CompressedFile.Close(); - RemoveFile(TempFilePath, Ec); - ZEN_UNUSED(Ec); - } - - CompressedBuffer CompressedBlob = - CompressedBuffer::Compress(SharedBuffer(std::move(RawSource)), OodleCompressor::Mermaid, CompressionLevel); - if (!CompressedBlob) - { - throw std::runtime_error(fmt::format("Failed to compress large blob {}", ChunkHash)); - } - ZEN_ASSERT_SLOW(CompressedBlob.DecodeRawHash() == ChunkHash); - ZEN_ASSERT_SLOW(CompressedBlob.DecodeRawSize() == ChunkSize); - - TempLooseChunksStats.CompressedChunkRawBytes += ChunkSize; - TempLooseChunksStats.CompressedChunkBytes += CompressedBlob.GetCompressedSize(); - - // If we use none-compression, the compressed blob references the data and has 64 kb in memory so we don't need to write it to disk - if (ShouldCompressChunk) - { - std::filesystem::path TempFilePath = m_Options.TempDir / (ChunkHash.ToHexString()); - IoBuffer TempPayload = WriteToTempFile(std::move(CompressedBlob).GetCompressed(), TempFilePath); - CompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(TempPayload)); - } - - TempLooseChunksStats.CompressedChunkCount++; - return std::move(CompressedBlob).GetCompressed(); -} - -BuildsOperationValidateBuildPart::BuildsOperationValidateBuildPart(LoggerRef Log, - ProgressBase& Progress, - BuildStorageBase& Storage, - std::atomic& AbortFlag, - std::atomic& PauseFlag, - WorkerThreadPool& IOWorkerPool, - WorkerThreadPool& NetworkPool, - const Oid& BuildId, - const Oid& BuildPartId, - const std::string_view BuildPartName, - const Options& Options) - -: m_Log(Log) -, m_Progress(Progress) -, m_Storage(Storage) -, m_AbortFlag(AbortFlag) -, m_PauseFlag(PauseFlag) -, m_IOWorkerPool(IOWorkerPool) -, m_NetworkPool(NetworkPool) -, m_BuildId(BuildId) -, m_BuildPartId(BuildPartId) -, m_BuildPartName(BuildPartName) -, m_Options(Options) -{ -} - -void -BuildsOperationValidateBuildPart::Execute() -{ - ZEN_TRACE_CPU("ValidateBuildPart"); - try - { - auto EndProgress = - MakeGuard([&]() { m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::StepCount, (uint32_t)TaskSteps::StepCount); }); - - Stopwatch Timer; - auto _ = MakeGuard([&]() { - if (!m_Options.IsQuiet) - { - ZEN_INFO("Validated build part {}/{} ('{}') in {}", - m_BuildId, - m_BuildPartId, - m_BuildPartName, - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - } - }); - - m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::FetchBuild, (uint32_t)TaskSteps::StepCount); - - ResolvedBuildPart Resolved = ResolveBuildPart(); - - ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - - const std::filesystem::path TempFolder = ".zen-tmp"; - - CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, TempFolder); - CreateDirectories(TempFolder); - auto __ = MakeGuard([this, TempFolder]() { CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, TempFolder); }); - - m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::ValidateBlobs, (uint32_t)TaskSteps::StepCount); - - std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Validate Blobs"); - - const uint64_t AttachmentsToVerifyCount = Resolved.ChunkAttachments.size() + Resolved.BlockAttachments.size(); - FilteredRate FilteredDownloadedBytesPerSecond; - FilteredRate FilteredVerifiedBytesPerSecond; - - ValidateBlobsContext Context{.Work = Work, - .AttachmentsToVerifyCount = AttachmentsToVerifyCount, - .FilteredDownloadedBytesPerSecond = FilteredDownloadedBytesPerSecond, - .FilteredVerifiedBytesPerSecond = FilteredVerifiedBytesPerSecond}; - - ScheduleChunkAttachmentValidation(Context, Resolved.ChunkAttachments, TempFolder, Resolved.PreferredMultipartChunkSize); - ScheduleBlockAttachmentValidation(Context, Resolved.BlockAttachments); - - Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(PendingWork); - - const uint64_t DownloadedAttachmentCount = m_DownloadStats.DownloadedChunkCount + m_DownloadStats.DownloadedBlockCount; - const uint64_t DownloadedByteCount = m_DownloadStats.DownloadedChunkByteCount + m_DownloadStats.DownloadedBlockByteCount; - - FilteredDownloadedBytesPerSecond.Update(DownloadedByteCount); - FilteredVerifiedBytesPerSecond.Update(m_ValidateStats.VerifiedByteCount); - - std::string Details = fmt::format("Downloaded {}/{} ({}, {}bits/s). Verified {}/{} ({}, {}B/s)", - DownloadedAttachmentCount, - AttachmentsToVerifyCount, - NiceBytes(DownloadedByteCount), - NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8), - m_ValidateStats.VerifiedAttachmentCount.load(), - AttachmentsToVerifyCount, - NiceBytes(m_ValidateStats.VerifiedByteCount.load()), - NiceNum(FilteredVerifiedBytesPerSecond.GetCurrent())); - - ProgressBar->UpdateState( - {.Task = "Validating blobs ", - .Details = Details, - .TotalCount = gsl::narrow(AttachmentsToVerifyCount * 2), - .RemainingCount = gsl::narrow(AttachmentsToVerifyCount * 2 - - (DownloadedAttachmentCount + m_ValidateStats.VerifiedAttachmentCount.load())), - .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }); - - ProgressBar->Finish(); - m_ValidateStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); - - m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::Cleanup, (uint32_t)TaskSteps::StepCount); - } - catch (const std::exception&) - { - m_AbortFlag = true; - throw; - } -} - -BuildsOperationValidateBuildPart::ResolvedBuildPart -BuildsOperationValidateBuildPart::ResolveBuildPart() -{ - ResolvedBuildPart Result; - Result.PreferredMultipartChunkSize = 32u * 1024u * 1024u; - - CbObject Build = m_Storage.GetBuild(m_BuildId); - if (!m_BuildPartName.empty()) - { - m_BuildPartId = Build["parts"sv].AsObjectView()[m_BuildPartName].AsObjectId(); - if (m_BuildPartId == Oid::Zero) - { - throw std::runtime_error(fmt::format("Build {} does not have a part named '{}'", m_BuildId, m_BuildPartName)); - } - } - m_ValidateStats.BuildBlobSize = Build.GetSize(); - if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) - { - Result.PreferredMultipartChunkSize = ChunkSize; - } - - m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::FetchBuildPart, (uint32_t)TaskSteps::StepCount); - - CbObject BuildPart = m_Storage.GetBuildPart(m_BuildId, m_BuildPartId); - m_ValidateStats.BuildPartSize = BuildPart.GetSize(); - if (!m_Options.IsQuiet) - { - ZEN_INFO("Validating build part {}/{} ({})", m_BuildId, m_BuildPartId, NiceBytes(BuildPart.GetSize())); - } - if (const CbObjectView ChunkAttachmentsView = BuildPart["chunkAttachments"sv].AsObjectView()) - { - for (CbFieldView LooseFileView : ChunkAttachmentsView["rawHashes"sv]) - { - Result.ChunkAttachments.push_back(LooseFileView.AsBinaryAttachment()); - } - } - m_ValidateStats.ChunkAttachmentCount = Result.ChunkAttachments.size(); - if (const CbObjectView BlockAttachmentsView = BuildPart["blockAttachments"sv].AsObjectView()) - { - for (CbFieldView BlocksView : BlockAttachmentsView["rawHashes"sv]) - { - Result.BlockAttachments.push_back(BlocksView.AsBinaryAttachment()); - } - } - m_ValidateStats.BlockAttachmentCount = Result.BlockAttachments.size(); - - std::vector VerifyBlockDescriptions = - ParseChunkBlockDescriptionList(m_Storage.GetBlockMetadatas(m_BuildId, Result.BlockAttachments)); - if (VerifyBlockDescriptions.size() != Result.BlockAttachments.size()) - { - throw std::runtime_error(fmt::format("Uploaded blocks metadata could not all be found, {} blocks metadata is missing", - Result.BlockAttachments.size() - VerifyBlockDescriptions.size())); - } - - return Result; -} - -void -BuildsOperationValidateBuildPart::ScheduleChunkAttachmentValidation(ValidateBlobsContext& Context, - std::span ChunkAttachments, - const std::filesystem::path& TempFolder, - uint64_t PreferredMultipartChunkSize) -{ - for (const IoHash& ChunkAttachment : ChunkAttachments) - { - Context.Work.ScheduleWork( - m_NetworkPool, - [this, &Context, &TempFolder, PreferredMultipartChunkSize, ChunkAttachment = IoHash(ChunkAttachment)](std::atomic&) { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("ValidateBuildPart_GetChunk"); - - Context.FilteredDownloadedBytesPerSecond.Start(); - DownloadLargeBlob( - m_Storage, - TempFolder, - m_BuildId, - ChunkAttachment, - PreferredMultipartChunkSize, - Context.Work, - m_NetworkPool, - m_DownloadStats.DownloadedChunkByteCount, - m_DownloadStats.MultipartAttachmentCount, - [this, &Context, ChunkHash = IoHash(ChunkAttachment)](IoBuffer&& Payload) { - m_DownloadStats.DownloadedChunkCount++; - Payload.SetContentType(ZenContentType::kCompressedBinary); - if (!m_AbortFlag) - { - Context.Work.ScheduleWork( - m_IOWorkerPool, - [this, &Context, Payload = IoBuffer(std::move(Payload)), ChunkHash](std::atomic&) mutable { - if (!m_AbortFlag) - { - ValidateDownloadedChunk(Context, ChunkHash, std::move(Payload)); - } - }); - } - }); - } - }); - } -} - -void -BuildsOperationValidateBuildPart::ScheduleBlockAttachmentValidation(ValidateBlobsContext& Context, std::span BlockAttachments) -{ - for (const IoHash& BlockAttachment : BlockAttachments) - { - Context.Work.ScheduleWork(m_NetworkPool, [this, &Context, BlockAttachment = IoHash(BlockAttachment)](std::atomic&) { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("ValidateBuildPart_GetBlock"); - - Context.FilteredDownloadedBytesPerSecond.Start(); - IoBuffer Payload = m_Storage.GetBuildBlob(m_BuildId, BlockAttachment); - m_DownloadStats.DownloadedBlockCount++; - m_DownloadStats.DownloadedBlockByteCount += Payload.GetSize(); - if (m_DownloadStats.DownloadedChunkCount + m_DownloadStats.DownloadedBlockCount == Context.AttachmentsToVerifyCount) - { - Context.FilteredDownloadedBytesPerSecond.Stop(); - } - if (!Payload) - { - throw std::runtime_error(fmt::format("Block attachment {} could not be found", BlockAttachment)); - } - if (!m_AbortFlag) - { - Context.Work.ScheduleWork(m_IOWorkerPool, - [this, &Context, Payload = std::move(Payload), BlockAttachment](std::atomic&) mutable { - if (!m_AbortFlag) - { - ValidateDownloadedBlock(Context, BlockAttachment, std::move(Payload)); - } - }); - } - } - }); - } -} - -void -BuildsOperationValidateBuildPart::ValidateDownloadedChunk(ValidateBlobsContext& Context, const IoHash& ChunkHash, IoBuffer Payload) -{ - ZEN_TRACE_CPU("ValidateBuildPart_Validate"); - - if (m_DownloadStats.DownloadedChunkCount + m_DownloadStats.DownloadedBlockCount == Context.AttachmentsToVerifyCount) - { - Context.FilteredDownloadedBytesPerSecond.Stop(); - } - - Context.FilteredVerifiedBytesPerSecond.Start(); - - uint64_t CompressedSize; - uint64_t DecompressedSize; - ValidateBlob(m_AbortFlag, std::move(Payload), ChunkHash, CompressedSize, DecompressedSize); - m_ValidateStats.VerifiedAttachmentCount++; - m_ValidateStats.VerifiedByteCount += DecompressedSize; - if (m_ValidateStats.VerifiedAttachmentCount.load() == Context.AttachmentsToVerifyCount) - { - Context.FilteredVerifiedBytesPerSecond.Stop(); - } -} - -void -BuildsOperationValidateBuildPart::ValidateDownloadedBlock(ValidateBlobsContext& Context, const IoHash& BlockAttachment, IoBuffer Payload) -{ - ZEN_TRACE_CPU("ValidateBuildPart_ValidateBlock"); - - Context.FilteredVerifiedBytesPerSecond.Start(); - - uint64_t CompressedSize; - uint64_t DecompressedSize; - ValidateChunkBlock(std::move(Payload), BlockAttachment, CompressedSize, DecompressedSize); - m_ValidateStats.VerifiedAttachmentCount++; - m_ValidateStats.VerifiedByteCount += DecompressedSize; - if (m_ValidateStats.VerifiedAttachmentCount.load() == Context.AttachmentsToVerifyCount) - { - Context.FilteredVerifiedBytesPerSecond.Stop(); - } -} - -BuildsOperationPrimeCache::BuildsOperationPrimeCache(LoggerRef Log, - ProgressBase& Progress, - StorageInstance& Storage, - std::atomic& AbortFlag, - std::atomic& PauseFlag, - WorkerThreadPool& NetworkPool, - const Oid& BuildId, - std::span BuildPartIds, - const Options& Options, - BuildStorageCache::Statistics& StorageCacheStats) -: m_Log(Log) -, m_Progress(Progress) -, m_Storage(Storage) -, m_AbortFlag(AbortFlag) -, m_PauseFlag(PauseFlag) -, m_NetworkPool(NetworkPool) -, m_BuildId(BuildId) -, m_BuildPartIds(BuildPartIds.begin(), BuildPartIds.end()) -, m_Options(Options) -, m_StorageCacheStats(StorageCacheStats) -{ - m_TempPath = m_Options.ZenFolderPath / "tmp"; - CreateDirectories(m_TempPath); -} - -void -BuildsOperationPrimeCache::Execute() -{ - ZEN_TRACE_CPU("BuildsOperationPrimeCache::Execute"); - - Stopwatch PrimeTimer; - - tsl::robin_map LooseChunkRawSizes; - tsl::robin_set BuildBlobs; - CollectReferencedBlobs(BuildBlobs, LooseChunkRawSizes); - - if (!m_Options.IsQuiet) - { - ZEN_INFO("Found {} referenced blobs", BuildBlobs.size()); - } - - if (BuildBlobs.empty()) - { - return; - } - - std::vector BlobsToDownload = FilterAlreadyCachedBlobs(BuildBlobs); - - if (BlobsToDownload.empty()) - { - return; - } - - std::atomic MultipartAttachmentCount; - std::atomic CompletedDownloadCount; - FilteredRate FilteredDownloadedBytesPerSecond; - - ScheduleBlobDownloads(BlobsToDownload, - LooseChunkRawSizes, - MultipartAttachmentCount, - CompletedDownloadCount, - FilteredDownloadedBytesPerSecond); - - if (m_AbortFlag) - { - return; - } - - if (m_Storage.CacheStorage) - { - m_Storage.CacheStorage->Flush(m_Progress.GetProgressUpdateDelayMS(), [this](intptr_t Remaining) -> bool { - ZEN_UNUSED(Remaining); - if (!m_Options.IsQuiet) - { - ZEN_INFO("Waiting for {} blobs to finish upload to '{}'", Remaining, m_Storage.CacheHost.Name); - } - return !m_AbortFlag; - }); - } - - if (!m_Options.IsQuiet) - { - uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() + m_DownloadStats.DownloadedBlockByteCount.load(); - ZEN_INFO("Downloaded {} ({}bits/s) in {}. {} as multipart. Completed in {}", - NiceBytes(DownloadedBytes), - NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)), - NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000), - MultipartAttachmentCount.load(), - NiceTimeSpanMs(PrimeTimer.GetElapsedTimeMs())); - } -} - -void -BuildsOperationPrimeCache::CollectReferencedBlobs(tsl::robin_set& OutBuildBlobs, - tsl::robin_map& OutLooseChunkRawSizes) -{ - for (const Oid& BuildPartId : m_BuildPartIds) - { - CbObject BuildPart = m_Storage.BuildStorage->GetBuildPart(m_BuildId, BuildPartId); - - CbObjectView BlockAttachmentsView = BuildPart["blockAttachments"sv].AsObjectView(); - std::vector BlockAttachments = compactbinary_helpers::ReadBinaryAttachmentArray("rawHashes"sv, BlockAttachmentsView); - - CbObjectView ChunkAttachmentsView = BuildPart["chunkAttachments"sv].AsObjectView(); - std::vector ChunkAttachments = compactbinary_helpers::ReadBinaryAttachmentArray("rawHashes"sv, ChunkAttachmentsView); - std::vector ChunkRawSizes = compactbinary_helpers::ReadArray("chunkRawSizes"sv, ChunkAttachmentsView); - if (ChunkAttachments.size() != ChunkRawSizes.size()) - { - throw std::runtime_error(fmt::format("Mismatch of loose chunk raw size array, expected {}, found {}", - ChunkAttachments.size(), - ChunkRawSizes.size())); - } - - OutBuildBlobs.reserve(ChunkAttachments.size() + BlockAttachments.size()); - OutBuildBlobs.insert(BlockAttachments.begin(), BlockAttachments.end()); - OutBuildBlobs.insert(ChunkAttachments.begin(), ChunkAttachments.end()); - - for (size_t ChunkAttachmentIndex = 0; ChunkAttachmentIndex < ChunkAttachments.size(); ChunkAttachmentIndex++) - { - OutLooseChunkRawSizes.insert_or_assign(ChunkAttachments[ChunkAttachmentIndex], ChunkRawSizes[ChunkAttachmentIndex]); - } - } -} - -std::vector -BuildsOperationPrimeCache::FilterAlreadyCachedBlobs(const tsl::robin_set& BuildBlobs) -{ - std::vector BlobsToDownload; - BlobsToDownload.reserve(BuildBlobs.size()); - - if (m_Storage.CacheStorage && !BuildBlobs.empty() && !m_Options.ForceUpload) - { - ZEN_TRACE_CPU("BlobCacheExistCheck"); - Stopwatch Timer; - - const std::vector BlobHashes(BuildBlobs.begin(), BuildBlobs.end()); - const std::vector CacheExistsResult = - m_Storage.CacheStorage->BlobsExists(m_BuildId, BlobHashes); - - if (CacheExistsResult.size() == BlobHashes.size()) - { - for (size_t BlobIndex = 0; BlobIndex < BlobHashes.size(); BlobIndex++) - { - if (!CacheExistsResult[BlobIndex].HasBody) - { - BlobsToDownload.push_back(BlobHashes[BlobIndex]); - } - } - size_t FoundCount = BuildBlobs.size() - BlobsToDownload.size(); - - if (FoundCount > 0 && !m_Options.IsQuiet) - { - ZEN_INFO("Remote cache : Found {} out of {} needed blobs in {}", - FoundCount, - BuildBlobs.size(), - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - } - } - } - else - { - BlobsToDownload.insert(BlobsToDownload.end(), BuildBlobs.begin(), BuildBlobs.end()); - } - return BlobsToDownload; -} - -void -BuildsOperationPrimeCache::ScheduleBlobDownloads(std::span BlobsToDownload, - const tsl::robin_map& LooseChunkRawSizes, - std::atomic& MultipartAttachmentCount, - std::atomic& CompletedDownloadCount, - FilteredRate& FilteredDownloadedBytesPerSecond) -{ - std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Downloading"); - - ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - - const size_t BlobCount = BlobsToDownload.size(); - - for (size_t BlobIndex = 0; BlobIndex < BlobCount; BlobIndex++) - { - Work.ScheduleWork( - m_NetworkPool, - [this, - &Work, - BlobsToDownload, - BlobCount, - &LooseChunkRawSizes, - &CompletedDownloadCount, - &FilteredDownloadedBytesPerSecond, - &MultipartAttachmentCount, - BlobIndex](std::atomic&) { - if (!m_AbortFlag) - { - const IoHash& BlobHash = BlobsToDownload[BlobIndex]; - bool IsLargeBlob = false; - if (auto It = LooseChunkRawSizes.find(BlobHash); It != LooseChunkRawSizes.end()) - { - IsLargeBlob = It->second >= m_Options.LargeAttachmentSize; - } - - FilteredDownloadedBytesPerSecond.Start(); - - if (IsLargeBlob) - { - DownloadLargeBlobForCache(Work, - BlobHash, - BlobCount, - CompletedDownloadCount, - MultipartAttachmentCount, - FilteredDownloadedBytesPerSecond); - } - else - { - DownloadSingleBlobForCache(BlobHash, BlobCount, CompletedDownloadCount, FilteredDownloadedBytesPerSecond); - } - } - }); - } - - Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(PendingWork); - - uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() + m_DownloadStats.DownloadedBlockByteCount.load(); - FilteredDownloadedBytesPerSecond.Update(DownloadedBytes); - - std::string DownloadRateString = (CompletedDownloadCount == BlobCount) - ? "" - : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8)); - std::string UploadDetails = m_Storage.CacheStorage ? fmt::format(" {} ({}) uploaded.", - m_StorageCacheStats.PutBlobCount.load(), - NiceBytes(m_StorageCacheStats.PutBlobByteCount.load())) - : ""; - - std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}", - CompletedDownloadCount.load(), - BlobCount, - NiceBytes(DownloadedBytes), - DownloadRateString, - UploadDetails); - ProgressBar->UpdateState({.Task = "Downloading", - .Details = Details, - .TotalCount = BlobCount, - .RemainingCount = BlobCount - CompletedDownloadCount.load(), - .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }); - - FilteredDownloadedBytesPerSecond.Stop(); - ProgressBar->Finish(); -} - -void -BuildsOperationPrimeCache::DownloadLargeBlobForCache(ParallelWork& Work, - const IoHash& BlobHash, - size_t BlobCount, - std::atomic& CompletedDownloadCount, - std::atomic& MultipartAttachmentCount, - FilteredRate& FilteredDownloadedBytesPerSecond) -{ - DownloadLargeBlob(*m_Storage.BuildStorage, - m_TempPath, - m_BuildId, - BlobHash, - m_Options.PreferredMultipartChunkSize, - Work, - m_NetworkPool, - m_DownloadStats.DownloadedChunkByteCount, - MultipartAttachmentCount, - [this, BlobCount, BlobHash, &FilteredDownloadedBytesPerSecond, &CompletedDownloadCount](IoBuffer&& Payload) { - m_DownloadStats.DownloadedChunkCount++; - m_DownloadStats.RequestsCompleteCount++; - - if (!m_AbortFlag) - { - if (Payload && m_Storage.CacheStorage) - { - m_Storage.CacheStorage->PutBuildBlob(m_BuildId, - BlobHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(SharedBuffer(Payload))); - } - } - if (CompletedDownloadCount.fetch_add(1) + 1 == BlobCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - }); -} - -void -BuildsOperationPrimeCache::DownloadSingleBlobForCache(const IoHash& BlobHash, - size_t BlobCount, - std::atomic& CompletedDownloadCount, - FilteredRate& FilteredDownloadedBytesPerSecond) -{ - IoBuffer Payload; - try - { - Payload = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, BlobHash); - - m_DownloadStats.DownloadedBlockCount++; - m_DownloadStats.DownloadedBlockByteCount += Payload.GetSize(); - m_DownloadStats.RequestsCompleteCount++; - } - catch (const std::exception&) - { - // Silence http errors due to abort - if (!m_AbortFlag) - { - throw; - } - } - - if (!m_AbortFlag) - { - if (Payload && m_Storage.CacheStorage) - { - m_Storage.CacheStorage->PutBuildBlob(m_BuildId, - BlobHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(SharedBuffer(std::move(Payload)))); - } - if (CompletedDownloadCount.fetch_add(1) + 1 == BlobCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - } -} - -CompositeBuffer -ValidateBlob(std::atomic& AbortFlag, - BuildStorageBase& Storage, - const Oid& BuildId, - const IoHash& BlobHash, - uint64_t& OutCompressedSize, - uint64_t& OutDecompressedSize) -{ - ZEN_TRACE_CPU("ValidateBlob"); - IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlobHash); - if (!Payload) - { - throw std::runtime_error(fmt::format("Blob {} could not be found", BlobHash)); - } - return ValidateBlob(AbortFlag, std::move(Payload), BlobHash, OutCompressedSize, OutDecompressedSize); -} - -ChunkBlockDescription -BuildsOperationValidateBuildPart::ValidateChunkBlock(IoBuffer&& Payload, - const IoHash& BlobHash, - uint64_t& OutCompressedSize, - uint64_t& OutDecompressedSize) -{ - CompositeBuffer BlockBuffer = ValidateBlob(m_AbortFlag, std::move(Payload), BlobHash, OutCompressedSize, OutDecompressedSize); - if (!BlockBuffer) - { - throw std::runtime_error(fmt::format("Chunk block blob {} is not compressed using 'None' compression level", BlobHash)); - } - return GetChunkBlockDescription(BlockBuffer.Flatten(), BlobHash); -} - -std::vector> -ResolveBuildPartNames(CbObjectView BuildObject, - const Oid& BuildId, - const std::vector& BuildPartIds, - std::span BuildPartNames, - std::uint64_t& OutPreferredMultipartChunkSize) -{ - std::vector> Result; - { - CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); - if (!PartsObject) - { - throw std::runtime_error("Build object does not have a 'parts' object"); - } - - OutPreferredMultipartChunkSize = BuildObject["chunkSize"sv].AsUInt64(OutPreferredMultipartChunkSize); - - std::vector> AvailableParts; - - for (CbFieldView PartView : PartsObject) - { - const std::string BuildPartName = std::string(PartView.GetName()); - const Oid BuildPartId = PartView.AsObjectId(); - if (BuildPartId == Oid::Zero) - { - ExtendableStringBuilder<128> SB; - for (CbFieldView ScanPartView : PartsObject) - { - SB.Append(fmt::format("\n {}: {}", ScanPartView.GetName(), ScanPartView.AsObjectId())); - } - throw std::runtime_error(fmt::format("Build object parts does not have a '{}' object id{}", BuildPartName, SB.ToView())); - } - AvailableParts.push_back({BuildPartId, BuildPartName}); - } - - if (BuildPartIds.empty() && BuildPartNames.empty()) - { - Result = AvailableParts; - } - else - { - for (const std::string& BuildPartName : BuildPartNames) - { - if (auto It = std::find_if(AvailableParts.begin(), - AvailableParts.end(), - [&BuildPartName](const auto& Part) { return Part.second == BuildPartName; }); - It != AvailableParts.end()) - { - Result.push_back(*It); - } - else - { - throw std::runtime_error(fmt::format("Build {} object does not have a part named '{}'", BuildId, BuildPartName)); - } - } - for (const Oid& BuildPartId : BuildPartIds) - { - if (auto It = std::find_if(AvailableParts.begin(), - AvailableParts.end(), - [&BuildPartId](const auto& Part) { return Part.first == BuildPartId; }); - It != AvailableParts.end()) - { - Result.push_back(*It); - } - else - { - throw std::runtime_error(fmt::format("Build {} object does not have a part with id '{}'", BuildId, BuildPartId)); - } - } - } - - if (Result.empty()) - { - throw std::runtime_error(fmt::format("Build object does not have any parts", BuildId)); - } - } - return Result; -} - -ChunkedFolderContent -GetRemoteContent(LoggerRef InLog, - StorageInstance& Storage, - const Oid& BuildId, - const std::vector>& BuildParts, - const BuildManifest& Manifest, - std::span IncludeWildcards, - std::span ExcludeWildcards, - std::unique_ptr& OutChunkController, - std::vector& OutPartContents, - std::vector& OutBlockDescriptions, - std::vector& OutLooseChunkHashes, - bool IsQuiet, - bool IsVerbose, - bool DoExtraContentVerify) -{ - ZEN_TRACE_CPU("GetRemoteContent"); - ZEN_SCOPED_LOG(InLog); - - Stopwatch GetBuildPartTimer; - const Oid BuildPartId = BuildParts[0].first; - const std::string_view BuildPartName = BuildParts[0].second; - CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); - if (!IsQuiet) - { - ZEN_INFO("GetBuildPart {} ('{}') took {}. Payload size: {}", - BuildPartId, - BuildPartName, - NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), - NiceBytes(BuildPartManifest.GetSize())); - ZEN_INFO("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv)); - } - - { - CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView(); - std::string_view ChunkerName = Chunker["name"sv].AsString(); - CbObjectView Parameters = Chunker["parameters"sv].AsObjectView(); - OutChunkController = CreateChunkingController(ChunkerName, Parameters); - } - - auto ParseBuildPartManifest = [&Log, IsQuiet, IsVerbose, DoExtraContentVerify](StorageInstance& Storage, - const Oid& BuildId, - const Oid& BuildPartId, - CbObject BuildPartManifest, - std::span IncludeWildcards, - std::span ExcludeWildcards, - const BuildManifest::Part* OptionalManifest, - ChunkedFolderContent& OutRemoteContent, - std::vector& OutBlockDescriptions, - std::vector& OutLooseChunkHashes) { - std::vector AbsoluteChunkOrders; - std::vector LooseChunkRawSizes; - std::vector BlockRawHashes; - - ReadBuildContentFromCompactBinary(BuildPartManifest, - OutRemoteContent.Platform, - OutRemoteContent.Paths, - OutRemoteContent.RawHashes, - OutRemoteContent.RawSizes, - OutRemoteContent.Attributes, - OutRemoteContent.ChunkedContent.SequenceRawHashes, - OutRemoteContent.ChunkedContent.ChunkCounts, - AbsoluteChunkOrders, - OutLooseChunkHashes, - LooseChunkRawSizes, - BlockRawHashes); - - // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them - - { - if (!IsQuiet) - { - ZEN_INFO("Fetching metadata for {} blocks", BlockRawHashes.size()); - } - - Stopwatch GetBlockMetadataTimer; - - bool AttemptFallback = false; - OutBlockDescriptions = GetBlockDescriptions(Log(), - *Storage.BuildStorage, - Storage.CacheStorage.get(), - BuildId, - BlockRawHashes, - AttemptFallback, - IsQuiet, - IsVerbose); - - if (!IsQuiet) - { - ZEN_INFO("GetBlockMetadata for {} took {}. Found {} blocks", - BuildPartId, - NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), - OutBlockDescriptions.size()); - } - } - - CalculateLocalChunkOrders(AbsoluteChunkOrders, - OutLooseChunkHashes, - LooseChunkRawSizes, - OutBlockDescriptions, - OutRemoteContent.ChunkedContent.ChunkHashes, - OutRemoteContent.ChunkedContent.ChunkRawSizes, - OutRemoteContent.ChunkedContent.ChunkOrders, - DoExtraContentVerify); - - std::vector DeletedPaths; - - if (OptionalManifest) - { - tsl::robin_set PathsInManifest; - PathsInManifest.reserve(OptionalManifest->Files.size()); - for (const std::filesystem::path& ManifestPath : OptionalManifest->Files) - { - PathsInManifest.insert(ToLower(ManifestPath.generic_string())); - } - for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths) - { - if (!PathsInManifest.contains(ToLower(RemotePath.generic_string()))) - { - DeletedPaths.push_back(RemotePath); - } - } - } - - if (!IncludeWildcards.empty() || !ExcludeWildcards.empty()) - { - for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths) - { - if (!IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(RemotePath.generic_string()), /*CaseSensitive*/ true)) - { - DeletedPaths.push_back(RemotePath); - } - } - } - - if (!DeletedPaths.empty()) - { - OutRemoteContent = DeletePathsFromChunkedContent(OutRemoteContent, DeletedPaths); - InlineRemoveUnusedHashes(OutLooseChunkHashes, OutRemoteContent.ChunkedContent.ChunkHashes); - } - -#if ZEN_BUILD_DEBUG - ValidateChunkedFolderContent(OutRemoteContent, OutBlockDescriptions, OutLooseChunkHashes, IncludeWildcards, ExcludeWildcards); -#endif // ZEN_BUILD_DEBUG - }; - - auto FindManifest = [&Manifest](const Oid& BuildPartId, std::string_view BuildPartName) -> const BuildManifest::Part* { - if (Manifest.Parts.empty()) - { - return nullptr; - } - if (Manifest.Parts.size() == 1) - { - if (Manifest.Parts[0].PartId == Oid::Zero && Manifest.Parts[0].PartName.empty()) - { - return &Manifest.Parts[0]; - } - } - - auto It = std::find_if(Manifest.Parts.begin(), Manifest.Parts.end(), [BuildPartId, BuildPartName](const BuildManifest::Part& Part) { - if (Part.PartId != Oid::Zero) - { - return Part.PartId == BuildPartId; - } - if (!Part.PartName.empty()) - { - return Part.PartName == BuildPartName; - } - return false; - }); - if (It != Manifest.Parts.end()) - { - return &(*It); - } - return nullptr; - }; - - OutPartContents.resize(1); - ParseBuildPartManifest(Storage, - BuildId, - BuildPartId, - BuildPartManifest, - IncludeWildcards, - ExcludeWildcards, - FindManifest(BuildPartId, BuildPartName), - OutPartContents[0], - OutBlockDescriptions, - OutLooseChunkHashes); - ChunkedFolderContent RemoteContent; - if (BuildParts.size() > 1) - { - std::vector OverlayBlockDescriptions; - std::vector OverlayLooseChunkHashes; - for (size_t PartIndex = 1; PartIndex < BuildParts.size(); PartIndex++) - { - const Oid& OverlayBuildPartId = BuildParts[PartIndex].first; - const std::string& OverlayBuildPartName = BuildParts[PartIndex].second; - Stopwatch GetOverlayBuildPartTimer; - CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId); - if (!IsQuiet) - { - ZEN_INFO("GetBuildPart {} ('{}') took {}. Payload size: {}", - OverlayBuildPartId, - OverlayBuildPartName, - NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()), - NiceBytes(OverlayBuildPartManifest.GetSize())); - } - - ChunkedFolderContent OverlayPartContent; - std::vector OverlayPartBlockDescriptions; - std::vector OverlayPartLooseChunkHashes; - - ParseBuildPartManifest(Storage, - BuildId, - OverlayBuildPartId, - OverlayBuildPartManifest, - IncludeWildcards, - ExcludeWildcards, - FindManifest(OverlayBuildPartId, OverlayBuildPartName), - OverlayPartContent, - OverlayPartBlockDescriptions, - OverlayPartLooseChunkHashes); - OutPartContents.push_back(OverlayPartContent); - OverlayBlockDescriptions.insert(OverlayBlockDescriptions.end(), - OverlayPartBlockDescriptions.begin(), - OverlayPartBlockDescriptions.end()); - OverlayLooseChunkHashes.insert(OverlayLooseChunkHashes.end(), - OverlayPartLooseChunkHashes.begin(), - OverlayPartLooseChunkHashes.end()); - } - - RemoteContent = MergeChunkedFolderContents(OutPartContents[0], std::span(OutPartContents).subspan(1)); - { - tsl::robin_set AllBlockHashes; - for (const ChunkBlockDescription& Description : OutBlockDescriptions) - { - AllBlockHashes.insert(Description.BlockHash); - } - for (const ChunkBlockDescription& Description : OverlayBlockDescriptions) - { - if (!AllBlockHashes.contains(Description.BlockHash)) - { - AllBlockHashes.insert(Description.BlockHash); - OutBlockDescriptions.push_back(Description); - } - } - } - { - tsl::robin_set AllLooseChunkHashes(OutLooseChunkHashes.begin(), OutLooseChunkHashes.end()); - for (const IoHash& OverlayLooseChunkHash : OverlayLooseChunkHashes) - { - if (!AllLooseChunkHashes.contains(OverlayLooseChunkHash)) - { - AllLooseChunkHashes.insert(OverlayLooseChunkHash); - OutLooseChunkHashes.push_back(OverlayLooseChunkHash); - } - } - } - } - else - { - RemoteContent = OutPartContents[0]; - } - return RemoteContent; -} - -std::string -GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix) -{ - ExtendableStringBuilder<512> SB; - std::vector> NameStringValuePairs; - for (CbFieldView Field : Object) - { - std::string_view Name = Field.GetName(); - switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) - { - case CbFieldType::String: - NameStringValuePairs.push_back({std::string(Name), std::string(Accessor.AsString())}); - break; - case CbFieldType::IntegerPositive: - NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerPositive())}); - break; - case CbFieldType::IntegerNegative: - NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerNegative())}); - break; - case CbFieldType::Float32: - { - const float Value = Accessor.AsFloat32(); - if (std::isfinite(Value)) - { - NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.9g}", Value)}); - } - else - { - NameStringValuePairs.push_back({std::string(Name), "null"}); - } - } - break; - case CbFieldType::Float64: - { - const double Value = Accessor.AsFloat64(); - if (std::isfinite(Value)) - { - NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.17g}", Value)}); - } - else - { - NameStringValuePairs.push_back({std::string(Name), "null"}); - } - } - break; - case CbFieldType::BoolFalse: - NameStringValuePairs.push_back({std::string(Name), "false"}); - break; - case CbFieldType::BoolTrue: - NameStringValuePairs.push_back({std::string(Name), "true"}); - break; - case CbFieldType::Hash: - { - NameStringValuePairs.push_back({std::string(Name), Accessor.AsHash().ToHexString()}); - } - break; - case CbFieldType::Uuid: - { - StringBuilder Builder; - Accessor.AsUuid().ToString(Builder); - NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); - } - break; - case CbFieldType::DateTime: - { - ExtendableStringBuilder<64> Builder; - Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); - NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); - } - break; - case CbFieldType::TimeSpan: - { - ExtendableStringBuilder<64> Builder; - const TimeSpan Span(Accessor.AsTimeSpanTicks()); - if (Span.GetDays() == 0) - { - Builder << Span.ToString("%h:%m:%s.%n"); - } - else - { - Builder << Span.ToString("%d.%h:%m:%s.%n"); - } - NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); - break; - } - case CbFieldType::ObjectId: - NameStringValuePairs.push_back({std::string(Name), Accessor.AsObjectId().ToString()}); - break; - } - } - std::string::size_type LongestKey = 0; - for (const std::pair& KeyValue : NameStringValuePairs) - { - LongestKey = Max(KeyValue.first.length(), LongestKey); - } - for (const std::pair& KeyValue : NameStringValuePairs) - { - SB.Append(fmt::format("{}{:<{}}: {}{}", Prefix, KeyValue.first, LongestKey, KeyValue.second, Suffix)); - } - return SB.ToString(); -} - -#if ZEN_WITH_TESTS - -namespace buildstorageoperations_testutils { - struct TestState - { - TestState(const std::filesystem::path& InRootPath) - : RootPath(InRootPath) - , LogOutput(CreateStandardProgress(Log)) - , ChunkController(CreateStandardChunkingController(StandardChunkingControllerSettings{})) - , ChunkCache(CreateMemoryChunkingCache()) - , WorkerPool(2) - , NetworkPool(2) - { - } - - void Initialize() - { - StoragePath = RootPath / "storage"; - TempPath = RootPath / "temp"; - SystemRootDir = RootPath / "sysroot"; - ZenFolderPath = RootPath / ".zen"; - - CreateDirectories(TempPath); - CreateDirectories(StoragePath); - - Storage.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false); - } - - void CreateSourceData(const std::filesystem::path& Source, std::span Paths, std::span Sizes) - { - const std::filesystem::path SourcePath = RootPath / Source; - CreateDirectories(SourcePath); - for (size_t FileIndex = 0; FileIndex < Paths.size(); FileIndex++) - { - const std::string& FilePath = Paths[FileIndex]; - const uint64_t FileSize = Sizes[FileIndex]; - IoBuffer FileData = FileSize > 0 ? CreateSemiRandomBlob(FileSize) : IoBuffer{}; - WriteFile(SourcePath / FilePath, FileData); - } - } - - std::vector> Upload(const Oid& BuildId, - const Oid& BuildPartId, - const std::string_view BuildPartName, - const std::filesystem::path& Source, - const std::filesystem::path& ManifestPath) - { - const std::filesystem::path SourcePath = RootPath / Source; - CbObject MetaData; - BuildsOperationUploadFolder Upload(Log, - *LogOutput, - Storage, - AbortFlag, - PauseFlag, - WorkerPool, - NetworkPool, - BuildId, - SourcePath, - true, - MetaData, - BuildsOperationUploadFolder::Options{.TempDir = TempPath}); - return Upload.Execute(BuildPartId, BuildPartName, ManifestPath, *ChunkController, *ChunkCache); - } - - void ValidateUpload(const Oid& BuildId, const std::vector>& Parts) - { - for (auto Part : Parts) - { - BuildsOperationValidateBuildPart Validate(Log, - *LogOutput, - *Storage.BuildStorage, - AbortFlag, - PauseFlag, - WorkerPool, - NetworkPool, - BuildId, - Part.first, - Part.second, - BuildsOperationValidateBuildPart::Options{}); - Validate.Execute(); - } - } - - FolderContent Download(const Oid& BuildId, - const Oid& BuildPartId, - const std::string_view BuildPartName, - const std::filesystem::path& Target, - bool Append) - { - const std::filesystem::path TargetPath = RootPath / Target; - - CreateDirectories(TargetPath); - - uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - CbObject BuildObject = Storage.BuildStorage->GetBuild(BuildId); - std::vector PartIds; - if (BuildPartId != Oid::Zero) - { - PartIds.push_back(BuildPartId); - } - std::vector PartNames; - if (!BuildPartName.empty()) - { - PartNames.push_back(std::string(BuildPartName)); - } - std::vector> AllBuildParts = - ResolveBuildPartNames(BuildObject, BuildId, PartIds, PartNames, PreferredMultipartChunkSize); - - std::vector PartContents; - - std::vector BlockDescriptions; - std::vector LooseChunkHashes; - - ChunkedFolderContent RemoteContent = GetRemoteContent(Log, - Storage, - BuildId, - AllBuildParts, - {}, - {}, - {}, - ChunkController, - PartContents, - BlockDescriptions, - LooseChunkHashes, - /*IsQuiet*/ false, - /*IsVerbose*/ false, - /*DoExtraContentVerify*/ true); - - GetFolderContentStatistics LocalFolderScanStats; - - struct ContentVisitor : public GetDirectoryContentVisitor - { - virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) - { - RwLock::ExclusiveLockScope _(ExistingPathsLock); - for (const std::filesystem::path& FileName : Content.FileNames) - { - if (RelativeRoot.empty()) - { - ExistingPaths.push_back(FileName); - } - else - { - ExistingPaths.push_back(RelativeRoot / FileName); - } - } - } - - RwLock ExistingPathsLock; - std::vector ExistingPaths; - } Visitor; - - Latch PendingWorkCount(1); - - GetDirectoryContent(TargetPath, - DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive, - Visitor, - WorkerPool, - PendingWorkCount); - - PendingWorkCount.CountDown(); - PendingWorkCount.Wait(); - - FolderContent CurrentLocalFolderState = GetValidFolderContent( - WorkerPool, - LocalFolderScanStats, - TargetPath, - Visitor.ExistingPaths, - [](uint64_t PathCount, uint64_t CompletedPathCount) { ZEN_UNUSED(PathCount, CompletedPathCount); }, - 1000, - AbortFlag, - PauseFlag); - - ChunkingStatistics LocalChunkingStats; - ChunkedFolderContent LocalContent = ChunkFolderContent( - LocalChunkingStats, - WorkerPool, - TargetPath, - CurrentLocalFolderState, - *ChunkController, - *ChunkCache, - 1000, - [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { ZEN_UNUSED(IsAborted, IsPaused); }, - AbortFlag, - PauseFlag); - - if (Append) - { - RemoteContent = ApplyChunkedContentOverlay(LocalContent, RemoteContent, {}, {}); - } - - const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent); - const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); - - BuildsOperationUpdateFolder Download(Log, - *LogOutput, - Storage, - AbortFlag, - PauseFlag, - WorkerPool, - NetworkPool, - BuildId, - TargetPath, - LocalContent, - LocalLookup, - RemoteContent, - RemoteLookup, - BlockDescriptions, - LooseChunkHashes, - BuildsOperationUpdateFolder::Options{.SystemRootDir = SystemRootDir, - .ZenFolderPath = ZenFolderPath, - .ValidateCompletedSequences = true}); - FolderContent ResultingState; - Download.Execute(ResultingState); - - return ResultingState; - } - - void ValidateDownload(std::span Paths, - std::span Sizes, - const std::filesystem::path& Source, - const std::filesystem::path& Target, - const FolderContent& DownloadContent) - { - const std::filesystem::path SourcePath = RootPath / Source; - const std::filesystem::path TargetPath = RootPath / Target; - - CHECK_EQ(Paths.size(), DownloadContent.Paths.size()); - tsl::robin_map ExpectedSizes; - tsl::robin_map ExpectedHashes; - for (size_t Index = 0; Index < Paths.size(); Index++) - { - const std::string LookupString = std::filesystem::path(Paths[Index]).generic_string(); - ExpectedSizes.insert_or_assign(LookupString, Sizes[Index]); - std::filesystem::path FilePath = SourcePath / Paths[Index]; - const IoHash SourceHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(FilePath.make_preferred())); - ExpectedHashes.insert_or_assign(LookupString, SourceHash); - } - for (size_t Index = 0; Index < DownloadContent.Paths.size(); Index++) - { - const std::string LookupString = std::filesystem::path(DownloadContent.Paths[Index]).generic_string(); - auto SizeIt = ExpectedSizes.find(LookupString); - CHECK_NE(SizeIt, ExpectedSizes.end()); - CHECK_EQ(SizeIt->second, DownloadContent.RawSizes[Index]); - std::filesystem::path FilePath = TargetPath / DownloadContent.Paths[Index]; - const IoHash DownloadedHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(FilePath.make_preferred())); - auto HashIt = ExpectedHashes.find(LookupString); - CHECK_NE(HashIt, ExpectedHashes.end()); - CHECK_EQ(HashIt->second, DownloadedHash); - } - } - - const std::filesystem::path RootPath; - std::filesystem::path StoragePath; - std::filesystem::path TempPath; - std::filesystem::path SystemRootDir; - std::filesystem::path ZenFolderPath; - - LoggerRef Log = ConsoleLog(); - std::unique_ptr LogOutput; - - std::unique_ptr ChunkController; - std::unique_ptr ChunkCache; - - StorageInstance Storage; - BuildStorageBase::Statistics StorageStats; - - WorkerThreadPool WorkerPool; - WorkerThreadPool NetworkPool; - - std::atomic AbortFlag; - std::atomic PauseFlag; - }; - -} // namespace buildstorageoperations_testutils - -TEST_SUITE_BEGIN("remotestore.buildstorageoperations"); - -TEST_CASE("buildstorageoperations.upload.folder") -{ - using namespace buildstorageoperations_testutils; - - FastRandom BaseRandom; - - const size_t FileCount = 11; - - const std::string Paths[FileCount] = {{"file_1"}, - {"file_2.exe"}, - {"file_3.txt"}, - {"dir_1/dir1_file_1.exe"}, - {"dir_1/dir1_file_2.pdb"}, - {"dir_1/dir1_file_3.txt"}, - {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"}, - {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"}, - {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"}, - {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"}, - {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}}; - const uint64_t Sizes[FileCount] = - {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u}; - - ScopedTemporaryDirectory SourceFolder; - TestState State(SourceFolder.Path()); - State.Initialize(); - State.CreateSourceData("source", Paths, Sizes); - - const Oid BuildId = Oid::NewOid(); - const Oid BuildPartId = Oid::NewOid(); - const std::string BuildPartName = "default"; - - auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", {}); - - CHECK_EQ(Result.size(), 1u); - CHECK_EQ(Result[0].first, BuildPartId); - CHECK_EQ(Result[0].second, BuildPartName); - State.ValidateUpload(BuildId, Result); - - FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false); - CHECK_EQ(DownloadContent.Paths.size(), FileCount); - State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent); -} - -TEST_CASE("buildstorageoperations.upload.manifest") -{ - using namespace buildstorageoperations_testutils; - - FastRandom BaseRandom; - - const size_t FileCount = 11; - - const std::string Paths[FileCount] = {{"file_1"}, - {"file_2.exe"}, - {"file_3.txt"}, - {"dir_1/dir1_file_1.exe"}, - {"dir_1/dir1_file_2.pdb"}, - {"dir_1/dir1_file_3.txt"}, - {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"}, - {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"}, - {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"}, - {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"}, - {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}}; - const uint64_t Sizes[FileCount] = - {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u}; - - ScopedTemporaryDirectory SourceFolder; - TestState State(SourceFolder.Path()); - State.Initialize(); - State.CreateSourceData("source", Paths, Sizes); - - std::span ManifestFiles(Paths); - ManifestFiles = ManifestFiles.subspan(0, FileCount / 2); - - std::span ManifestSizes(Sizes); - ManifestSizes = ManifestSizes.subspan(0, FileCount / 2); - - ExtendableStringBuilder<1024> Manifest; - for (const std::string& FilePath : ManifestFiles) - { - Manifest << FilePath << "\n"; - } - - WriteFile(State.RootPath / "manifest.txt", IoBuffer(IoBuffer::Wrap, Manifest.Data(), Manifest.Size())); - - const Oid BuildId = Oid::NewOid(); - const Oid BuildPartId = Oid::NewOid(); - const std::string BuildPartName = "default"; - - auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", State.RootPath / "manifest.txt"); - - CHECK_EQ(Result.size(), 1u); - CHECK_EQ(Result[0].first, BuildPartId); - CHECK_EQ(Result[0].second, BuildPartName); - State.ValidateUpload(BuildId, Result); - - FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false); - State.ValidateDownload(ManifestFiles, ManifestSizes, "source", "download", DownloadContent); -} - -TEST_CASE("buildstorageoperations.memorychunkingcache") -{ - using namespace buildstorageoperations_testutils; - - FastRandom BaseRandom; - - const size_t FileCount = 11; - - const std::string Paths[FileCount] = {{"file_1"}, - {"file_2.exe"}, - {"file_3.txt"}, - {"dir_1/dir1_file_1.exe"}, - {"dir_1/dir1_file_2.pdb"}, - {"dir_1/dir1_file_3.txt"}, - {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"}, - {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"}, - {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"}, - {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"}, - {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}}; - const uint64_t Sizes[FileCount] = - {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u}; - - ScopedTemporaryDirectory SourceFolder; - TestState State(SourceFolder.Path()); - State.Initialize(); - State.CreateSourceData("source", Paths, Sizes); - - const Oid BuildId = Oid::NewOid(); - const Oid BuildPartId = Oid::NewOid(); - const std::string BuildPartName = "default"; - - { - const std::filesystem::path SourcePath = SourceFolder.Path() / "source"; - CbObject MetaData; - BuildsOperationUploadFolder Upload(State.Log, - *State.LogOutput, - State.Storage, - State.AbortFlag, - State.PauseFlag, - State.WorkerPool, - State.NetworkPool, - BuildId, - SourcePath, - true, - MetaData, - BuildsOperationUploadFolder::Options{.TempDir = State.TempPath}); - auto Result = Upload.Execute(BuildPartId, BuildPartName, {}, *State.ChunkController, *State.ChunkCache); - - CHECK_EQ(Upload.m_ChunkingStats.FilesStoredInCache.load(), FileCount - 1); // Zero size files are not stored in cache - CHECK_EQ(Upload.m_ChunkingStats.BytesStoredInCache.load(), std::accumulate(&Sizes[0], &Sizes[FileCount], uint64_t(0))); - CHECK(Upload.m_ChunkingStats.ChunksStoredInCache.load() >= FileCount - 1); // Zero size files are not stored in cache - - CHECK_EQ(Result.size(), 1u); - CHECK_EQ(Result[0].first, BuildPartId); - CHECK_EQ(Result[0].second, BuildPartName); - } - - auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", {}); - - const Oid BuildId2 = Oid::NewOid(); - const Oid BuildPartId2 = Oid::NewOid(); - - { - const std::filesystem::path SourcePath = SourceFolder.Path() / "source"; - CbObject MetaData; - BuildsOperationUploadFolder Upload(State.Log, - *State.LogOutput, - State.Storage, - State.AbortFlag, - State.PauseFlag, - State.WorkerPool, - State.NetworkPool, - BuildId2, - SourcePath, - true, - MetaData, - BuildsOperationUploadFolder::Options{.TempDir = State.TempPath}); - Upload.Execute(BuildPartId2, BuildPartName, {}, *State.ChunkController, *State.ChunkCache); - - CHECK_EQ(Upload.m_ChunkingStats.FilesFoundInCache.load(), FileCount - 1); // Zero size files are not stored in cache - CHECK_EQ(Upload.m_ChunkingStats.BytesFoundInCache.load(), std::accumulate(&Sizes[0], &Sizes[FileCount], uint64_t(0))); - CHECK(Upload.m_ChunkingStats.ChunksFoundInCache.load() >= FileCount - 1); // Zero size files are not stored in cache - } - - FolderContent DownloadContent = State.Download(BuildId2, BuildPartId2, {}, "download", /* Append */ false); - State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent); -} - -TEST_CASE("buildstorageoperations.upload.multipart") -{ - // Disabled since it relies on authentication and specific block being present in cloud storage - if (false) - { - using namespace buildstorageoperations_testutils; - - FastRandom BaseRandom; - - const size_t FileCount = 11; - - const std::string Paths[FileCount] = {{"file_1"}, - {"file_2.exe"}, - {"file_3.txt"}, - {"dir_1/dir1_file_1.exe"}, - {"dir_1/dir1_file_2.pdb"}, - {"dir_1/dir1_file_3.txt"}, - {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"}, - {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"}, - {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"}, - {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"}, - {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}}; - const uint64_t Sizes[FileCount] = - {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u}; - - ScopedTemporaryDirectory SourceFolder; - TestState State(SourceFolder.Path()); - State.Initialize(); - State.CreateSourceData("source", Paths, Sizes); - - std::span ManifestFiles1(Paths); - ManifestFiles1 = ManifestFiles1.subspan(0, FileCount / 2); - - std::span ManifestSizes1(Sizes); - ManifestSizes1 = ManifestSizes1.subspan(0, FileCount / 2); - - std::span ManifestFiles2(Paths); - ManifestFiles2 = ManifestFiles2.subspan(FileCount / 2 - 1); - - std::span ManifestSizes2(Sizes); - ManifestSizes2 = ManifestSizes2.subspan(FileCount / 2 - 1); - - const Oid BuildPart1Id = Oid::NewOid(); - const std::string BuildPart1Name = "part1"; - const Oid BuildPart2Id = Oid::NewOid(); - const std::string BuildPart2Name = "part2"; - { - CbObjectWriter Writer; - Writer.BeginObject("parts"sv); - { - Writer.BeginObject(BuildPart1Name); - { - Writer.AddObjectId("partId"sv, BuildPart1Id); - Writer.BeginArray("files"sv); - for (const std::string& ManifestFile : ManifestFiles1) - { - Writer.AddString(ManifestFile); - } - Writer.EndArray(); // files - } - Writer.EndObject(); // part1 - - Writer.BeginObject(BuildPart2Name); - { - Writer.AddObjectId("partId"sv, BuildPart2Id); - Writer.BeginArray("files"sv); - for (const std::string& ManifestFile : ManifestFiles2) - { - Writer.AddString(ManifestFile); - } - Writer.EndArray(); // files - } - Writer.EndObject(); // part2 - } - Writer.EndObject(); // parts - - ExtendableStringBuilder<1024> Manifest; - CompactBinaryToJson(Writer.Save(), Manifest); - WriteFile(State.RootPath / "manifest.json", IoBuffer(IoBuffer::Wrap, Manifest.Data(), Manifest.Size())); - } - - const Oid BuildId = Oid::NewOid(); - - auto Result = State.Upload(BuildId, {}, {}, "source", State.RootPath / "manifest.json"); - - CHECK_EQ(Result.size(), 2u); - CHECK_EQ(Result[0].first, BuildPart1Id); - CHECK_EQ(Result[0].second, BuildPart1Name); - CHECK_EQ(Result[1].first, BuildPart2Id); - CHECK_EQ(Result[1].second, BuildPart2Name); - State.ValidateUpload(BuildId, Result); - - FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false); - State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent); - - FolderContent Part1DownloadContent = State.Download(BuildId, BuildPart1Id, {}, "download_part1", /* Append */ false); - State.ValidateDownload(ManifestFiles1, ManifestSizes1, "source", "download_part1", Part1DownloadContent); - - FolderContent Part2DownloadContent = State.Download(BuildId, Oid::Zero, BuildPart2Name, "download_part2", /* Append */ false); - State.ValidateDownload(ManifestFiles2, ManifestSizes2, "source", "download_part2", Part2DownloadContent); - - (void)State.Download(BuildId, BuildPart1Id, BuildPart1Name, "download_part1+2", /* Append */ false); - FolderContent Part1And2DownloadContent = State.Download(BuildId, BuildPart2Id, {}, "download_part1+2", /* Append */ true); - State.ValidateDownload(Paths, Sizes, "source", "download_part1+2", Part1And2DownloadContent); - } -} - -TEST_CASE("buildstorageoperations.partial.block.download" * doctest::skip(true)) -{ - const std::string OidcExecutableName = "OidcToken" ZEN_EXE_SUFFIX_LITERAL; - std::filesystem::path OidcTokenExePath = (GetRunningExecutablePath().parent_path() / OidcExecutableName).make_preferred(); - - HttpClientSettings ClientSettings{ - .LogCategory = "httpbuildsclient", - .AccessTokenProvider = - httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, "https://jupiter.devtools.epicgames.com", true, false, false), - .AssumeHttp2 = false, - .AllowResume = true, - .RetryCount = 0, - .Verbose = false}; - - HttpClient HttpClient("https://euc.jupiter.devtools.epicgames.com", ClientSettings); - - const std::string_view Namespace = "fortnite.oplog"; - const std::string_view Bucket = "fortnitegame.staged-build.fortnite-main.ps4-client"; - const Oid BuildId = Oid::FromHexString("09a76ea92ad301d4724fafad"); - - { - HttpClient::Response Response = HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}", Namespace, Bucket, BuildId), - HttpClient::Accept(ZenContentType::kCbObject)); - CbValidateError ValidateResult = CbValidateError::None; - CbObject Object = ValidateAndReadCompactBinaryObject(IoBuffer(Response.ResponsePayload), ValidateResult); - REQUIRE(ValidateResult == CbValidateError::None); - } - - std::vector BlockDescriptions; - { - CbObjectWriter Request; - - Request.BeginArray("blocks"sv); - { - Request.AddHash(IoHash::FromHexString("7c353ed782675a5e8f968e61e51fc797ecdc2882")); - } - Request.EndArray(); - - IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer(); - Payload.SetContentType(ZenContentType::kCbObject); - - HttpClient::Response BlockDescriptionsResponse = - HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/blocks/getBlockMetadata", Namespace, Bucket, BuildId), - Payload, - HttpClient::Accept(ZenContentType::kCbObject)); - REQUIRE(BlockDescriptionsResponse.IsSuccess()); - - CbValidateError ValidateResult = CbValidateError::None; - CbObject Object = ValidateAndReadCompactBinaryObject(IoBuffer(BlockDescriptionsResponse.ResponsePayload), ValidateResult); - REQUIRE(ValidateResult == CbValidateError::None); - - { - CbArrayView BlocksArray = Object["blocks"sv].AsArrayView(); - for (CbFieldView Block : BlocksArray) - { - ChunkBlockDescription Description = ParseChunkBlockDescription(Block.AsObjectView()); - BlockDescriptions.emplace_back(std::move(Description)); - } - } - } - - REQUIRE(!BlockDescriptions.empty()); - - const IoHash BlockHash = BlockDescriptions.back().BlockHash; - - const ChunkBlockDescription& BlockDescription = BlockDescriptions.front(); - REQUIRE(!BlockDescription.ChunkRawHashes.empty()); - REQUIRE(!BlockDescription.ChunkCompressedLengths.empty()); - - std::vector> ChunkOffsetAndSizes; - uint64_t Offset = gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); - - for (uint32_t ChunkCompressedSize : BlockDescription.ChunkCompressedLengths) - { - ChunkOffsetAndSizes.push_back(std::make_pair(Offset, ChunkCompressedSize)); - Offset += ChunkCompressedSize; - } - - ScopedTemporaryDirectory SourceFolder; - - auto Validate = [&](std::span ChunkIndexesToFetch) { - std::vector> Ranges; - for (uint32_t ChunkIndex : ChunkIndexesToFetch) - { - Ranges.push_back(ChunkOffsetAndSizes[ChunkIndex]); - } - - HttpClient::KeyValueMap Headers; - if (!Ranges.empty()) - { - ExtendableStringBuilder<512> SB; - for (const std::pair& R : Ranges) - { - if (SB.Size() > 0) - { - SB << ", "; - } - SB << R.first << "-" << R.first + R.second - 1; - } - Headers.Entries.insert({"Range", fmt::format("bytes={}", SB.ToView())}); - } - - HttpClient::Response GetBlobRangesResponse = HttpClient.Download( - fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect=false", Namespace, Bucket, BuildId, BlockHash), - SourceFolder.Path(), - Headers); - - REQUIRE(GetBlobRangesResponse.IsSuccess()); - [[maybe_unused]] MemoryView RangesMemoryView = GetBlobRangesResponse.ResponsePayload.GetView(); - - std::vector> PayloadRanges = GetBlobRangesResponse.GetRanges(Ranges); - if (PayloadRanges.empty()) - { - // We got the whole blob, use the ranges as is - PayloadRanges = Ranges; - } - - REQUIRE(PayloadRanges.size() == Ranges.size()); - - for (uint32_t RangeIndex = 0; RangeIndex < PayloadRanges.size(); RangeIndex++) - { - const std::pair& PayloadRange = PayloadRanges[RangeIndex]; - - CHECK_EQ(PayloadRange.second, Ranges[RangeIndex].second); - - IoBuffer ChunkPayload(GetBlobRangesResponse.ResponsePayload, PayloadRange.first, PayloadRange.second); - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed(SharedBuffer(ChunkPayload), RawHash, RawSize); - CHECK(CompressedChunk); - CHECK_EQ(RawHash, BlockDescription.ChunkRawHashes[ChunkIndexesToFetch[RangeIndex]]); - CHECK_EQ(RawSize, BlockDescription.ChunkRawLengths[ChunkIndexesToFetch[RangeIndex]]); - } - }; - - { - // Single - std::vector ChunkIndexesToFetch{uint32_t(BlockDescription.ChunkCompressedLengths.size() / 2)}; - Validate(ChunkIndexesToFetch); - } - { - // Many - std::vector ChunkIndexesToFetch; - for (uint32_t Index = 0; Index < BlockDescription.ChunkCompressedLengths.size() / 16; Index++) - { - ChunkIndexesToFetch.push_back(uint32_t(BlockDescription.ChunkCompressedLengths.size() / 6 + Index * 7)); - ChunkIndexesToFetch.push_back(uint32_t(BlockDescription.ChunkCompressedLengths.size() / 6 + Index * 7 + 1)); - ChunkIndexesToFetch.push_back(uint32_t(BlockDescription.ChunkCompressedLengths.size() / 6 + Index * 7 + 3)); - } - Validate(ChunkIndexesToFetch); - } - - { - // First and last - std::vector ChunkIndexesToFetch{0, uint32_t(BlockDescription.ChunkCompressedLengths.size() - 1)}; - Validate(ChunkIndexesToFetch); - } -} -TEST_SUITE_END(); - -void -buildstorageoperations_forcelink() -{ -} - -#endif // ZEN_WITH_TESTS - -} // namespace zen diff --git a/src/zenremotestore/builds/buildstorageresolve.cpp b/src/zenremotestore/builds/buildstorageresolve.cpp new file mode 100644 index 000000000..b33d7af29 --- /dev/null +++ b/src/zenremotestore/builds/buildstorageresolve.cpp @@ -0,0 +1,249 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include + +namespace zen { + +namespace { + std::string ConnectionSettingsToString(const HttpClientSettings& ClientSettings) + { + ExtendableStringBuilder<128> SB; + SB << "\n LogCategory: " << ClientSettings.LogCategory; + SB << "\n ConnectTimeout: " << ClientSettings.ConnectTimeout.count() << " ms"; + SB << "\n Timeout: " << ClientSettings.Timeout.count() << " ms"; + SB << "\n AccessTokenProvider: " << ClientSettings.AccessTokenProvider.has_value(); + SB << "\n AssumeHttp2: " << ClientSettings.AssumeHttp2; + SB << "\n AllowResume: " << ClientSettings.AllowResume; + SB << "\n RetryCount: " << ClientSettings.RetryCount; + SB << "\n SessionId: " << ClientSettings.SessionId.ToString(); + SB << "\n Verbose: " << ClientSettings.Verbose; + SB << "\n MaximumInMemoryDownloadSize: " << ClientSettings.MaximumInMemoryDownloadSize; + return SB.ToString(); + } +} // namespace + +BuildStorageResolveResult +ResolveBuildStorage(LoggerRef InLog, + const HttpClientSettings& ClientSettings, + std::string_view Host, + std::string_view OverrideHost, + std::string_view ZenCacheHost, + ZenCacheResolveMode ZenResolveMode, + bool Verbose) +{ + ZEN_SCOPED_LOG(InLog); + + bool AllowZenCacheDiscovery = ZenResolveMode == ZenCacheResolveMode::Discovery || ZenResolveMode == ZenCacheResolveMode::All; + bool AllowLocalZenCache = ZenResolveMode == ZenCacheResolveMode::LocalHost || ZenResolveMode == ZenCacheResolveMode::All; + + auto GetHostNameFromUrl = [](std::string_view Url) -> std::string_view { + std::string::size_type HostnameStart = 0; + std::string::size_type HostnameLength = std::string::npos; + if (auto StartPos = Url.find("//"); StartPos != std::string::npos) + { + HostnameStart = StartPos + 2; + } + if (auto EndPos = Url.find("/", HostnameStart); EndPos != std::string::npos) + { + HostnameLength = EndPos - HostnameStart; + } + if (auto EndPos = Url.find(":", HostnameStart); EndPos != std::string::npos) + { + HostnameLength = EndPos - HostnameStart; + } + return Url.substr(HostnameStart, HostnameLength); + }; + + std::string HostUrl; + std::string HostName; + double HostLatencySec = -1.0; + uint64_t HostMaxRangeCountPerRequest = 1; + + std::string CacheUrl; + std::string CacheName; + bool HostAssumeHttp2 = ClientSettings.AssumeHttp2; + bool CacheAssumeHttp2 = ClientSettings.AssumeHttp2; + double CacheLatencySec = -1.0; + uint64_t CacheMaxRangeCountPerRequest = 1; + + JupiterServerDiscovery DiscoveryResponse; + const std::string_view DiscoveryHost = Host.empty() ? OverrideHost : Host; + + if (OverrideHost.empty() || (ZenCacheHost.empty() && AllowZenCacheDiscovery)) + { + if (Verbose) + { + ZEN_INFO("Querying servers at '{}/api/v1/status/servers'\n Connection settings:{}", + DiscoveryHost, + ConnectionSettingsToString(ClientSettings)); + } + + DiscoveryResponse = DiscoverJupiterEndpoints(DiscoveryHost, ClientSettings); + } + + if (!OverrideHost.empty()) + { + if (Verbose) + { + ZEN_INFO("Testing server endpoint at '{}/health/live'. Assume http2: {}", OverrideHost, HostAssumeHttp2); + } + if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(OverrideHost, HostAssumeHttp2, ClientSettings.Verbose); + TestResult.Success) + { + if (Verbose) + { + ZEN_INFO("Server endpoint at '{}/api/v1/status/servers' succeeded", OverrideHost); + } + HostUrl = OverrideHost; + HostName = GetHostNameFromUrl(OverrideHost); + HostLatencySec = TestResult.LatencySeconds; + HostMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; + } + else + { + throw std::runtime_error(fmt::format("Host {} could not be reached. Reason: {}", OverrideHost, TestResult.FailureReason)); + } + } + else + { + if (DiscoveryResponse.ServerEndPoints.empty()) + { + throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", DiscoveryHost)); + } + + for (const JupiterServerDiscovery::EndPoint& ServerEndpoint : DiscoveryResponse.ServerEndPoints) + { + if (!ServerEndpoint.BaseUrl.empty()) + { + if (Verbose) + { + ZEN_INFO("Testing server endpoint at '{}/health/live'. Assume http2: {}", + ServerEndpoint.BaseUrl, + ServerEndpoint.AssumeHttp2); + } + + if (JupiterEndpointTestResult TestResult = + TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2, ClientSettings.Verbose); + TestResult.Success) + { + if (Verbose) + { + ZEN_INFO("Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl); + } + + HostUrl = ServerEndpoint.BaseUrl; + HostAssumeHttp2 = ServerEndpoint.AssumeHttp2; + HostName = ServerEndpoint.Name; + HostLatencySec = TestResult.LatencySeconds; + HostMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; + break; + } + else + { + ZEN_DEBUG("Unable to reach host {}. Reason: {}", ServerEndpoint.BaseUrl, TestResult.FailureReason); + } + } + } + if (HostUrl.empty()) + { + throw std::runtime_error(fmt::format("Failed to find any usable builds hosts out of {} using {}", + DiscoveryResponse.ServerEndPoints.size(), + DiscoveryHost)); + } + } + if (ZenCacheHost.empty()) + { + if (AllowZenCacheDiscovery) + { + for (const JupiterServerDiscovery::EndPoint& CacheEndpoint : DiscoveryResponse.CacheEndPoints) + { + if (!CacheEndpoint.BaseUrl.empty()) + { + if (Verbose) + { + ZEN_INFO("Testing cache endpoint at '{}/status/builds'. Assume http2: {}", + CacheEndpoint.BaseUrl, + CacheEndpoint.AssumeHttp2); + } + + if (ZenCacheEndpointTestResult TestResult = + TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2, ClientSettings.Verbose); + TestResult.Success) + { + if (Verbose) + { + ZEN_INFO("Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl); + } + + CacheUrl = CacheEndpoint.BaseUrl; + CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2; + CacheName = CacheEndpoint.Name; + CacheLatencySec = TestResult.LatencySeconds; + CacheMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; + break; + } + } + } + } + if (CacheUrl.empty() && AllowLocalZenCache) + { + ZenServerState State; + if (State.InitializeReadOnly()) + { + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + if (CacheUrl.empty()) + { + std::string ZenServerLocalHostUrl = fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load()); + if (ZenCacheEndpointTestResult TestResult = + TestZenCacheEndpoint(ZenServerLocalHostUrl, /*AssumeHttp2*/ false, ClientSettings.Verbose); + TestResult.Success) + { + CacheUrl = ZenServerLocalHostUrl; + CacheAssumeHttp2 = false; + CacheName = "localhost"; + CacheLatencySec = TestResult.LatencySeconds; + } + } + }); + } + } + } + else + { + if (Verbose) + { + ZEN_INFO("Testing cache endpoint at '{}/status/builds'. Assume http2: {}", ZenCacheHost, false); + } + if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenCacheHost, /*AssumeHttp2*/ false, ClientSettings.Verbose); + TestResult.Success) + { + CacheUrl = ZenCacheHost; + CacheName = GetHostNameFromUrl(ZenCacheHost); + CacheLatencySec = TestResult.LatencySeconds; + CacheMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; + } + else + { + ZEN_WARN("Unable to reach cache host {}. Reason: {}", ZenCacheHost, TestResult.FailureReason); + } + } + + return BuildStorageResolveResult{ + .Cloud = {.Address = HostUrl, + .Name = HostName, + .AssumeHttp2 = HostAssumeHttp2, + .LatencySec = HostLatencySec, + .Caps = BuildStorageResolveResult::Capabilities{.MaxRangeCountPerRequest = HostMaxRangeCountPerRequest}}, + .Cache = {.Address = CacheUrl, + .Name = CacheName, + .AssumeHttp2 = CacheAssumeHttp2, + .LatencySec = CacheLatencySec, + .Caps = BuildStorageResolveResult::Capabilities{.MaxRangeCountPerRequest = CacheMaxRangeCountPerRequest}}}; +} + +} // namespace zen diff --git a/src/zenremotestore/builds/buildstorageutil.cpp b/src/zenremotestore/builds/buildstorageutil.cpp index 144964e37..7a7b1aea7 100644 --- a/src/zenremotestore/builds/buildstorageutil.cpp +++ b/src/zenremotestore/builds/buildstorageutil.cpp @@ -2,252 +2,40 @@ #include +#include +#include +#include +#include #include +#include #include +#include +#include +#include +#include #include #include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include -namespace zen { -namespace { - std::string ConnectionSettingsToString(const HttpClientSettings& ClientSettings) - { - ExtendableStringBuilder<128> SB; - SB << "\n LogCategory: " << ClientSettings.LogCategory; - SB << "\n ConnectTimeout: " << ClientSettings.ConnectTimeout.count() << " ms"; - SB << "\n Timeout: " << ClientSettings.Timeout.count() << " ms"; - SB << "\n AccessTokenProvider: " << ClientSettings.AccessTokenProvider.has_value(); - SB << "\n AssumeHttp2: " << ClientSettings.AssumeHttp2; - SB << "\n AllowResume: " << ClientSettings.AllowResume; - SB << "\n RetryCount: " << ClientSettings.RetryCount; - SB << "\n SessionId: " << ClientSettings.SessionId.ToString(); - SB << "\n Verbose: " << ClientSettings.Verbose; - SB << "\n MaximumInMemoryDownloadSize: " << ClientSettings.MaximumInMemoryDownloadSize; - return SB.ToString(); - } -} // namespace - -BuildStorageResolveResult -ResolveBuildStorage(LoggerRef InLog, - const HttpClientSettings& ClientSettings, - std::string_view Host, - std::string_view OverrideHost, - std::string_view ZenCacheHost, - ZenCacheResolveMode ZenResolveMode, - bool Verbose) -{ - ZEN_SCOPED_LOG(InLog); - - bool AllowZenCacheDiscovery = ZenResolveMode == ZenCacheResolveMode::Discovery || ZenResolveMode == ZenCacheResolveMode::All; - bool AllowLocalZenCache = ZenResolveMode == ZenCacheResolveMode::LocalHost || ZenResolveMode == ZenCacheResolveMode::All; - - auto GetHostNameFromUrl = [](std::string_view Url) -> std::string_view { - std::string::size_type HostnameStart = 0; - std::string::size_type HostnameLength = std::string::npos; - if (auto StartPos = Url.find("//"); StartPos != std::string::npos) - { - HostnameStart = StartPos + 2; - } - if (auto EndPos = Url.find("/", HostnameStart); EndPos != std::string::npos) - { - HostnameLength = EndPos - HostnameStart; - } - if (auto EndPos = Url.find(":", HostnameStart); EndPos != std::string::npos) - { - HostnameLength = EndPos - HostnameStart; - } - return Url.substr(HostnameStart, HostnameLength); - }; - - std::string HostUrl; - std::string HostName; - double HostLatencySec = -1.0; - uint64_t HostMaxRangeCountPerRequest = 1; - - std::string CacheUrl; - std::string CacheName; - bool HostAssumeHttp2 = ClientSettings.AssumeHttp2; - bool CacheAssumeHttp2 = ClientSettings.AssumeHttp2; - double CacheLatencySec = -1.0; - uint64_t CacheMaxRangeCountPerRequest = 1; - - JupiterServerDiscovery DiscoveryResponse; - const std::string_view DiscoveryHost = Host.empty() ? OverrideHost : Host; - - if (OverrideHost.empty() || (ZenCacheHost.empty() && AllowZenCacheDiscovery)) - { - if (Verbose) - { - ZEN_INFO("Querying servers at '{}/api/v1/status/servers'\n Connection settings:{}", - DiscoveryHost, - ConnectionSettingsToString(ClientSettings)); - } - - DiscoveryResponse = DiscoverJupiterEndpoints(DiscoveryHost, ClientSettings); - } - - if (!OverrideHost.empty()) - { - if (Verbose) - { - ZEN_INFO("Testing server endpoint at '{}/health/live'. Assume http2: {}", OverrideHost, HostAssumeHttp2); - } - if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(OverrideHost, HostAssumeHttp2, ClientSettings.Verbose); - TestResult.Success) - { - if (Verbose) - { - ZEN_INFO("Server endpoint at '{}/api/v1/status/servers' succeeded", OverrideHost); - } - HostUrl = OverrideHost; - HostName = GetHostNameFromUrl(OverrideHost); - HostLatencySec = TestResult.LatencySeconds; - HostMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; - } - else - { - throw std::runtime_error(fmt::format("Host {} could not be reached. Reason: {}", OverrideHost, TestResult.FailureReason)); - } - } - else - { - if (DiscoveryResponse.ServerEndPoints.empty()) - { - throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", DiscoveryHost)); - } - - for (const JupiterServerDiscovery::EndPoint& ServerEndpoint : DiscoveryResponse.ServerEndPoints) - { - if (!ServerEndpoint.BaseUrl.empty()) - { - if (Verbose) - { - ZEN_INFO("Testing server endpoint at '{}/health/live'. Assume http2: {}", - ServerEndpoint.BaseUrl, - ServerEndpoint.AssumeHttp2); - } - - if (JupiterEndpointTestResult TestResult = - TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2, ClientSettings.Verbose); - TestResult.Success) - { - if (Verbose) - { - ZEN_INFO("Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl); - } - - HostUrl = ServerEndpoint.BaseUrl; - HostAssumeHttp2 = ServerEndpoint.AssumeHttp2; - HostName = ServerEndpoint.Name; - HostLatencySec = TestResult.LatencySeconds; - HostMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; - break; - } - else - { - ZEN_DEBUG("Unable to reach host {}. Reason: {}", ServerEndpoint.BaseUrl, TestResult.FailureReason); - } - } - } - if (HostUrl.empty()) - { - throw std::runtime_error(fmt::format("Failed to find any usable builds hosts out of {} using {}", - DiscoveryResponse.ServerEndPoints.size(), - DiscoveryHost)); - } - } - if (ZenCacheHost.empty()) - { - if (AllowZenCacheDiscovery) - { - for (const JupiterServerDiscovery::EndPoint& CacheEndpoint : DiscoveryResponse.CacheEndPoints) - { - if (!CacheEndpoint.BaseUrl.empty()) - { - if (Verbose) - { - ZEN_INFO("Testing cache endpoint at '{}/status/builds'. Assume http2: {}", - CacheEndpoint.BaseUrl, - CacheEndpoint.AssumeHttp2); - } +#include - if (ZenCacheEndpointTestResult TestResult = - TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2, ClientSettings.Verbose); - TestResult.Success) - { - if (Verbose) - { - ZEN_INFO("Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl); - } +#if ZEN_WITH_TESTS +# include +# include +# include +# include +#endif // ZEN_WITH_TESTS - CacheUrl = CacheEndpoint.BaseUrl; - CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2; - CacheName = CacheEndpoint.Name; - CacheLatencySec = TestResult.LatencySeconds; - CacheMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; - break; - } - } - } - } - if (CacheUrl.empty() && AllowLocalZenCache) - { - ZenServerState State; - if (State.InitializeReadOnly()) - { - State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { - if (CacheUrl.empty()) - { - std::string ZenServerLocalHostUrl = fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load()); - if (ZenCacheEndpointTestResult TestResult = - TestZenCacheEndpoint(ZenServerLocalHostUrl, /*AssumeHttp2*/ false, ClientSettings.Verbose); - TestResult.Success) - { - CacheUrl = ZenServerLocalHostUrl; - CacheAssumeHttp2 = false; - CacheName = "localhost"; - CacheLatencySec = TestResult.LatencySeconds; - } - } - }); - } - } - } - else - { - if (Verbose) - { - ZEN_INFO("Testing cache endpoint at '{}/status/builds'. Assume http2: {}", ZenCacheHost, false); - } - if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenCacheHost, /*AssumeHttp2*/ false, ClientSettings.Verbose); - TestResult.Success) - { - CacheUrl = ZenCacheHost; - CacheName = GetHostNameFromUrl(ZenCacheHost); - CacheLatencySec = TestResult.LatencySeconds; - CacheMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; - } - else - { - ZEN_WARN("Unable to reach cache host {}. Reason: {}", ZenCacheHost, TestResult.FailureReason); - } - } +namespace zen { - return BuildStorageResolveResult{ - .Cloud = {.Address = HostUrl, - .Name = HostName, - .AssumeHttp2 = HostAssumeHttp2, - .LatencySec = HostLatencySec, - .Caps = BuildStorageResolveResult::Capabilities{.MaxRangeCountPerRequest = HostMaxRangeCountPerRequest}}, - .Cache = {.Address = CacheUrl, - .Name = CacheName, - .AssumeHttp2 = CacheAssumeHttp2, - .LatencySec = CacheLatencySec, - .Caps = BuildStorageResolveResult::Capabilities{.MaxRangeCountPerRequest = CacheMaxRangeCountPerRequest}}}; -} +using namespace std::literals; std::vector ParseBlockMetadatas(std::span BlockMetadatas) @@ -447,4 +235,1398 @@ GetBlockDescriptions(LoggerRef InLog, return Result; } +////////////////////// Shared helpers + +std::filesystem::path +ZenStateFilePath(const std::filesystem::path& ZenFolderPath) +{ + return ZenFolderPath / "current_state.cbo"; +} +std::filesystem::path +ZenTempFolderPath(const std::filesystem::path& ZenFolderPath) +{ + return ZenFolderPath / "tmp"; +} + +CbObject +GetBuild(BuildStorageBase& Storage, const Oid& BuildId, bool IsQuiet) +{ + Stopwatch GetBuildTimer; + CbObject BuildObject = Storage.GetBuild(BuildId); + if (!IsQuiet) + { + ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", + NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), + BuildObject["name"sv].AsString(), + NiceBytes(BuildObject.GetSize())); + + ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv)); + } + return BuildObject; +} + +uint64_t +GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory) +{ + return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u; +} + +void +DownloadLargeBlob(BuildStorageBase& Storage, + const std::filesystem::path& DownloadFolder, + const Oid& BuildId, + const IoHash& ChunkHash, + const std::uint64_t PreferredMultipartChunkSize, + ParallelWork& Work, + WorkerThreadPool& NetworkPool, + std::atomic& DownloadedChunkByteCount, + std::atomic& MultipartAttachmentCount, + std::function&& OnDownloadComplete) +{ + ZEN_TRACE_CPU("DownloadLargeBlob"); + + struct WorkloadData + { + TemporaryFile TempFile; + }; + std::shared_ptr Workload(std::make_shared()); + + std::error_code Ec; + Workload->TempFile.CreateTemporary(DownloadFolder, Ec); + if (Ec) + { + throw std::runtime_error( + fmt::format("Failed opening temporary file '{}', reason: ({}) {}", Workload->TempFile.GetPath(), Ec.message(), Ec.value())); + } + std::vector> WorkItems = Storage.GetLargeBuildBlob( + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + [&Work, Workload, &DownloadedChunkByteCount](uint64_t Offset, const IoBuffer& Chunk) { + DownloadedChunkByteCount += Chunk.GetSize(); + + if (!Work.IsAborted()) + { + ZEN_TRACE_CPU("Async_DownloadLargeBlob_OnReceive"); + Workload->TempFile.Write(Chunk.GetView(), Offset); + } + }, + [&Work, Workload, OnDownloadComplete = std::move(OnDownloadComplete)]() { + if (!Work.IsAborted()) + { + ZEN_TRACE_CPU("Async_DownloadLargeBlob_OnComplete"); + + uint64_t PayloadSize = Workload->TempFile.FileSize(); + void* FileHandle = Workload->TempFile.Detach(); + ZEN_ASSERT(FileHandle != nullptr); + IoBuffer Payload(IoBuffer::File, FileHandle, 0, PayloadSize, true); + Payload.SetDeleteOnClose(true); + OnDownloadComplete(std::move(Payload)); + } + }); + if (!WorkItems.empty()) + { + MultipartAttachmentCount++; + } + for (auto& WorkItem : WorkItems) + { + Work.ScheduleWork(NetworkPool, [WorkItem = std::move(WorkItem)](std::atomic& AbortFlag) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("Async_DownloadLargeBlob_Work"); + + WorkItem(); + } + }); + } +} + +CompositeBuffer +ValidateBlob(std::atomic& AbortFlag, + IoBuffer&& Payload, + const IoHash& BlobHash, + uint64_t& OutCompressedSize, + uint64_t& OutDecompressedSize) +{ + ZEN_TRACE_CPU("ValidateBlob"); + + if (Payload.GetContentType() != ZenContentType::kCompressedBinary) + { + throw std::runtime_error(fmt::format("Blob {} ({} bytes) has unexpected content type '{}'", + BlobHash, + Payload.GetSize(), + ToString(Payload.GetContentType()))); + } + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize); + if (!Compressed) + { + throw std::runtime_error(fmt::format("Blob {} ({} bytes) compressed header is invalid", BlobHash, Payload.GetSize())); + } + if (RawHash != BlobHash) + { + throw std::runtime_error( + fmt::format("Blob {} ({} bytes) compressed header has a mismatching raw hash {}", BlobHash, Payload.GetSize(), RawHash)); + } + + IoHashStream Hash; + bool CouldDecompress = Compressed.DecompressToStream( + 0, + RawSize, + [&AbortFlag, &Hash](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { + ZEN_UNUSED(SourceOffset, SourceSize, Offset); + if (!AbortFlag) + { + for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) + { + Hash.Append(Segment.GetView()); + } + return true; + } + return false; + }); + + if (AbortFlag) + { + return CompositeBuffer{}; + } + + if (!CouldDecompress) + { + throw std::runtime_error( + fmt::format("Blob {} ({} bytes) failed to decompress - header information mismatch", BlobHash, Payload.GetSize())); + } + IoHash ValidateRawHash = Hash.GetHash(); + if (ValidateRawHash != BlobHash) + { + throw std::runtime_error(fmt::format("Blob {} ({} bytes) decompressed hash {} does not match header information", + BlobHash, + Payload.GetSize(), + ValidateRawHash)); + } + OodleCompressor Compressor; + OodleCompressionLevel CompressionLevel; + uint64_t BlockSize; + if (!Compressed.TryGetCompressParameters(Compressor, CompressionLevel, BlockSize)) + { + throw std::runtime_error(fmt::format("Blob {} ({} bytes) failed to get compression details", BlobHash, Payload.GetSize())); + } + OutCompressedSize = Payload.GetSize(); + OutDecompressedSize = RawSize; + if (CompressionLevel == OodleCompressionLevel::None) + { + // Only decompress to composite if we need it for block verification + CompositeBuffer DecompressedComposite = Compressed.DecompressToComposite(); + if (!DecompressedComposite) + { + throw std::runtime_error(fmt::format("Blob {} ({} bytes) failed to decompress to composite", BlobHash, Payload.GetSize())); + } + return DecompressedComposite; + } + return CompositeBuffer{}; +} + +CompositeBuffer +ValidateBlob(std::atomic& AbortFlag, + BuildStorageBase& Storage, + const Oid& BuildId, + const IoHash& BlobHash, + uint64_t& OutCompressedSize, + uint64_t& OutDecompressedSize) +{ + ZEN_TRACE_CPU("ValidateBlob"); + IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlobHash); + if (!Payload) + { + throw std::runtime_error(fmt::format("Blob {} could not be found", BlobHash)); + } + return ValidateBlob(AbortFlag, std::move(Payload), BlobHash, OutCompressedSize, OutDecompressedSize); +} + +std::vector> +ResolveBuildPartNames(CbObjectView BuildObject, + const Oid& BuildId, + const std::vector& BuildPartIds, + std::span BuildPartNames, + std::uint64_t& OutPreferredMultipartChunkSize) +{ + std::vector> Result; + { + CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); + if (!PartsObject) + { + throw std::runtime_error("Build object does not have a 'parts' object"); + } + + OutPreferredMultipartChunkSize = BuildObject["chunkSize"sv].AsUInt64(OutPreferredMultipartChunkSize); + + std::vector> AvailableParts; + + for (CbFieldView PartView : PartsObject) + { + const std::string BuildPartName = std::string(PartView.GetName()); + const Oid BuildPartId = PartView.AsObjectId(); + if (BuildPartId == Oid::Zero) + { + ExtendableStringBuilder<128> SB; + for (CbFieldView ScanPartView : PartsObject) + { + SB.Append(fmt::format("\n {}: {}", ScanPartView.GetName(), ScanPartView.AsObjectId())); + } + throw std::runtime_error(fmt::format("Build object parts does not have a '{}' object id{}", BuildPartName, SB.ToView())); + } + AvailableParts.push_back({BuildPartId, BuildPartName}); + } + + if (BuildPartIds.empty() && BuildPartNames.empty()) + { + Result = AvailableParts; + } + else + { + for (const std::string& BuildPartName : BuildPartNames) + { + if (auto It = std::find_if(AvailableParts.begin(), + AvailableParts.end(), + [&BuildPartName](const auto& Part) { return Part.second == BuildPartName; }); + It != AvailableParts.end()) + { + Result.push_back(*It); + } + else + { + throw std::runtime_error(fmt::format("Build {} object does not have a part named '{}'", BuildId, BuildPartName)); + } + } + for (const Oid& BuildPartId : BuildPartIds) + { + if (auto It = std::find_if(AvailableParts.begin(), + AvailableParts.end(), + [&BuildPartId](const auto& Part) { return Part.first == BuildPartId; }); + It != AvailableParts.end()) + { + Result.push_back(*It); + } + else + { + throw std::runtime_error(fmt::format("Build {} object does not have a part with id '{}'", BuildId, BuildPartId)); + } + } + } + + if (Result.empty()) + { + throw std::runtime_error(fmt::format("Build object does not have any parts", BuildId)); + } + } + return Result; +} + +ChunkedFolderContent +GetRemoteContent(LoggerRef InLog, + StorageInstance& Storage, + const Oid& BuildId, + const std::vector>& BuildParts, + const BuildManifest& Manifest, + std::span IncludeWildcards, + std::span ExcludeWildcards, + std::unique_ptr& OutChunkController, + std::vector& OutPartContents, + std::vector& OutBlockDescriptions, + std::vector& OutLooseChunkHashes, + bool IsQuiet, + bool IsVerbose, + bool DoExtraContentVerify) +{ + ZEN_TRACE_CPU("GetRemoteContent"); + ZEN_SCOPED_LOG(InLog); + + Stopwatch GetBuildPartTimer; + const Oid BuildPartId = BuildParts[0].first; + const std::string_view BuildPartName = BuildParts[0].second; + CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); + if (!IsQuiet) + { + ZEN_INFO("GetBuildPart {} ('{}') took {}. Payload size: {}", + BuildPartId, + BuildPartName, + NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), + NiceBytes(BuildPartManifest.GetSize())); + ZEN_INFO("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv)); + } + + { + CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView(); + std::string_view ChunkerName = Chunker["name"sv].AsString(); + CbObjectView Parameters = Chunker["parameters"sv].AsObjectView(); + OutChunkController = CreateChunkingController(ChunkerName, Parameters); + } + + auto ParseBuildPartManifest = [&Log, IsQuiet, IsVerbose, DoExtraContentVerify](StorageInstance& Storage, + const Oid& BuildId, + const Oid& BuildPartId, + CbObject BuildPartManifest, + std::span IncludeWildcards, + std::span ExcludeWildcards, + const BuildManifest::Part* OptionalManifest, + ChunkedFolderContent& OutRemoteContent, + std::vector& OutBlockDescriptions, + std::vector& OutLooseChunkHashes) { + std::vector AbsoluteChunkOrders; + std::vector LooseChunkRawSizes; + std::vector BlockRawHashes; + + ReadBuildContentFromCompactBinary(BuildPartManifest, + OutRemoteContent.Platform, + OutRemoteContent.Paths, + OutRemoteContent.RawHashes, + OutRemoteContent.RawSizes, + OutRemoteContent.Attributes, + OutRemoteContent.ChunkedContent.SequenceRawHashes, + OutRemoteContent.ChunkedContent.ChunkCounts, + AbsoluteChunkOrders, + OutLooseChunkHashes, + LooseChunkRawSizes, + BlockRawHashes); + + // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them + + { + if (!IsQuiet) + { + ZEN_INFO("Fetching metadata for {} blocks", BlockRawHashes.size()); + } + + Stopwatch GetBlockMetadataTimer; + + bool AttemptFallback = false; + OutBlockDescriptions = GetBlockDescriptions(Log(), + *Storage.BuildStorage, + Storage.CacheStorage.get(), + BuildId, + BlockRawHashes, + AttemptFallback, + IsQuiet, + IsVerbose); + + if (!IsQuiet) + { + ZEN_INFO("GetBlockMetadata for {} took {}. Found {} blocks", + BuildPartId, + NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), + OutBlockDescriptions.size()); + } + } + + CalculateLocalChunkOrders(AbsoluteChunkOrders, + OutLooseChunkHashes, + LooseChunkRawSizes, + OutBlockDescriptions, + OutRemoteContent.ChunkedContent.ChunkHashes, + OutRemoteContent.ChunkedContent.ChunkRawSizes, + OutRemoteContent.ChunkedContent.ChunkOrders, + DoExtraContentVerify); + + std::vector DeletedPaths; + + if (OptionalManifest) + { + tsl::robin_set PathsInManifest; + PathsInManifest.reserve(OptionalManifest->Files.size()); + for (const std::filesystem::path& ManifestPath : OptionalManifest->Files) + { + PathsInManifest.insert(ToLower(ManifestPath.generic_string())); + } + for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths) + { + if (!PathsInManifest.contains(ToLower(RemotePath.generic_string()))) + { + DeletedPaths.push_back(RemotePath); + } + } + } + + if (!IncludeWildcards.empty() || !ExcludeWildcards.empty()) + { + for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths) + { + if (!IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(RemotePath.generic_string()), /*CaseSensitive*/ true)) + { + DeletedPaths.push_back(RemotePath); + } + } + } + + if (!DeletedPaths.empty()) + { + OutRemoteContent = DeletePathsFromChunkedContent(OutRemoteContent, DeletedPaths); + InlineRemoveUnusedHashes(OutLooseChunkHashes, OutRemoteContent.ChunkedContent.ChunkHashes); + } + +#if ZEN_BUILD_DEBUG + ValidateChunkedFolderContent(OutRemoteContent, OutBlockDescriptions, OutLooseChunkHashes, IncludeWildcards, ExcludeWildcards); +#endif // ZEN_BUILD_DEBUG + }; + + auto FindManifest = [&Manifest](const Oid& BuildPartId, std::string_view BuildPartName) -> const BuildManifest::Part* { + if (Manifest.Parts.empty()) + { + return nullptr; + } + if (Manifest.Parts.size() == 1) + { + if (Manifest.Parts[0].PartId == Oid::Zero && Manifest.Parts[0].PartName.empty()) + { + return &Manifest.Parts[0]; + } + } + + auto It = std::find_if(Manifest.Parts.begin(), Manifest.Parts.end(), [BuildPartId, BuildPartName](const BuildManifest::Part& Part) { + if (Part.PartId != Oid::Zero) + { + return Part.PartId == BuildPartId; + } + if (!Part.PartName.empty()) + { + return Part.PartName == BuildPartName; + } + return false; + }); + if (It != Manifest.Parts.end()) + { + return &(*It); + } + return nullptr; + }; + + OutPartContents.resize(1); + ParseBuildPartManifest(Storage, + BuildId, + BuildPartId, + BuildPartManifest, + IncludeWildcards, + ExcludeWildcards, + FindManifest(BuildPartId, BuildPartName), + OutPartContents[0], + OutBlockDescriptions, + OutLooseChunkHashes); + ChunkedFolderContent RemoteContent; + if (BuildParts.size() > 1) + { + std::vector OverlayBlockDescriptions; + std::vector OverlayLooseChunkHashes; + for (size_t PartIndex = 1; PartIndex < BuildParts.size(); PartIndex++) + { + const Oid& OverlayBuildPartId = BuildParts[PartIndex].first; + const std::string& OverlayBuildPartName = BuildParts[PartIndex].second; + Stopwatch GetOverlayBuildPartTimer; + CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId); + if (!IsQuiet) + { + ZEN_INFO("GetBuildPart {} ('{}') took {}. Payload size: {}", + OverlayBuildPartId, + OverlayBuildPartName, + NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()), + NiceBytes(OverlayBuildPartManifest.GetSize())); + } + + ChunkedFolderContent OverlayPartContent; + std::vector OverlayPartBlockDescriptions; + std::vector OverlayPartLooseChunkHashes; + + ParseBuildPartManifest(Storage, + BuildId, + OverlayBuildPartId, + OverlayBuildPartManifest, + IncludeWildcards, + ExcludeWildcards, + FindManifest(OverlayBuildPartId, OverlayBuildPartName), + OverlayPartContent, + OverlayPartBlockDescriptions, + OverlayPartLooseChunkHashes); + OutPartContents.push_back(OverlayPartContent); + OverlayBlockDescriptions.insert(OverlayBlockDescriptions.end(), + OverlayPartBlockDescriptions.begin(), + OverlayPartBlockDescriptions.end()); + OverlayLooseChunkHashes.insert(OverlayLooseChunkHashes.end(), + OverlayPartLooseChunkHashes.begin(), + OverlayPartLooseChunkHashes.end()); + } + + RemoteContent = MergeChunkedFolderContents(OutPartContents[0], std::span(OutPartContents).subspan(1)); + { + tsl::robin_set AllBlockHashes; + for (const ChunkBlockDescription& Description : OutBlockDescriptions) + { + AllBlockHashes.insert(Description.BlockHash); + } + for (const ChunkBlockDescription& Description : OverlayBlockDescriptions) + { + if (!AllBlockHashes.contains(Description.BlockHash)) + { + AllBlockHashes.insert(Description.BlockHash); + OutBlockDescriptions.push_back(Description); + } + } + } + { + tsl::robin_set AllLooseChunkHashes(OutLooseChunkHashes.begin(), OutLooseChunkHashes.end()); + for (const IoHash& OverlayLooseChunkHash : OverlayLooseChunkHashes) + { + if (!AllLooseChunkHashes.contains(OverlayLooseChunkHash)) + { + AllLooseChunkHashes.insert(OverlayLooseChunkHash); + OutLooseChunkHashes.push_back(OverlayLooseChunkHash); + } + } + } + } + else + { + RemoteContent = OutPartContents[0]; + } + return RemoteContent; +} +std::string +GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix) +{ + ExtendableStringBuilder<512> SB; + std::vector> NameStringValuePairs; + for (CbFieldView Field : Object) + { + std::string_view Name = Field.GetName(); + switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + { + case CbFieldType::String: + NameStringValuePairs.push_back({std::string(Name), std::string(Accessor.AsString())}); + break; + case CbFieldType::IntegerPositive: + NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerPositive())}); + break; + case CbFieldType::IntegerNegative: + NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerNegative())}); + break; + case CbFieldType::Float32: + { + const float Value = Accessor.AsFloat32(); + if (std::isfinite(Value)) + { + NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.9g}", Value)}); + } + else + { + NameStringValuePairs.push_back({std::string(Name), "null"}); + } + } + break; + case CbFieldType::Float64: + { + const double Value = Accessor.AsFloat64(); + if (std::isfinite(Value)) + { + NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.17g}", Value)}); + } + else + { + NameStringValuePairs.push_back({std::string(Name), "null"}); + } + } + break; + case CbFieldType::BoolFalse: + NameStringValuePairs.push_back({std::string(Name), "false"}); + break; + case CbFieldType::BoolTrue: + NameStringValuePairs.push_back({std::string(Name), "true"}); + break; + case CbFieldType::Hash: + { + NameStringValuePairs.push_back({std::string(Name), Accessor.AsHash().ToHexString()}); + } + break; + case CbFieldType::Uuid: + { + StringBuilder Builder; + Accessor.AsUuid().ToString(Builder); + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + } + break; + case CbFieldType::DateTime: + { + ExtendableStringBuilder<64> Builder; + Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + } + break; + case CbFieldType::TimeSpan: + { + ExtendableStringBuilder<64> Builder; + const TimeSpan Span(Accessor.AsTimeSpanTicks()); + if (Span.GetDays() == 0) + { + Builder << Span.ToString("%h:%m:%s.%n"); + } + else + { + Builder << Span.ToString("%d.%h:%m:%s.%n"); + } + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + break; + } + case CbFieldType::ObjectId: + NameStringValuePairs.push_back({std::string(Name), Accessor.AsObjectId().ToString()}); + break; + } + } + std::string::size_type LongestKey = 0; + for (const std::pair& KeyValue : NameStringValuePairs) + { + LongestKey = Max(KeyValue.first.length(), LongestKey); + } + for (const std::pair& KeyValue : NameStringValuePairs) + { + SB.Append(fmt::format("{}{:<{}}: {}{}", Prefix, KeyValue.first, LongestKey, KeyValue.second, Suffix)); + } + return SB.ToString(); +} + +#if ZEN_WITH_TESTS + +namespace buildstorageoperations_testutils { + struct TestState + { + TestState(const std::filesystem::path& InRootPath) + : RootPath(InRootPath) + , LogOutput(CreateStandardProgress(Log)) + , ChunkController(CreateStandardChunkingController(StandardChunkingControllerSettings{})) + , ChunkCache(CreateMemoryChunkingCache()) + , WorkerPool(2) + , NetworkPool(2) + { + } + + void Initialize() + { + StoragePath = RootPath / "storage"; + TempPath = RootPath / "temp"; + SystemRootDir = RootPath / "sysroot"; + ZenFolderPath = RootPath / ".zen"; + + CreateDirectories(TempPath); + CreateDirectories(StoragePath); + + Storage.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false); + } + + void CreateSourceData(const std::filesystem::path& Source, std::span Paths, std::span Sizes) + { + const std::filesystem::path SourcePath = RootPath / Source; + CreateDirectories(SourcePath); + for (size_t FileIndex = 0; FileIndex < Paths.size(); FileIndex++) + { + const std::string& FilePath = Paths[FileIndex]; + const uint64_t FileSize = Sizes[FileIndex]; + IoBuffer FileData = FileSize > 0 ? CreateSemiRandomBlob(FileSize) : IoBuffer{}; + WriteFile(SourcePath / FilePath, FileData); + } + } + + std::vector> Upload(const Oid& BuildId, + const Oid& BuildPartId, + const std::string_view BuildPartName, + const std::filesystem::path& Source, + const std::filesystem::path& ManifestPath) + { + const std::filesystem::path SourcePath = RootPath / Source; + CbObject MetaData; + BuildsOperationUploadFolder Upload(Log, + *LogOutput, + Storage, + AbortFlag, + PauseFlag, + WorkerPool, + NetworkPool, + BuildId, + SourcePath, + true, + MetaData, + BuildsOperationUploadFolder::Options{.TempDir = TempPath}); + return Upload.Execute(BuildPartId, BuildPartName, ManifestPath, *ChunkController, *ChunkCache); + } + + void ValidateUpload(const Oid& BuildId, const std::vector>& Parts) + { + for (auto Part : Parts) + { + BuildsOperationValidateBuildPart Validate(Log, + *LogOutput, + *Storage.BuildStorage, + AbortFlag, + PauseFlag, + WorkerPool, + NetworkPool, + BuildId, + Part.first, + Part.second, + BuildsOperationValidateBuildPart::Options{}); + Validate.Execute(); + } + } + + FolderContent Download(const Oid& BuildId, + const Oid& BuildPartId, + const std::string_view BuildPartName, + const std::filesystem::path& Target, + bool Append) + { + const std::filesystem::path TargetPath = RootPath / Target; + + CreateDirectories(TargetPath); + + uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; + CbObject BuildObject = Storage.BuildStorage->GetBuild(BuildId); + std::vector PartIds; + if (BuildPartId != Oid::Zero) + { + PartIds.push_back(BuildPartId); + } + std::vector PartNames; + if (!BuildPartName.empty()) + { + PartNames.push_back(std::string(BuildPartName)); + } + std::vector> AllBuildParts = + ResolveBuildPartNames(BuildObject, BuildId, PartIds, PartNames, PreferredMultipartChunkSize); + + std::vector PartContents; + + std::vector BlockDescriptions; + std::vector LooseChunkHashes; + + ChunkedFolderContent RemoteContent = GetRemoteContent(Log, + Storage, + BuildId, + AllBuildParts, + {}, + {}, + {}, + ChunkController, + PartContents, + BlockDescriptions, + LooseChunkHashes, + /*IsQuiet*/ false, + /*IsVerbose*/ false, + /*DoExtraContentVerify*/ true); + + GetFolderContentStatistics LocalFolderScanStats; + + struct ContentVisitor : public GetDirectoryContentVisitor + { + virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) + { + RwLock::ExclusiveLockScope _(ExistingPathsLock); + for (const std::filesystem::path& FileName : Content.FileNames) + { + if (RelativeRoot.empty()) + { + ExistingPaths.push_back(FileName); + } + else + { + ExistingPaths.push_back(RelativeRoot / FileName); + } + } + } + + RwLock ExistingPathsLock; + std::vector ExistingPaths; + } Visitor; + + Latch PendingWorkCount(1); + + GetDirectoryContent(TargetPath, + DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive, + Visitor, + WorkerPool, + PendingWorkCount); + + PendingWorkCount.CountDown(); + PendingWorkCount.Wait(); + + FolderContent CurrentLocalFolderState = GetValidFolderContent( + WorkerPool, + LocalFolderScanStats, + TargetPath, + Visitor.ExistingPaths, + [](uint64_t PathCount, uint64_t CompletedPathCount) { ZEN_UNUSED(PathCount, CompletedPathCount); }, + 1000, + AbortFlag, + PauseFlag); + + ChunkingStatistics LocalChunkingStats; + ChunkedFolderContent LocalContent = ChunkFolderContent( + LocalChunkingStats, + WorkerPool, + TargetPath, + CurrentLocalFolderState, + *ChunkController, + *ChunkCache, + 1000, + [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { ZEN_UNUSED(IsAborted, IsPaused); }, + AbortFlag, + PauseFlag); + + if (Append) + { + RemoteContent = ApplyChunkedContentOverlay(LocalContent, RemoteContent, {}, {}); + } + + const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent); + const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); + + BuildsOperationUpdateFolder Download(Log, + *LogOutput, + Storage, + AbortFlag, + PauseFlag, + WorkerPool, + NetworkPool, + BuildId, + TargetPath, + LocalContent, + LocalLookup, + RemoteContent, + RemoteLookup, + BlockDescriptions, + LooseChunkHashes, + BuildsOperationUpdateFolder::Options{.SystemRootDir = SystemRootDir, + .ZenFolderPath = ZenFolderPath, + .ValidateCompletedSequences = true}); + FolderContent ResultingState; + Download.Execute(ResultingState); + + return ResultingState; + } + + void ValidateDownload(std::span Paths, + std::span Sizes, + const std::filesystem::path& Source, + const std::filesystem::path& Target, + const FolderContent& DownloadContent) + { + const std::filesystem::path SourcePath = RootPath / Source; + const std::filesystem::path TargetPath = RootPath / Target; + + CHECK_EQ(Paths.size(), DownloadContent.Paths.size()); + tsl::robin_map ExpectedSizes; + tsl::robin_map ExpectedHashes; + for (size_t Index = 0; Index < Paths.size(); Index++) + { + const std::string LookupString = std::filesystem::path(Paths[Index]).generic_string(); + ExpectedSizes.insert_or_assign(LookupString, Sizes[Index]); + std::filesystem::path FilePath = SourcePath / Paths[Index]; + const IoHash SourceHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(FilePath.make_preferred())); + ExpectedHashes.insert_or_assign(LookupString, SourceHash); + } + for (size_t Index = 0; Index < DownloadContent.Paths.size(); Index++) + { + const std::string LookupString = std::filesystem::path(DownloadContent.Paths[Index]).generic_string(); + auto SizeIt = ExpectedSizes.find(LookupString); + CHECK_NE(SizeIt, ExpectedSizes.end()); + CHECK_EQ(SizeIt->second, DownloadContent.RawSizes[Index]); + std::filesystem::path FilePath = TargetPath / DownloadContent.Paths[Index]; + const IoHash DownloadedHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(FilePath.make_preferred())); + auto HashIt = ExpectedHashes.find(LookupString); + CHECK_NE(HashIt, ExpectedHashes.end()); + CHECK_EQ(HashIt->second, DownloadedHash); + } + } + + const std::filesystem::path RootPath; + std::filesystem::path StoragePath; + std::filesystem::path TempPath; + std::filesystem::path SystemRootDir; + std::filesystem::path ZenFolderPath; + + LoggerRef Log = ConsoleLog(); + std::unique_ptr LogOutput; + + std::unique_ptr ChunkController; + std::unique_ptr ChunkCache; + + StorageInstance Storage; + BuildStorageBase::Statistics StorageStats; + + WorkerThreadPool WorkerPool; + WorkerThreadPool NetworkPool; + + std::atomic AbortFlag; + std::atomic PauseFlag; + }; + +} // namespace buildstorageoperations_testutils + +TEST_SUITE_BEGIN("remotestore.buildstorageutil"); + +TEST_CASE("buildstorageoperations.upload.folder") +{ + using namespace buildstorageoperations_testutils; + + FastRandom BaseRandom; + + const size_t FileCount = 11; + + const std::string Paths[FileCount] = {{"file_1"}, + {"file_2.exe"}, + {"file_3.txt"}, + {"dir_1/dir1_file_1.exe"}, + {"dir_1/dir1_file_2.pdb"}, + {"dir_1/dir1_file_3.txt"}, + {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"}, + {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"}, + {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"}, + {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"}, + {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}}; + const uint64_t Sizes[FileCount] = + {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u}; + + ScopedTemporaryDirectory SourceFolder; + TestState State(SourceFolder.Path()); + State.Initialize(); + State.CreateSourceData("source", Paths, Sizes); + + const Oid BuildId = Oid::NewOid(); + const Oid BuildPartId = Oid::NewOid(); + const std::string BuildPartName = "default"; + + auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", {}); + + CHECK_EQ(Result.size(), 1u); + CHECK_EQ(Result[0].first, BuildPartId); + CHECK_EQ(Result[0].second, BuildPartName); + State.ValidateUpload(BuildId, Result); + + FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false); + CHECK_EQ(DownloadContent.Paths.size(), FileCount); + State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent); +} + +TEST_CASE("buildstorageoperations.upload.manifest") +{ + using namespace buildstorageoperations_testutils; + + FastRandom BaseRandom; + + const size_t FileCount = 11; + + const std::string Paths[FileCount] = {{"file_1"}, + {"file_2.exe"}, + {"file_3.txt"}, + {"dir_1/dir1_file_1.exe"}, + {"dir_1/dir1_file_2.pdb"}, + {"dir_1/dir1_file_3.txt"}, + {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"}, + {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"}, + {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"}, + {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"}, + {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}}; + const uint64_t Sizes[FileCount] = + {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u}; + + ScopedTemporaryDirectory SourceFolder; + TestState State(SourceFolder.Path()); + State.Initialize(); + State.CreateSourceData("source", Paths, Sizes); + + std::span ManifestFiles(Paths); + ManifestFiles = ManifestFiles.subspan(0, FileCount / 2); + + std::span ManifestSizes(Sizes); + ManifestSizes = ManifestSizes.subspan(0, FileCount / 2); + + ExtendableStringBuilder<1024> Manifest; + for (const std::string& FilePath : ManifestFiles) + { + Manifest << FilePath << "\n"; + } + + WriteFile(State.RootPath / "manifest.txt", IoBuffer(IoBuffer::Wrap, Manifest.Data(), Manifest.Size())); + + const Oid BuildId = Oid::NewOid(); + const Oid BuildPartId = Oid::NewOid(); + const std::string BuildPartName = "default"; + + auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", State.RootPath / "manifest.txt"); + + CHECK_EQ(Result.size(), 1u); + CHECK_EQ(Result[0].first, BuildPartId); + CHECK_EQ(Result[0].second, BuildPartName); + State.ValidateUpload(BuildId, Result); + + FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false); + State.ValidateDownload(ManifestFiles, ManifestSizes, "source", "download", DownloadContent); +} + +TEST_CASE("buildstorageoperations.memorychunkingcache") +{ + using namespace buildstorageoperations_testutils; + + FastRandom BaseRandom; + + const size_t FileCount = 11; + + const std::string Paths[FileCount] = {{"file_1"}, + {"file_2.exe"}, + {"file_3.txt"}, + {"dir_1/dir1_file_1.exe"}, + {"dir_1/dir1_file_2.pdb"}, + {"dir_1/dir1_file_3.txt"}, + {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"}, + {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"}, + {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"}, + {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"}, + {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}}; + const uint64_t Sizes[FileCount] = + {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u}; + + ScopedTemporaryDirectory SourceFolder; + TestState State(SourceFolder.Path()); + State.Initialize(); + State.CreateSourceData("source", Paths, Sizes); + + const Oid BuildId = Oid::NewOid(); + const Oid BuildPartId = Oid::NewOid(); + const std::string BuildPartName = "default"; + + { + const std::filesystem::path SourcePath = SourceFolder.Path() / "source"; + CbObject MetaData; + BuildsOperationUploadFolder Upload(State.Log, + *State.LogOutput, + State.Storage, + State.AbortFlag, + State.PauseFlag, + State.WorkerPool, + State.NetworkPool, + BuildId, + SourcePath, + true, + MetaData, + BuildsOperationUploadFolder::Options{.TempDir = State.TempPath}); + auto Result = Upload.Execute(BuildPartId, BuildPartName, {}, *State.ChunkController, *State.ChunkCache); + + CHECK_EQ(Upload.m_ChunkingStats.FilesStoredInCache.load(), FileCount - 1); // Zero size files are not stored in cache + CHECK_EQ(Upload.m_ChunkingStats.BytesStoredInCache.load(), std::accumulate(&Sizes[0], &Sizes[FileCount], uint64_t(0))); + CHECK(Upload.m_ChunkingStats.ChunksStoredInCache.load() >= FileCount - 1); // Zero size files are not stored in cache + + CHECK_EQ(Result.size(), 1u); + CHECK_EQ(Result[0].first, BuildPartId); + CHECK_EQ(Result[0].second, BuildPartName); + } + + auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", {}); + + const Oid BuildId2 = Oid::NewOid(); + const Oid BuildPartId2 = Oid::NewOid(); + + { + const std::filesystem::path SourcePath = SourceFolder.Path() / "source"; + CbObject MetaData; + BuildsOperationUploadFolder Upload(State.Log, + *State.LogOutput, + State.Storage, + State.AbortFlag, + State.PauseFlag, + State.WorkerPool, + State.NetworkPool, + BuildId2, + SourcePath, + true, + MetaData, + BuildsOperationUploadFolder::Options{.TempDir = State.TempPath}); + Upload.Execute(BuildPartId2, BuildPartName, {}, *State.ChunkController, *State.ChunkCache); + + CHECK_EQ(Upload.m_ChunkingStats.FilesFoundInCache.load(), FileCount - 1); // Zero size files are not stored in cache + CHECK_EQ(Upload.m_ChunkingStats.BytesFoundInCache.load(), std::accumulate(&Sizes[0], &Sizes[FileCount], uint64_t(0))); + CHECK(Upload.m_ChunkingStats.ChunksFoundInCache.load() >= FileCount - 1); // Zero size files are not stored in cache + } + + FolderContent DownloadContent = State.Download(BuildId2, BuildPartId2, {}, "download", /* Append */ false); + State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent); +} + +TEST_CASE("buildstorageoperations.upload.multipart") +{ + // Disabled since it relies on authentication and specific block being present in cloud storage + if (false) + { + using namespace buildstorageoperations_testutils; + + FastRandom BaseRandom; + + const size_t FileCount = 11; + + const std::string Paths[FileCount] = {{"file_1"}, + {"file_2.exe"}, + {"file_3.txt"}, + {"dir_1/dir1_file_1.exe"}, + {"dir_1/dir1_file_2.pdb"}, + {"dir_1/dir1_file_3.txt"}, + {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"}, + {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"}, + {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"}, + {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"}, + {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}}; + const uint64_t Sizes[FileCount] = + {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u}; + + ScopedTemporaryDirectory SourceFolder; + TestState State(SourceFolder.Path()); + State.Initialize(); + State.CreateSourceData("source", Paths, Sizes); + + std::span ManifestFiles1(Paths); + ManifestFiles1 = ManifestFiles1.subspan(0, FileCount / 2); + + std::span ManifestSizes1(Sizes); + ManifestSizes1 = ManifestSizes1.subspan(0, FileCount / 2); + + std::span ManifestFiles2(Paths); + ManifestFiles2 = ManifestFiles2.subspan(FileCount / 2 - 1); + + std::span ManifestSizes2(Sizes); + ManifestSizes2 = ManifestSizes2.subspan(FileCount / 2 - 1); + + const Oid BuildPart1Id = Oid::NewOid(); + const std::string BuildPart1Name = "part1"; + const Oid BuildPart2Id = Oid::NewOid(); + const std::string BuildPart2Name = "part2"; + { + CbObjectWriter Writer; + Writer.BeginObject("parts"sv); + { + Writer.BeginObject(BuildPart1Name); + { + Writer.AddObjectId("partId"sv, BuildPart1Id); + Writer.BeginArray("files"sv); + for (const std::string& ManifestFile : ManifestFiles1) + { + Writer.AddString(ManifestFile); + } + Writer.EndArray(); // files + } + Writer.EndObject(); // part1 + + Writer.BeginObject(BuildPart2Name); + { + Writer.AddObjectId("partId"sv, BuildPart2Id); + Writer.BeginArray("files"sv); + for (const std::string& ManifestFile : ManifestFiles2) + { + Writer.AddString(ManifestFile); + } + Writer.EndArray(); // files + } + Writer.EndObject(); // part2 + } + Writer.EndObject(); // parts + + ExtendableStringBuilder<1024> Manifest; + CompactBinaryToJson(Writer.Save(), Manifest); + WriteFile(State.RootPath / "manifest.json", IoBuffer(IoBuffer::Wrap, Manifest.Data(), Manifest.Size())); + } + + const Oid BuildId = Oid::NewOid(); + + auto Result = State.Upload(BuildId, {}, {}, "source", State.RootPath / "manifest.json"); + + CHECK_EQ(Result.size(), 2u); + CHECK_EQ(Result[0].first, BuildPart1Id); + CHECK_EQ(Result[0].second, BuildPart1Name); + CHECK_EQ(Result[1].first, BuildPart2Id); + CHECK_EQ(Result[1].second, BuildPart2Name); + State.ValidateUpload(BuildId, Result); + + FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false); + State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent); + + FolderContent Part1DownloadContent = State.Download(BuildId, BuildPart1Id, {}, "download_part1", /* Append */ false); + State.ValidateDownload(ManifestFiles1, ManifestSizes1, "source", "download_part1", Part1DownloadContent); + + FolderContent Part2DownloadContent = State.Download(BuildId, Oid::Zero, BuildPart2Name, "download_part2", /* Append */ false); + State.ValidateDownload(ManifestFiles2, ManifestSizes2, "source", "download_part2", Part2DownloadContent); + + (void)State.Download(BuildId, BuildPart1Id, BuildPart1Name, "download_part1+2", /* Append */ false); + FolderContent Part1And2DownloadContent = State.Download(BuildId, BuildPart2Id, {}, "download_part1+2", /* Append */ true); + State.ValidateDownload(Paths, Sizes, "source", "download_part1+2", Part1And2DownloadContent); + } +} + +TEST_CASE("buildstorageoperations.partial.block.download" * doctest::skip(true)) +{ + const std::string OidcExecutableName = "OidcToken" ZEN_EXE_SUFFIX_LITERAL; + std::filesystem::path OidcTokenExePath = (GetRunningExecutablePath().parent_path() / OidcExecutableName).make_preferred(); + + HttpClientSettings ClientSettings{ + .LogCategory = "httpbuildsclient", + .AccessTokenProvider = + httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, "https://jupiter.devtools.epicgames.com", true, false, false), + .AssumeHttp2 = false, + .AllowResume = true, + .RetryCount = 0, + .Verbose = false}; + + HttpClient HttpClient("https://euc.jupiter.devtools.epicgames.com", ClientSettings); + + const std::string_view Namespace = "fortnite.oplog"; + const std::string_view Bucket = "fortnitegame.staged-build.fortnite-main.ps4-client"; + const Oid BuildId = Oid::FromHexString("09a76ea92ad301d4724fafad"); + + { + HttpClient::Response Response = HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}", Namespace, Bucket, BuildId), + HttpClient::Accept(ZenContentType::kCbObject)); + CbValidateError ValidateResult = CbValidateError::None; + CbObject Object = ValidateAndReadCompactBinaryObject(IoBuffer(Response.ResponsePayload), ValidateResult); + REQUIRE(ValidateResult == CbValidateError::None); + } + + std::vector BlockDescriptions; + { + CbObjectWriter Request; + + Request.BeginArray("blocks"sv); + { + Request.AddHash(IoHash::FromHexString("7c353ed782675a5e8f968e61e51fc797ecdc2882")); + } + Request.EndArray(); + + IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + + HttpClient::Response BlockDescriptionsResponse = + HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/blocks/getBlockMetadata", Namespace, Bucket, BuildId), + Payload, + HttpClient::Accept(ZenContentType::kCbObject)); + REQUIRE(BlockDescriptionsResponse.IsSuccess()); + + CbValidateError ValidateResult = CbValidateError::None; + CbObject Object = ValidateAndReadCompactBinaryObject(IoBuffer(BlockDescriptionsResponse.ResponsePayload), ValidateResult); + REQUIRE(ValidateResult == CbValidateError::None); + + { + CbArrayView BlocksArray = Object["blocks"sv].AsArrayView(); + for (CbFieldView Block : BlocksArray) + { + ChunkBlockDescription Description = ParseChunkBlockDescription(Block.AsObjectView()); + BlockDescriptions.emplace_back(std::move(Description)); + } + } + } + + REQUIRE(!BlockDescriptions.empty()); + + const IoHash BlockHash = BlockDescriptions.back().BlockHash; + + const ChunkBlockDescription& BlockDescription = BlockDescriptions.front(); + REQUIRE(!BlockDescription.ChunkRawHashes.empty()); + REQUIRE(!BlockDescription.ChunkCompressedLengths.empty()); + + std::vector> ChunkOffsetAndSizes; + uint64_t Offset = gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + + for (uint32_t ChunkCompressedSize : BlockDescription.ChunkCompressedLengths) + { + ChunkOffsetAndSizes.push_back(std::make_pair(Offset, ChunkCompressedSize)); + Offset += ChunkCompressedSize; + } + + ScopedTemporaryDirectory SourceFolder; + + auto Validate = [&](std::span ChunkIndexesToFetch) { + std::vector> Ranges; + for (uint32_t ChunkIndex : ChunkIndexesToFetch) + { + Ranges.push_back(ChunkOffsetAndSizes[ChunkIndex]); + } + + HttpClient::KeyValueMap Headers; + if (!Ranges.empty()) + { + ExtendableStringBuilder<512> SB; + for (const std::pair& R : Ranges) + { + if (SB.Size() > 0) + { + SB << ", "; + } + SB << R.first << "-" << R.first + R.second - 1; + } + Headers.Entries.insert({"Range", fmt::format("bytes={}", SB.ToView())}); + } + + HttpClient::Response GetBlobRangesResponse = HttpClient.Download( + fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect=false", Namespace, Bucket, BuildId, BlockHash), + SourceFolder.Path(), + Headers); + + REQUIRE(GetBlobRangesResponse.IsSuccess()); + [[maybe_unused]] MemoryView RangesMemoryView = GetBlobRangesResponse.ResponsePayload.GetView(); + + std::vector> PayloadRanges = GetBlobRangesResponse.GetRanges(Ranges); + if (PayloadRanges.empty()) + { + // We got the whole blob, use the ranges as is + PayloadRanges = Ranges; + } + + REQUIRE(PayloadRanges.size() == Ranges.size()); + + for (uint32_t RangeIndex = 0; RangeIndex < PayloadRanges.size(); RangeIndex++) + { + const std::pair& PayloadRange = PayloadRanges[RangeIndex]; + + CHECK_EQ(PayloadRange.second, Ranges[RangeIndex].second); + + IoBuffer ChunkPayload(GetBlobRangesResponse.ResponsePayload, PayloadRange.first, PayloadRange.second); + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed(SharedBuffer(ChunkPayload), RawHash, RawSize); + CHECK(CompressedChunk); + CHECK_EQ(RawHash, BlockDescription.ChunkRawHashes[ChunkIndexesToFetch[RangeIndex]]); + CHECK_EQ(RawSize, BlockDescription.ChunkRawLengths[ChunkIndexesToFetch[RangeIndex]]); + } + }; + + { + // Single + std::vector ChunkIndexesToFetch{uint32_t(BlockDescription.ChunkCompressedLengths.size() / 2)}; + Validate(ChunkIndexesToFetch); + } + { + // Many + std::vector ChunkIndexesToFetch; + for (uint32_t Index = 0; Index < BlockDescription.ChunkCompressedLengths.size() / 16; Index++) + { + ChunkIndexesToFetch.push_back(uint32_t(BlockDescription.ChunkCompressedLengths.size() / 6 + Index * 7)); + ChunkIndexesToFetch.push_back(uint32_t(BlockDescription.ChunkCompressedLengths.size() / 6 + Index * 7 + 1)); + ChunkIndexesToFetch.push_back(uint32_t(BlockDescription.ChunkCompressedLengths.size() / 6 + Index * 7 + 3)); + } + Validate(ChunkIndexesToFetch); + } + + { + // First and last + std::vector ChunkIndexesToFetch{0, uint32_t(BlockDescription.ChunkCompressedLengths.size() - 1)}; + Validate(ChunkIndexesToFetch); + } +} +TEST_SUITE_END(); + +void +buildstorageutil_forcelink() +{ +} + +#endif // ZEN_WITH_TESTS + } // namespace zen diff --git a/src/zenremotestore/builds/buildupdatefolder.cpp b/src/zenremotestore/builds/buildupdatefolder.cpp new file mode 100644 index 000000000..e1a9a553d --- /dev/null +++ b/src/zenremotestore/builds/buildupdatefolder.cpp @@ -0,0 +1,4947 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +using namespace std::literals; + +namespace { + std::filesystem::path ZenTempCacheFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "cache"; // Decompressed and verified data - chunks & sequences + } + std::filesystem::path ZenTempBlockFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "blocks"; // Temp storage for whole and partial blocks + } + std::filesystem::path ZenTempDownloadFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "download"; // Temp storage for decompressed and validated chunks + } + std::filesystem::path GetTempChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash) + { + return CacheFolderPath / (RawHash.ToHexString() + ".tmp"); + } + + std::filesystem::path GetFinalChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash) + { + return CacheFolderPath / RawHash.ToHexString(); + } + bool CleanDirectory(LoggerRef InLog, + ProgressBase& Progress, + WorkerThreadPool& IOWorkerPool, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + bool IsQuiet, + const std::filesystem::path& Path, + std::span ExcludeDirectories) + { + ZEN_TRACE_CPU("CleanDirectory"); + ZEN_SCOPED_LOG(InLog); + Stopwatch Timer; + + std::unique_ptr ProgressBar = Progress.CreateProgressBar("Clean Folder"); + + CleanDirectoryResult Result = CleanDirectory( + IOWorkerPool, + AbortFlag, + PauseFlag, + Path, + ExcludeDirectories, + [&](const std::string_view Details, uint64_t TotalCount, uint64_t RemainingCount, bool IsPaused, bool IsAborted) { + ProgressBar->UpdateState({.Task = "Cleaning folder ", + .Details = std::string(Details), + .TotalCount = TotalCount, + .RemainingCount = RemainingCount, + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); + }, + Progress.GetProgressUpdateDelayMS()); + + ProgressBar->Finish(); + + if (AbortFlag) + { + return false; + } + + uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); + + if (!Result.FailedRemovePaths.empty()) + { + ExtendableStringBuilder<512> SB; + for (size_t FailedPathIndex = 0; FailedPathIndex < Result.FailedRemovePaths.size(); FailedPathIndex++) + { + SB << fmt::format("\n '{}': ({}) {}", + Result.FailedRemovePaths[FailedPathIndex].first, + Result.FailedRemovePaths[FailedPathIndex].second.value(), + Result.FailedRemovePaths[FailedPathIndex].second.message()); + } + ZEN_WARN("Clean failed to remove files from '{}': {}", Path, SB.ToView()); + } + + if (ElapsedTimeMs >= 200 && !IsQuiet) + { + ZEN_INFO("Wiped folder '{}' {} ({}) in {}", + Path, + Result.FoundCount, + NiceBytes(Result.DeletedByteCount), + NiceTimeSpanMs(ElapsedTimeMs)); + } + + return Result.FailedRemovePaths.empty(); + } + uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes) + { +#if ZEN_PLATFORM_WINDOWS + if (SourcePlatform == SourcePlatform::Windows) + { + SetFileAttributesToPath(FilePath, Attributes); + return Attributes; + } + else + { + uint32_t CurrentAttributes = GetFileAttributesFromPath(FilePath); + uint32_t NewAttributes = zen::MakeFileAttributeReadOnly(CurrentAttributes, zen::IsFileModeReadOnly(Attributes)); + if (CurrentAttributes != NewAttributes) + { + SetFileAttributesToPath(FilePath, NewAttributes); + } + return NewAttributes; + } +#endif // ZEN_PLATFORM_WINDOWS +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + if (SourcePlatform != SourcePlatform::Windows) + { + zen::SetFileMode(FilePath, Attributes); + return Attributes; + } + else + { + uint32_t CurrentMode = zen::GetFileMode(FilePath); + uint32_t NewMode = zen::MakeFileModeReadOnly(CurrentMode, zen::IsFileAttributeReadOnly(Attributes)); + if (CurrentMode != NewMode) + { + zen::SetFileMode(FilePath, NewMode); + } + return NewMode; + } +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + }; + + uint32_t GetNativeFileAttributes(const std::filesystem::path FilePath) + { +#if ZEN_PLATFORM_WINDOWS + return GetFileAttributesFromPath(FilePath); +#endif // ZEN_PLATFORM_WINDOWS +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + return GetFileMode(FilePath); +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + } + std::filesystem::path TryMoveDownloadedChunk(IoBuffer& BlockBuffer, const std::filesystem::path& Path, bool ForceDiskBased) + { + uint64_t BlockSize = BlockBuffer.GetSize(); + IoBufferFileReference FileRef; + if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && (FileRef.FileChunkSize == BlockSize)) + { + ZEN_TRACE_CPU("MoveTempFullBlock"); + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) + { + BlockBuffer.SetDeleteOnClose(false); + BlockBuffer = {}; + RenameFile(TempBlobPath, Path, Ec); + if (Ec) + { + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); + BlockBuffer.SetDeleteOnClose(true); + } + else + { + return Path; + } + } + } + + if (ForceDiskBased) + { + // Could not be moved and rather large, lets store it on disk + ZEN_TRACE_CPU("WriteTempFullBlock"); + TemporaryFile::SafeWriteFile(Path, BlockBuffer); + BlockBuffer = {}; + return Path; + } + + return {}; + } + bool IsSingleFileChunk(const ChunkedFolderContent& RemoteContent, + const std::vector Locations) + { + if (Locations.size() == 1) + { + const uint32_t FirstSequenceIndex = Locations[0]->SequenceIndex; + if (RemoteContent.ChunkedContent.ChunkCounts[FirstSequenceIndex] == 1) + { + ZEN_ASSERT_SLOW(Locations[0]->Offset == 0); + return true; + } + } + return false; + } + IoBuffer MakeBufferMemoryBased(const CompositeBuffer& PartialBlockBuffer) + { + ZEN_TRACE_CPU("MakeBufferMemoryBased"); + IoBuffer BlockMemoryBuffer; + std::span Segments = PartialBlockBuffer.GetSegments(); + if (Segments.size() == 1) + { + IoBufferFileReference FileRef = {}; + if (PartialBlockBuffer.GetSegments().front().AsIoBuffer().GetFileReference(FileRef)) + { + BlockMemoryBuffer = UniqueBuffer::Alloc(FileRef.FileChunkSize).MoveToShared().AsIoBuffer(); + BasicFile Reader; + Reader.Attach(FileRef.FileHandle); + auto _ = MakeGuard([&Reader]() { Reader.Detach(); }); + MutableMemoryView ReadMem = BlockMemoryBuffer.GetMutableView(); + Reader.Read(ReadMem.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset); + return BlockMemoryBuffer; + } + else + { + return PartialBlockBuffer.GetSegments().front().AsIoBuffer(); + } + } + else + { + // Not a homogenous memory buffer, read all to memory + + BlockMemoryBuffer = UniqueBuffer::Alloc(PartialBlockBuffer.GetSize()).MoveToShared().AsIoBuffer(); + MutableMemoryView ReadMem = BlockMemoryBuffer.GetMutableView(); + for (const SharedBuffer& Segment : Segments) + { + IoBufferFileReference FileRef = {}; + if (Segment.AsIoBuffer().GetFileReference(FileRef)) + { + BasicFile Reader; + Reader.Attach(FileRef.FileHandle); + auto _ = MakeGuard([&Reader]() { Reader.Detach(); }); + Reader.Read(ReadMem.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset); + ReadMem = ReadMem.Mid(FileRef.FileChunkSize); + } + else + { + ReadMem = ReadMem.CopyFrom(Segment.AsIoBuffer().GetView()); + } + } + return BlockMemoryBuffer; + } + } + + FolderContent CheckFolderFiles(ProgressBase& Progress, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + std::string_view ProgressLabel, + TransferThreadWorkers& Workers, + GetFolderContentStatistics& LocalFolderScanStats, + const std::filesystem::path& Path, + std::span PathsToCheck) + { + std::unique_ptr ProgressBar = Progress.CreateProgressBar(ProgressLabel); + FolderContent Result = GetValidFolderContent( + Workers.GetIOWorkerPool(), + LocalFolderScanStats, + Path, + PathsToCheck, + [&ProgressBar, &LocalFolderScanStats, &AbortFlag, &PauseFlag](uint64_t PathCount, uint64_t CompletedPathCount) { + std::string Details = + fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load()); + ProgressBar->UpdateState({.Task = "Checking files ", + .Details = Details, + .TotalCount = PathCount, + .RemainingCount = PathCount - CompletedPathCount, + .Status = ProgressBase::ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, + false); + }, + Progress.GetProgressUpdateDelayMS(), + AbortFlag, + PauseFlag); + ProgressBar->Finish(); + return Result; + } + + ChunkedFolderContent ScanFolderFiles(ProgressBase& Progress, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + std::string_view ProgressLabel, + TransferThreadWorkers& Workers, + const std::filesystem::path& Path, + const FolderContent& FolderSource, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + ChunkingStatistics& OutChunkingStats) + { + uint64_t ByteCountToScan = 0; + for (const uint64_t RawSize : FolderSource.RawSizes) + { + ByteCountToScan += RawSize; + } + std::unique_ptr ProgressBar = Progress.CreateProgressBar(ProgressLabel); + FilteredRate FilteredBytesHashed; + FilteredBytesHashed.Start(); + ChunkingStatistics LocalChunkingStats; + ChunkedFolderContent Result = ChunkFolderContent( + LocalChunkingStats, + Workers.GetIOWorkerPool(), + Path, + FolderSource, + ChunkController, + ChunkCache, + Progress.GetProgressUpdateDelayMS(), + [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { + FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load()); + std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", + LocalChunkingStats.FilesProcessed.load(), + FolderSource.Paths.size(), + NiceBytes(LocalChunkingStats.BytesHashed.load()), + NiceBytes(ByteCountToScan), + NiceNum(FilteredBytesHashed.GetCurrent()), + LocalChunkingStats.UniqueChunksFound.load(), + NiceBytes(LocalChunkingStats.UniqueBytesFound.load())); + ProgressBar->UpdateState({.Task = "Scanning files ", + .Details = Details, + .TotalCount = ByteCountToScan, + .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(), + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); + }, + AbortFlag, + PauseFlag); + OutChunkingStats += LocalChunkingStats; + FilteredBytesHashed.Stop(); + ProgressBar->Finish(); + return Result; + } +} // namespace + +BuildsOperationUpdateFolder::BuildsOperationUpdateFolder(LoggerRef Log, + ProgressBase& Progress, + StorageInstance& Storage, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + WorkerThreadPool& IOWorkerPool, + WorkerThreadPool& NetworkPool, + const Oid& BuildId, + const std::filesystem::path& Path, + const ChunkedFolderContent& LocalContent, + const ChunkedContentLookup& LocalLookup, + const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& RemoteLookup, + const std::vector& BlockDescriptions, + const std::vector& LooseChunkHashes, + const Options& Options) +: m_Log(Log) +, m_Progress(Progress) +, m_Storage(Storage) +, m_AbortFlag(AbortFlag) +, m_PauseFlag(PauseFlag) +, m_IOWorkerPool(IOWorkerPool) +, m_NetworkPool(NetworkPool) +, m_BuildId(BuildId) +, m_Path(Path) +, m_LocalContent(LocalContent) +, m_LocalLookup(LocalLookup) +, m_RemoteContent(RemoteContent) +, m_RemoteLookup(RemoteLookup) +, m_BlockDescriptions(BlockDescriptions) +, m_LooseChunkHashes(LooseChunkHashes) +, m_Options(Options) +, m_CacheFolderPath(ZenTempCacheFolderPath(m_Options.ZenFolderPath)) +, m_TempDownloadFolderPath(ZenTempDownloadFolderPath(m_Options.ZenFolderPath)) +, m_TempBlockFolderPath(ZenTempBlockFolderPath(m_Options.ZenFolderPath)) +{ +} + +void +BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) +{ + ZEN_TRACE_CPU("BuildsOperationUpdateFolder::Execute"); + try + { + enum class TaskSteps : uint32_t + { + ScanExistingData, + WriteChunks, + PrepareTarget, + FinalizeTarget, + Cleanup, + StepCount + }; + + auto EndProgress = + MakeGuard([&]() { m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::StepCount, (uint32_t)TaskSteps::StepCount); }); + + m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::ScanExistingData, (uint32_t)TaskSteps::StepCount); + + CreateDirectories(m_CacheFolderPath); + CreateDirectories(m_TempDownloadFolderPath); + CreateDirectories(m_TempBlockFolderPath); + + std::vector> SequenceIndexChunksLeftToWriteCounters(m_RemoteContent.ChunkedContent.SequenceRawHashes.size()); + std::vector RemoteChunkIndexNeedsCopyFromLocalFileFlags(m_RemoteContent.ChunkedContent.ChunkHashes.size()); + std::vector> RemoteChunkIndexNeedsCopyFromSourceFlags(m_RemoteContent.ChunkedContent.ChunkHashes.size()); + + tsl::robin_map CachedChunkHashesFound; + tsl::robin_map CachedSequenceHashesFound; + ScanCacheFolder(CachedChunkHashesFound, CachedSequenceHashesFound); + + tsl::robin_map CachedBlocksFound; + ScanTempBlocksFolder(CachedBlocksFound); + + tsl::robin_map SequenceIndexesLeftToFindToRemoteIndex; + InitializeSequenceCounters(SequenceIndexChunksLeftToWriteCounters, + SequenceIndexesLeftToFindToRemoteIndex, + CachedChunkHashesFound, + CachedSequenceHashesFound); + + std::vector ScavengedContents; + std::vector ScavengedLookups; + std::vector ScavengedPaths; + + std::vector ScavengedSequenceCopyOperations; + uint64_t ScavengedPathsCount = 0; + + if (m_Options.EnableOtherDownloadsScavenging) + { + ZEN_TRACE_CPU("GetScavengedSequences"); + + Stopwatch ScavengeTimer; + + if (!SequenceIndexesLeftToFindToRemoteIndex.empty()) + { + std::vector ScavengeSources = FindScavengeSources(); + ScanScavengeSources(ScavengeSources, ScavengedContents, ScavengedLookups, ScavengedPaths); + if (m_AbortFlag) + { + return; + } + + MatchScavengedSequencesToRemote(ScavengedContents, + ScavengedLookups, + ScavengedPaths, + SequenceIndexesLeftToFindToRemoteIndex, + SequenceIndexChunksLeftToWriteCounters, + ScavengedSequenceCopyOperations, + ScavengedPathsCount); + } + m_CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs(); + } + + uint32_t RemainingChunkCount = 0; + for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < m_RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++) + { + uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex); + if (ChunkWriteCount > 0) + { + RemainingChunkCount++; + } + } + + // Pick up all chunks in current local state + tsl::robin_map RawHashToCopyChunkDataIndex; + std::vector CopyChunkDatas; + + if (m_Options.EnableTargetFolderScavenging) + { + ZEN_TRACE_CPU("GetLocalChunks"); + + Stopwatch LocalTimer; + + ScavengeSourceForChunks(RemainingChunkCount, + RemoteChunkIndexNeedsCopyFromLocalFileFlags, + RawHashToCopyChunkDataIndex, + SequenceIndexChunksLeftToWriteCounters, + m_LocalContent, + m_LocalLookup, + CopyChunkDatas, + uint32_t(-1), + m_CacheMappingStats.LocalChunkMatchingRemoteCount, + m_CacheMappingStats.LocalChunkMatchingRemoteByteCount); + + m_CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs(); + } + + if (m_Options.EnableOtherDownloadsScavenging) + { + ZEN_TRACE_CPU("GetScavengeChunks"); + + Stopwatch ScavengeTimer; + + for (uint32_t ScavengedContentIndex = 0; ScavengedContentIndex < ScavengedContents.size() && (RemainingChunkCount > 0); + ScavengedContentIndex++) + { + const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengedContentIndex]; + const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengedContentIndex]; + + ScavengeSourceForChunks(RemainingChunkCount, + RemoteChunkIndexNeedsCopyFromLocalFileFlags, + RawHashToCopyChunkDataIndex, + SequenceIndexChunksLeftToWriteCounters, + ScavengedContent, + ScavengedLookup, + CopyChunkDatas, + ScavengedContentIndex, + m_CacheMappingStats.ScavengedChunkMatchingRemoteCount, + m_CacheMappingStats.ScavengedChunkMatchingRemoteByteCount); + } + m_CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs(); + } + + if (!m_Options.IsQuiet) + { + if (m_CacheMappingStats.CacheSequenceHashesCount > 0 || m_CacheMappingStats.CacheChunkCount > 0 || + m_CacheMappingStats.CacheBlockCount > 0) + { + ZEN_INFO("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}", + m_CacheMappingStats.CacheSequenceHashesCount, + NiceBytes(m_CacheMappingStats.CacheSequenceHashesByteCount), + m_CacheMappingStats.CacheChunkCount, + NiceBytes(m_CacheMappingStats.CacheChunkByteCount), + m_CacheMappingStats.CacheBlockCount, + NiceBytes(m_CacheMappingStats.CacheBlocksByteCount), + NiceTimeSpanMs(m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000)); + } + + if (m_CacheMappingStats.LocalPathsMatchingSequencesCount > 0 || m_CacheMappingStats.LocalChunkMatchingRemoteCount > 0) + { + ZEN_INFO("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}", + m_CacheMappingStats.LocalPathsMatchingSequencesCount, + NiceBytes(m_CacheMappingStats.LocalPathsMatchingSequencesByteCount), + m_CacheMappingStats.LocalChunkMatchingRemoteCount, + NiceBytes(m_CacheMappingStats.LocalChunkMatchingRemoteByteCount), + NiceTimeSpanMs(m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000)); + } + if (m_CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || m_CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0) + { + ZEN_INFO("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}", + ScavengedPathsCount, + m_CacheMappingStats.ScavengedPathsMatchingSequencesCount, + NiceBytes(m_CacheMappingStats.ScavengedPathsMatchingSequencesByteCount), + m_CacheMappingStats.ScavengedChunkMatchingRemoteCount, + NiceBytes(m_CacheMappingStats.ScavengedChunkMatchingRemoteByteCount), + NiceTimeSpanMs(m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000)); + } + } + + uint64_t BytesToWrite = CalculateBytesToWriteAndFlagNeededChunks(SequenceIndexChunksLeftToWriteCounters, + RemoteChunkIndexNeedsCopyFromLocalFileFlags, + RemoteChunkIndexNeedsCopyFromSourceFlags); + + for (const ScavengedSequenceCopyOperation& ScavengeCopyOp : ScavengedSequenceCopyOperations) + { + BytesToWrite += ScavengeCopyOp.RawSize; + } + + uint64_t BytesToValidate = m_Options.ValidateCompletedSequences ? BytesToWrite : 0; + + uint64_t TotalRequestCount = 0; + uint64_t TotalPartWriteCount = 0; + std::atomic WritePartsComplete = 0; + + tsl::robin_map RemotePathToRemoteIndex; + RemotePathToRemoteIndex.reserve(m_RemoteContent.Paths.size()); + for (uint32_t RemotePathIndex = 0; RemotePathIndex < m_RemoteContent.Paths.size(); RemotePathIndex++) + { + RemotePathToRemoteIndex.insert({m_RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex}); + } + + CheckRequiredDiskSpace(RemotePathToRemoteIndex); + + BlobsExistsResult ExistsResult; + { + ChunkBlockAnalyser BlockAnalyser( + Log(), + m_BlockDescriptions, + ChunkBlockAnalyser::Options{.IsQuiet = m_Options.IsQuiet, + .IsVerbose = m_Options.IsVerbose, + .HostLatencySec = m_Storage.BuildStorageHost.LatencySec, + .HostHighSpeedLatencySec = m_Storage.CacheHost.LatencySec, + .HostMaxRangeCountPerRequest = m_Storage.BuildStorageHost.Caps.MaxRangeCountPerRequest, + .HostHighSpeedMaxRangeCountPerRequest = m_Storage.CacheHost.Caps.MaxRangeCountPerRequest}); + + std::vector NeededBlocks = BlockAnalyser.GetNeeded( + m_RemoteLookup.ChunkHashToChunkIndex, + [&](uint32_t RemoteChunkIndex) -> bool { return RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]; }); + + std::vector FetchBlockIndexes; + std::vector CachedChunkBlockIndexes; + ClassifyCachedAndFetchBlocks(NeededBlocks, CachedBlocksFound, TotalPartWriteCount, CachedChunkBlockIndexes, FetchBlockIndexes); + + std::vector NeededLooseChunkIndexes = DetermineNeededLooseChunkIndexes(SequenceIndexChunksLeftToWriteCounters, + RemoteChunkIndexNeedsCopyFromLocalFileFlags, + RemoteChunkIndexNeedsCopyFromSourceFlags); + + ExistsResult = QueryBlobCacheExists(NeededLooseChunkIndexes, FetchBlockIndexes); + + std::vector BlockPartialDownloadModes = + DeterminePartialDownloadModes(ExistsResult); + ZEN_ASSERT(BlockPartialDownloadModes.size() == m_BlockDescriptions.size()); + + ChunkBlockAnalyser::BlockResult PartialBlocks = + BlockAnalyser.CalculatePartialBlockDownloads(NeededBlocks, BlockPartialDownloadModes); + + TotalRequestCount += NeededLooseChunkIndexes.size(); + TotalPartWriteCount += NeededLooseChunkIndexes.size(); + TotalRequestCount += PartialBlocks.BlockRanges.size(); + TotalPartWriteCount += PartialBlocks.BlockRanges.size(); + TotalRequestCount += PartialBlocks.FullBlockIndexes.size(); + TotalPartWriteCount += PartialBlocks.FullBlockIndexes.size(); + + std::vector LooseChunkHashWorks = + BuildLooseChunkHashWorks(NeededLooseChunkIndexes, SequenceIndexChunksLeftToWriteCounters); + + ZEN_TRACE_CPU("WriteChunks"); + + m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::WriteChunks, (uint32_t)TaskSteps::StepCount); + + Stopwatch WriteTimer; + + FilteredRate FilteredDownloadedBytesPerSecond; + FilteredRate FilteredWrittenBytesPerSecond; + + std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Writing"); + ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + + TotalPartWriteCount += CopyChunkDatas.size(); + TotalPartWriteCount += ScavengedSequenceCopyOperations.size(); + + BufferedWriteFileCache WriteCache; + + WriteChunksContext Context{.Work = Work, + .WriteCache = WriteCache, + .SequenceIndexChunksLeftToWriteCounters = SequenceIndexChunksLeftToWriteCounters, + .RemoteChunkIndexNeedsCopyFromSourceFlags = RemoteChunkIndexNeedsCopyFromSourceFlags, + .WritePartsComplete = WritePartsComplete, + .TotalPartWriteCount = TotalPartWriteCount, + .TotalRequestCount = TotalRequestCount, + .ExistsResult = ExistsResult, + .FilteredDownloadedBytesPerSecond = FilteredDownloadedBytesPerSecond, + .FilteredWrittenBytesPerSecond = FilteredWrittenBytesPerSecond}; + + ScheduleScavengedSequenceWrites(Context, ScavengedSequenceCopyOperations, ScavengedContents, ScavengedPaths); + ScheduleLooseChunkWrites(Context, LooseChunkHashWorks); + + std::unique_ptr CloneQuery = + m_Options.AllowFileClone ? GetCloneQueryInterface(m_CacheFolderPath) : nullptr; + + ScheduleLocalChunkCopies(Context, CopyChunkDatas, CloneQuery.get(), ScavengedContents, ScavengedLookups, ScavengedPaths); + ScheduleCachedBlockWrites(Context, CachedChunkBlockIndexes); + SchedulePartialBlockDownloads(Context, PartialBlocks); + ScheduleFullBlockDownloads(Context, PartialBlocks.FullBlockIndexes); + + { + ZEN_TRACE_CPU("WriteChunks_Wait"); + + Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); + uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() + + m_DownloadStats.DownloadedBlockByteCount.load() + + +m_DownloadStats.DownloadedPartialBlockByteCount.load(); + FilteredWrittenBytesPerSecond.Update(m_DiskStats.WriteByteCount.load()); + FilteredDownloadedBytesPerSecond.Update(DownloadedBytes); + std::string DownloadRateString = + (m_DownloadStats.RequestsCompleteCount == TotalRequestCount) + ? "" + : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8)); + std::string CloneDetails; + if (m_DiskStats.CloneCount.load() > 0) + { + CloneDetails = fmt::format(" ({} cloned)", NiceBytes(m_DiskStats.CloneByteCount.load())); + } + std::string WriteDetails = fmt::format(" {}/{} ({}B/s) written{}", + NiceBytes(m_WrittenChunkByteCount.load()), + NiceBytes(BytesToWrite), + NiceNum(FilteredWrittenBytesPerSecond.GetCurrent()), + CloneDetails); + + std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}", + m_DownloadStats.RequestsCompleteCount.load(), + TotalRequestCount, + NiceBytes(DownloadedBytes), + DownloadRateString, + WriteDetails); + + std::string Task; + if ((m_WrittenChunkByteCount < BytesToWrite) || (BytesToValidate == 0)) + { + Task = "Writing chunks "; + } + else + { + Task = "Verifying chunks "; + } + + ProgressBar->UpdateState({.Task = Task, + .Details = Details, + .TotalCount = (BytesToWrite + BytesToValidate), + .RemainingCount = ((BytesToWrite + BytesToValidate) - + (m_WrittenChunkByteCount.load() + m_ValidatedChunkByteCount.load())), + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); + }); + } + + CloneQuery.reset(); + + FilteredWrittenBytesPerSecond.Stop(); + FilteredDownloadedBytesPerSecond.Stop(); + + ProgressBar->Finish(); + if (m_AbortFlag) + { + return; + } + + VerifyWriteChunksComplete(SequenceIndexChunksLeftToWriteCounters, BytesToWrite, BytesToValidate); + + const uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() + + m_DownloadStats.DownloadedBlockByteCount.load() + + m_DownloadStats.DownloadedPartialBlockByteCount.load(); + if (!m_Options.IsQuiet) + { + std::string CloneDetails; + if (m_DiskStats.CloneCount.load() > 0) + { + CloneDetails = fmt::format(" ({} cloned)", NiceBytes(m_DiskStats.CloneByteCount.load())); + } + ZEN_INFO("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s){} in {}. Completed in {}", + NiceBytes(DownloadedBytes), + NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)), + NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000), + NiceBytes(m_WrittenChunkByteCount.load()), + NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), m_DiskStats.WriteByteCount.load())), + CloneDetails, + NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000), + NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs())); + } + + m_WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs(); + m_WriteChunkStats.DownloadTimeUs = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(); + m_WriteChunkStats.WriteTimeUs = FilteredWrittenBytesPerSecond.GetElapsedTimeUS(); + } + + m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::PrepareTarget, (uint32_t)TaskSteps::StepCount); + + if (m_AbortFlag) + { + return; + } + + LocalPathCategorization Categorization = CategorizeLocalPaths(RemotePathToRemoteIndex); + + if (m_AbortFlag) + { + return; + } + + std::atomic CachedCount = 0; + std::atomic CachedByteCount = 0; + ScheduleLocalFileCaching(Categorization.FilesToCache, CachedCount, CachedByteCount); + if (m_AbortFlag) + { + return; + } + + ZEN_DEBUG( + "Local state prep: Match: {}, PathMismatch: {}, HashMismatch: {}, Cached: {} ({}), Skipped: {}, " + "Delete: {}", + Categorization.MatchCount, + Categorization.PathMismatchCount, + Categorization.HashMismatchCount, + CachedCount.load(), + NiceBytes(CachedByteCount.load()), + Categorization.SkippedCount, + Categorization.DeleteCount); + + m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::FinalizeTarget, (uint32_t)TaskSteps::StepCount); + + if (m_Options.WipeTargetFolder) + { + ZEN_TRACE_CPU("WipeTarget"); + Stopwatch Timer; + + // Clean target folder + if (!CleanDirectory(Log(), + m_Progress, + m_IOWorkerPool, + m_AbortFlag, + m_PauseFlag, + m_Options.IsQuiet, + m_Path, + m_Options.ExcludeFolders)) + { + ZEN_WARN("Some files in {} could not be removed", m_Path); + } + m_RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs(); + } + + if (m_AbortFlag) + { + return; + } + + { + ZEN_TRACE_CPU("FinalizeTree"); + + Stopwatch Timer; + + std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Rebuild State"); + ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + + OutLocalFolderState.Paths.resize(m_RemoteContent.Paths.size()); + OutLocalFolderState.RawSizes.resize(m_RemoteContent.Paths.size()); + OutLocalFolderState.Attributes.resize(m_RemoteContent.Paths.size()); + OutLocalFolderState.ModificationTicks.resize(m_RemoteContent.Paths.size()); + + std::atomic DeletedCount = 0; + std::atomic TargetsComplete = 0; + + ScheduleLocalFileRemovals(Work, Categorization.RemoveLocalPathIndexes, DeletedCount); + + std::vector Targets = BuildSortedFinalizeTargets(); + + ScheduleTargetFinalization(Work, + Targets, + Categorization.SequenceHashToLocalPathIndex, + Categorization.RemotePathIndexToLocalPathIndex, + OutLocalFolderState, + TargetsComplete); + + { + ZEN_TRACE_CPU("FinalizeTree_Wait"); + + Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); + const uint64_t WorkTotal = Targets.size() + Categorization.RemoveLocalPathIndexes.size(); + const uint64_t WorkComplete = TargetsComplete.load() + DeletedCount.load(); + std::string Details = fmt::format("{}/{} files", WorkComplete, WorkTotal); + ProgressBar->UpdateState({.Task = "Rebuilding state ", + .Details = Details, + .TotalCount = gsl::narrow(WorkTotal), + .RemainingCount = gsl::narrow(WorkTotal - WorkComplete), + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); + }); + } + + m_RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs = Timer.GetElapsedTimeUs(); + ProgressBar->Finish(); + } + m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::Cleanup, (uint32_t)TaskSteps::StepCount); + } + catch (const std::exception&) + { + m_AbortFlag = true; + throw; + } +} + +void +BuildsOperationUpdateFolder::ScanCacheFolder(tsl::robin_map& OutCachedChunkHashesFound, + tsl::robin_map& OutCachedSequenceHashesFound) +{ + ZEN_TRACE_CPU("ScanCacheFolder"); + + Stopwatch CacheTimer; + + DirectoryContent CacheDirContent; + GetDirectoryContent(m_CacheFolderPath, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, CacheDirContent); + for (size_t Index = 0; Index < CacheDirContent.Files.size(); Index++) + { + if (m_Options.EnableTargetFolderScavenging) + { + IoHash FileHash; + if (IoHash::TryParse(CacheDirContent.Files[Index].filename().string(), FileHash)) + { + if (auto ChunkIt = m_RemoteLookup.ChunkHashToChunkIndex.find(FileHash); + ChunkIt != m_RemoteLookup.ChunkHashToChunkIndex.end()) + { + const uint32_t ChunkIndex = ChunkIt->second; + const uint64_t ChunkSize = m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + if (ChunkSize == CacheDirContent.FileSizes[Index]) + { + OutCachedChunkHashesFound.insert({FileHash, ChunkIndex}); + m_CacheMappingStats.CacheChunkCount++; + m_CacheMappingStats.CacheChunkByteCount += ChunkSize; + continue; + } + } + else if (auto SequenceIt = m_RemoteLookup.RawHashToSequenceIndex.find(FileHash); + SequenceIt != m_RemoteLookup.RawHashToSequenceIndex.end()) + { + const uint32_t SequenceIndex = SequenceIt->second; + const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + const uint64_t SequenceSize = m_RemoteContent.RawSizes[PathIndex]; + if (SequenceSize == CacheDirContent.FileSizes[Index]) + { + OutCachedSequenceHashesFound.insert({FileHash, SequenceIndex}); + m_CacheMappingStats.CacheSequenceHashesCount++; + m_CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize; + + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(m_CacheFolderPath, + m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); + + continue; + } + } + } + } + std::error_code Ec = TryRemoveFile(CacheDirContent.Files[Index]); + if (Ec) + { + ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", CacheDirContent.Files[Index], Ec.value(), Ec.message()); + } + } + m_CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs(); +} + +void +BuildsOperationUpdateFolder::ScanTempBlocksFolder(tsl::robin_map& OutCachedBlocksFound) +{ + ZEN_TRACE_CPU("ScanTempBlocksFolder"); + + Stopwatch CacheTimer; + + tsl::robin_map AllBlockSizes; + AllBlockSizes.reserve(m_BlockDescriptions.size()); + for (uint32_t BlockIndex = 0; BlockIndex < m_BlockDescriptions.size(); BlockIndex++) + { + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; + AllBlockSizes.insert({BlockDescription.BlockHash, BlockIndex}); + } + + DirectoryContent BlockDirContent; + GetDirectoryContent(m_TempBlockFolderPath, + DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, + BlockDirContent); + OutCachedBlocksFound.reserve(BlockDirContent.Files.size()); + for (size_t Index = 0; Index < BlockDirContent.Files.size(); Index++) + { + if (m_Options.EnableTargetFolderScavenging) + { + IoHash FileHash; + if (IoHash::TryParse(BlockDirContent.Files[Index].filename().string(), FileHash)) + { + if (auto BlockIt = AllBlockSizes.find(FileHash); BlockIt != AllBlockSizes.end()) + { + const uint32_t BlockIndex = BlockIt->second; + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; + uint64_t BlockSize = CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize; + for (uint64_t ChunkSize : BlockDescription.ChunkCompressedLengths) + { + BlockSize += ChunkSize; + } + + if (BlockSize == BlockDirContent.FileSizes[Index]) + { + OutCachedBlocksFound.insert({FileHash, BlockIndex}); + m_CacheMappingStats.CacheBlockCount++; + m_CacheMappingStats.CacheBlocksByteCount += BlockSize; + continue; + } + } + } + } + std::error_code Ec = TryRemoveFile(BlockDirContent.Files[Index]); + if (Ec) + { + ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", BlockDirContent.Files[Index], Ec.value(), Ec.message()); + } + } + + m_CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs(); +} + +void +BuildsOperationUpdateFolder::InitializeSequenceCounters(std::vector>& OutSequenceCounters, + tsl::robin_map& OutSequencesLeftToFind, + const tsl::robin_map& CachedChunkHashesFound, + const tsl::robin_map& CachedSequenceHashesFound) +{ + if (m_Options.EnableTargetFolderScavenging) + { + // Pick up all whole files we can use from current local state + ZEN_TRACE_CPU("GetLocalSequences"); + + std::vector MissingSequenceIndexes = ScanTargetFolder(CachedChunkHashesFound, CachedSequenceHashesFound); + + for (uint32_t RemoteSequenceIndex : MissingSequenceIndexes) + { + // We must write the sequence + const uint32_t ChunkCount = m_RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; + const IoHash& RemoteSequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + OutSequenceCounters[RemoteSequenceIndex] = ChunkCount; + OutSequencesLeftToFind.insert({RemoteSequenceRawHash, RemoteSequenceIndex}); + } + } + else + { + for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < m_RemoteContent.ChunkedContent.SequenceRawHashes.size(); + RemoteSequenceIndex++) + { + OutSequenceCounters[RemoteSequenceIndex] = m_RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; + } + } +} + +void +BuildsOperationUpdateFolder::MatchScavengedSequencesToRemote(std::span Contents, + std::span Lookups, + std::span Paths, + tsl::robin_map& InOutSequencesLeftToFind, + std::vector>& InOutSequenceCounters, + std::vector& OutCopyOperations, + uint64_t& OutScavengedPathsCount) +{ + for (uint32_t ScavengedContentIndex = 0; ScavengedContentIndex < Contents.size() && !InOutSequencesLeftToFind.empty(); + ScavengedContentIndex++) + { + const std::filesystem::path& ScavengePath = Paths[ScavengedContentIndex]; + if (ScavengePath.empty()) + { + continue; + } + const ChunkedFolderContent& ScavengedLocalContent = Contents[ScavengedContentIndex]; + const ChunkedContentLookup& ScavengedLookup = Lookups[ScavengedContentIndex]; + + for (uint32_t ScavengedSequenceIndex = 0; ScavengedSequenceIndex < ScavengedLocalContent.ChunkedContent.SequenceRawHashes.size(); + ScavengedSequenceIndex++) + { + const IoHash& SequenceRawHash = ScavengedLocalContent.ChunkedContent.SequenceRawHashes[ScavengedSequenceIndex]; + auto It = InOutSequencesLeftToFind.find(SequenceRawHash); + if (It == InOutSequencesLeftToFind.end()) + { + continue; + } + const uint32_t RemoteSequenceIndex = It->second; + const uint64_t RawSize = m_RemoteContent.RawSizes[m_RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]]; + ZEN_ASSERT(RawSize > 0); + + const uint32_t ScavengedPathIndex = ScavengedLookup.SequenceIndexFirstPathIndex[ScavengedSequenceIndex]; + ZEN_ASSERT_SLOW(IsFile((ScavengePath / ScavengedLocalContent.Paths[ScavengedPathIndex]).make_preferred())); + + OutCopyOperations.push_back({.ScavengedContentIndex = ScavengedContentIndex, + .ScavengedPathIndex = ScavengedPathIndex, + .RemoteSequenceIndex = RemoteSequenceIndex, + .RawSize = RawSize}); + + InOutSequencesLeftToFind.erase(SequenceRawHash); + InOutSequenceCounters[RemoteSequenceIndex] = 0; + + m_CacheMappingStats.ScavengedPathsMatchingSequencesCount++; + m_CacheMappingStats.ScavengedPathsMatchingSequencesByteCount += RawSize; + } + OutScavengedPathsCount++; + } +} + +uint64_t +BuildsOperationUpdateFolder::CalculateBytesToWriteAndFlagNeededChunks(std::span> SequenceCounters, + const std::vector& NeedsCopyFromLocalFileFlags, + std::span> OutNeedsCopyFromSourceFlags) +{ + uint64_t BytesToWrite = 0; + for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < m_RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++) + { + const uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceCounters, RemoteChunkIndex); + if (ChunkWriteCount > 0) + { + BytesToWrite += m_RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] * ChunkWriteCount; + if (!NeedsCopyFromLocalFileFlags[RemoteChunkIndex]) + { + OutNeedsCopyFromSourceFlags[RemoteChunkIndex] = true; + } + } + } + return BytesToWrite; +} + +void +BuildsOperationUpdateFolder::ClassifyCachedAndFetchBlocks(std::span NeededBlocks, + const tsl::robin_map& CachedBlocksFound, + uint64_t& TotalPartWriteCount, + std::vector& OutCachedChunkBlockIndexes, + std::vector& OutFetchBlockIndexes) +{ + ZEN_TRACE_CPU("BlockCacheFileExists"); + for (const ChunkBlockAnalyser::NeededBlock& NeededBlock : NeededBlocks) + { + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[NeededBlock.BlockIndex]; + bool UsingCachedBlock = false; + if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end()) + { + TotalPartWriteCount++; + + std::filesystem::path BlockPath = m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString(); + if (IsFile(BlockPath)) + { + OutCachedChunkBlockIndexes.push_back(NeededBlock.BlockIndex); + UsingCachedBlock = true; + } + } + if (!UsingCachedBlock) + { + OutFetchBlockIndexes.push_back(NeededBlock.BlockIndex); + } + } +} + +std::vector +BuildsOperationUpdateFolder::DetermineNeededLooseChunkIndexes(std::span> SequenceCounters, + const std::vector& NeedsCopyFromLocalFileFlags, + std::span> NeedsCopyFromSourceFlags) +{ + std::vector NeededLooseChunkIndexes; + NeededLooseChunkIndexes.reserve(m_LooseChunkHashes.size()); + for (uint32_t LooseChunkIndex = 0; LooseChunkIndex < m_LooseChunkHashes.size(); LooseChunkIndex++) + { + const IoHash& ChunkHash = m_LooseChunkHashes[LooseChunkIndex]; + auto RemoteChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); + ZEN_ASSERT(RemoteChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end()); + const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second; + + if (NeedsCopyFromLocalFileFlags[RemoteChunkIndex]) + { + if (m_Options.IsVerbose) + { + ZEN_INFO("Skipping chunk {} due to cache reuse", m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]); + } + continue; + } + + bool NeedsCopy = true; + if (NeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false)) + { + const uint64_t WriteCount = GetChunkWriteCount(SequenceCounters, RemoteChunkIndex); + if (WriteCount == 0) + { + if (m_Options.IsVerbose) + { + ZEN_INFO("Skipping chunk {} due to cache reuse", m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]); + } + } + else + { + NeededLooseChunkIndexes.push_back(LooseChunkIndex); + } + } + } + return NeededLooseChunkIndexes; +} + +BuildsOperationUpdateFolder::BlobsExistsResult +BuildsOperationUpdateFolder::QueryBlobCacheExists(std::span NeededLooseChunkIndexes, + std::span FetchBlockIndexes) +{ + BlobsExistsResult Result; + if (!m_Storage.CacheStorage) + { + return Result; + } + + ZEN_TRACE_CPU("BlobCacheExistCheck"); + Stopwatch Timer; + + std::vector BlobHashes; + BlobHashes.reserve(NeededLooseChunkIndexes.size() + FetchBlockIndexes.size()); + + for (const uint32_t LooseChunkIndex : NeededLooseChunkIndexes) + { + BlobHashes.push_back(m_LooseChunkHashes[LooseChunkIndex]); + } + + for (uint32_t BlockIndex : FetchBlockIndexes) + { + BlobHashes.push_back(m_BlockDescriptions[BlockIndex].BlockHash); + } + + const std::vector CacheExistsResult = m_Storage.CacheStorage->BlobsExists(m_BuildId, BlobHashes); + + if (CacheExistsResult.size() == BlobHashes.size()) + { + Result.ExistingBlobs.reserve(CacheExistsResult.size()); + for (size_t BlobIndex = 0; BlobIndex < BlobHashes.size(); BlobIndex++) + { + if (CacheExistsResult[BlobIndex].HasBody) + { + Result.ExistingBlobs.insert(BlobHashes[BlobIndex]); + } + } + } + Result.ElapsedTimeMs = Timer.GetElapsedTimeMs(); + if (!Result.ExistingBlobs.empty() && !m_Options.IsQuiet) + { + ZEN_INFO("Remote cache : Found {} out of {} needed blobs in {}", + Result.ExistingBlobs.size(), + BlobHashes.size(), + NiceTimeSpanMs(Result.ElapsedTimeMs)); + } + return Result; +} + +std::vector +BuildsOperationUpdateFolder::DeterminePartialDownloadModes(const BlobsExistsResult& ExistsResult) +{ + std::vector Modes; + + if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::Off) + { + Modes.resize(m_BlockDescriptions.size(), ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); + return Modes; + } + + const bool MultiRangeCache = m_Storage.CacheHost.Caps.MaxRangeCountPerRequest > 1; + const bool MultiRangeBuild = m_Storage.BuildStorageHost.Caps.MaxRangeCountPerRequest > 1; + ChunkBlockAnalyser::EPartialBlockDownloadMode CachePartialDownloadMode = + MultiRangeCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange; + ChunkBlockAnalyser::EPartialBlockDownloadMode CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::Off; + + switch (m_Options.PartialBlockRequestMode) + { + case EPartialBlockRequestMode::Off: + break; + case EPartialBlockRequestMode::ZenCacheOnly: + CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::Off; + break; + case EPartialBlockRequestMode::Mixed: + CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange; + break; + case EPartialBlockRequestMode::All: + CloudPartialDownloadMode = MultiRangeBuild ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange + : ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange; + break; + default: + ZEN_ASSERT(false); + break; + } + + Modes.reserve(m_BlockDescriptions.size()); + for (uint32_t BlockIndex = 0; BlockIndex < m_BlockDescriptions.size(); BlockIndex++) + { + const bool BlockExistInCache = ExistsResult.ExistingBlobs.contains(m_BlockDescriptions[BlockIndex].BlockHash); + Modes.push_back(BlockExistInCache ? CachePartialDownloadMode : CloudPartialDownloadMode); + } + return Modes; +} + +std::vector +BuildsOperationUpdateFolder::BuildLooseChunkHashWorks(std::span NeededLooseChunkIndexes, + std::span> SequenceCounters) +{ + std::vector LooseChunkHashWorks; + LooseChunkHashWorks.reserve(NeededLooseChunkIndexes.size()); + for (uint32_t LooseChunkIndex : NeededLooseChunkIndexes) + { + const IoHash& ChunkHash = m_LooseChunkHashes[LooseChunkIndex]; + auto RemoteChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); + ZEN_ASSERT(RemoteChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end()); + const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second; + + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(SequenceCounters, RemoteChunkIndex); + + ZEN_ASSERT(!ChunkTargetPtrs.empty()); + LooseChunkHashWorks.push_back(LooseChunkHashWorkData{.ChunkTargetPtrs = ChunkTargetPtrs, .RemoteChunkIndex = RemoteChunkIndex}); + } + return LooseChunkHashWorks; +} + +void +BuildsOperationUpdateFolder::VerifyWriteChunksComplete(std::span> SequenceCounters, + uint64_t BytesToWrite, + uint64_t BytesToValidate) +{ + uint32_t RawSequencesMissingWriteCount = 0; + for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceCounters.size(); SequenceIndex++) + { + const auto& Counter = SequenceCounters[SequenceIndex]; + if (Counter.load() != 0) + { + RawSequencesMissingWriteCount++; + const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + const std::filesystem::path& IncompletePath = m_RemoteContent.Paths[PathIndex]; + ZEN_ASSERT(!IncompletePath.empty()); + const uint32_t ExpectedSequenceCount = m_RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; + if (!m_Options.IsQuiet) + { + ZEN_INFO("{}: Max count {}, Current count {}", IncompletePath, ExpectedSequenceCount, Counter.load()); + } + ZEN_ASSERT(Counter.load() <= ExpectedSequenceCount); + } + } + ZEN_ASSERT(RawSequencesMissingWriteCount == 0); + ZEN_ASSERT(m_WrittenChunkByteCount == BytesToWrite); + ZEN_ASSERT(m_ValidatedChunkByteCount == BytesToValidate); +} + +std::vector +BuildsOperationUpdateFolder::BuildSortedFinalizeTargets() +{ + std::vector Targets; + Targets.reserve(m_RemoteContent.Paths.size()); + for (uint32_t RemotePathIndex = 0; RemotePathIndex < m_RemoteContent.Paths.size(); RemotePathIndex++) + { + Targets.push_back(FinalizeTarget{.RawHash = m_RemoteContent.RawHashes[RemotePathIndex], .RemotePathIndex = RemotePathIndex}); + } + std::sort(Targets.begin(), Targets.end(), [](const FinalizeTarget& Lhs, const FinalizeTarget& Rhs) { + return std::tie(Lhs.RawHash, Lhs.RemotePathIndex) < std::tie(Rhs.RawHash, Rhs.RemotePathIndex); + }); + return Targets; +} + +void +BuildsOperationUpdateFolder::ScanScavengeSources(std::span Sources, + std::vector& OutContents, + std::vector& OutLookups, + std::vector& OutPaths) +{ + ZEN_TRACE_CPU("ScanScavengeSources"); + + const size_t ScavengePathCount = Sources.size(); + OutContents.resize(ScavengePathCount); + OutLookups.resize(ScavengePathCount); + OutPaths.resize(ScavengePathCount); + + std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Scavenging"); + + ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + + std::atomic PathsFound(0); + std::atomic ChunksFound(0); + std::atomic PathsScavenged(0); + + for (size_t ScavengeIndex = 0; ScavengeIndex < ScavengePathCount; ScavengeIndex++) + { + Work.ScheduleWork(m_IOWorkerPool, + [this, &Sources, &OutContents, &OutPaths, &OutLookups, &PathsFound, &ChunksFound, &PathsScavenged, ScavengeIndex]( + std::atomic&) { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("Async_FindScavengeContent"); + + const ScavengeSource& Source = Sources[ScavengeIndex]; + ChunkedFolderContent& ScavengedLocalContent = OutContents[ScavengeIndex]; + ChunkedContentLookup& ScavengedLookup = OutLookups[ScavengeIndex]; + + if (FindScavengeContent(Source, ScavengedLocalContent, ScavengedLookup)) + { + OutPaths[ScavengeIndex] = Source.Path; + PathsFound += ScavengedLocalContent.Paths.size(); + ChunksFound += ScavengedLocalContent.ChunkedContent.ChunkHashes.size(); + } + else + { + OutPaths[ScavengeIndex].clear(); + } + PathsScavenged++; + } + }); + } + { + ZEN_TRACE_CPU("ScavengeScan_Wait"); + + Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); + std::string Details = fmt::format("{}/{} scanned. {} paths and {} chunks found for scavenging", + PathsScavenged.load(), + ScavengePathCount, + PathsFound.load(), + ChunksFound.load()); + ProgressBar->UpdateState({.Task = "Scavenging ", + .Details = Details, + .TotalCount = ScavengePathCount, + .RemainingCount = ScavengePathCount - PathsScavenged.load(), + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); + }); + } + + ProgressBar->Finish(); +} + +BuildsOperationUpdateFolder::LocalPathCategorization +BuildsOperationUpdateFolder::CategorizeLocalPaths(const tsl::robin_map& RemotePathToRemoteIndex) +{ + ZEN_TRACE_CPU("PrepareTarget"); + + LocalPathCategorization Result; + tsl::robin_set CachedRemoteSequences; + + Result.RemotePathIndexToLocalPathIndex.reserve(m_RemoteContent.Paths.size()); + + for (uint32_t LocalPathIndex = 0; LocalPathIndex < m_LocalContent.Paths.size(); LocalPathIndex++) + { + if (m_AbortFlag) + { + break; + } + const IoHash& RawHash = m_LocalContent.RawHashes[LocalPathIndex]; + const std::filesystem::path& LocalPath = m_LocalContent.Paths[LocalPathIndex]; + + ZEN_ASSERT_SLOW(IsFile((m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred())); + + if (m_Options.EnableTargetFolderScavenging) + { + if (!m_Options.WipeTargetFolder) + { + // Check if it is already in the correct place + if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string()); + RemotePathIt != RemotePathToRemoteIndex.end()) + { + const uint32_t RemotePathIndex = RemotePathIt->second; + if (m_RemoteContent.RawHashes[RemotePathIndex] == RawHash) + { + // It is already in it's correct place + Result.RemotePathIndexToLocalPathIndex[RemotePathIndex] = LocalPathIndex; + Result.SequenceHashToLocalPathIndex.insert({RawHash, LocalPathIndex}); + Result.MatchCount++; + continue; + } + else + { + Result.HashMismatchCount++; + } + } + else + { + Result.PathMismatchCount++; + } + } + + // Do we need it? + if (m_RemoteLookup.RawHashToSequenceIndex.contains(RawHash)) + { + if (!CachedRemoteSequences.contains(RawHash)) + { + // We need it, make sure we move it to the cache + Result.FilesToCache.push_back(LocalPathIndex); + CachedRemoteSequences.insert(RawHash); + continue; + } + else + { + Result.SkippedCount++; + } + } + } + + if (!m_Options.WipeTargetFolder) + { + // Explicitly delete the unneeded local file + Result.RemoveLocalPathIndexes.push_back(LocalPathIndex); + Result.DeleteCount++; + } + } + + return Result; +} + +void +BuildsOperationUpdateFolder::ScheduleLocalFileCaching(std::span FilesToCache, + std::atomic& OutCachedCount, + std::atomic& OutCachedByteCount) +{ + ZEN_TRACE_CPU("CopyToCache"); + + std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Cache Local Data"); + ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + + for (uint32_t LocalPathIndex : FilesToCache) + { + if (m_AbortFlag) + { + break; + } + Work.ScheduleWork(m_IOWorkerPool, [this, &OutCachedCount, &OutCachedByteCount, LocalPathIndex](std::atomic&) { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("Async_CopyToCache"); + + const IoHash& RawHash = m_LocalContent.RawHashes[LocalPathIndex]; + const std::filesystem::path& LocalPath = m_LocalContent.Paths[LocalPathIndex]; + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RawHash); + ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath)); + const std::filesystem::path LocalFilePath = (m_Path / LocalPath).make_preferred(); + + std::error_code Ec = RenameFileWithRetry(LocalFilePath, CacheFilePath); + if (Ec) + { + ZEN_WARN("Failed to move file from '{}' to '{}', reason: ({}) {}, retrying...", + LocalFilePath, + CacheFilePath, + Ec.value(), + Ec.message()); + Ec = RenameFileWithRetry(LocalFilePath, CacheFilePath); + if (Ec) + { + throw std::system_error(std::error_code(Ec.value(), std::system_category()), + fmt::format("Failed to file from '{}' to '{}', reason: ({}) {}", + LocalFilePath, + CacheFilePath, + Ec.value(), + Ec.message())); + } + } + + OutCachedCount++; + OutCachedByteCount += m_LocalContent.RawSizes[LocalPathIndex]; + } + }); + } + + { + ZEN_TRACE_CPU("CopyToCache_Wait"); + + Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); + const uint64_t WorkTotal = FilesToCache.size(); + const uint64_t WorkComplete = OutCachedCount.load(); + std::string Details = fmt::format("{}/{} ({}) files", WorkComplete, WorkTotal, NiceBytes(OutCachedByteCount)); + ProgressBar->UpdateState({.Task = "Caching local ", + .Details = Details, + .TotalCount = gsl::narrow(WorkTotal), + .RemainingCount = gsl::narrow(WorkTotal - WorkComplete), + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); + }); + } + + ProgressBar->Finish(); +} + +void +BuildsOperationUpdateFolder::ScheduleScavengedSequenceWrites(WriteChunksContext& Context, + std::span CopyOperations, + const std::vector& ScavengedContents, + const std::vector& ScavengedPaths) +{ + for (uint32_t ScavengeOpIndex = 0; ScavengeOpIndex < CopyOperations.size(); ScavengeOpIndex++) + { + if (m_AbortFlag) + { + break; + } + Context.Work.ScheduleWork( + m_IOWorkerPool, + [this, &Context, &CopyOperations, &ScavengedContents, &ScavengedPaths, ScavengeOpIndex](std::atomic&) { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("Async_WriteScavenged"); + + Context.FilteredWrittenBytesPerSecond.Start(); + + const ScavengedSequenceCopyOperation& ScavengeOp = CopyOperations[ScavengeOpIndex]; + const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengeOp.ScavengedContentIndex]; + const std::filesystem::path& ScavengeRootPath = ScavengedPaths[ScavengeOp.ScavengedContentIndex]; + + WriteScavengedSequenceToCache(ScavengeRootPath, ScavengedContent, ScavengeOp); + + if (Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount) + { + Context.FilteredWrittenBytesPerSecond.Stop(); + } + } + }); + } +} + +void +BuildsOperationUpdateFolder::ScheduleLooseChunkWrites(WriteChunksContext& Context, std::vector& LooseChunkHashWorks) +{ + for (uint32_t LooseChunkHashWorkIndex = 0; LooseChunkHashWorkIndex < LooseChunkHashWorks.size(); LooseChunkHashWorkIndex++) + { + if (m_AbortFlag) + { + break; + } + + Context.Work.ScheduleWork( + m_IOWorkerPool, + [this, &Context, &LooseChunkHashWorks, LooseChunkHashWorkIndex](std::atomic&) { + ZEN_TRACE_CPU("Async_ReadPreDownloadedChunk"); + if (!m_AbortFlag) + { + LooseChunkHashWorkData& LooseChunkHashWork = LooseChunkHashWorks[LooseChunkHashWorkIndex]; + const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex; + WriteLooseChunk(RemoteChunkIndex, + Context.ExistsResult, + Context.SequenceIndexChunksLeftToWriteCounters, + Context.WritePartsComplete, + std::move(LooseChunkHashWork.ChunkTargetPtrs), + Context.WriteCache, + Context.Work, + Context.TotalRequestCount, + Context.TotalPartWriteCount, + Context.FilteredDownloadedBytesPerSecond, + Context.FilteredWrittenBytesPerSecond); + } + }, + WorkerThreadPool::EMode::EnableBacklog); + } +} + +void +BuildsOperationUpdateFolder::ScheduleLocalChunkCopies(WriteChunksContext& Context, + std::span CopyChunkDatas, + CloneQueryInterface* CloneQuery, + const std::vector& ScavengedContents, + const std::vector& ScavengedLookups, + const std::vector& ScavengedPaths) +{ + for (size_t CopyDataIndex = 0; CopyDataIndex < CopyChunkDatas.size(); CopyDataIndex++) + { + if (m_AbortFlag) + { + break; + } + + Context.Work.ScheduleWork( + m_IOWorkerPool, + [this, &Context, CloneQuery, &CopyChunkDatas, &ScavengedContents, &ScavengedLookups, &ScavengedPaths, CopyDataIndex]( + std::atomic&) { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("Async_CopyLocal"); + + Context.FilteredWrittenBytesPerSecond.Start(); + const CopyChunkData& CopyData = CopyChunkDatas[CopyDataIndex]; + + std::vector WrittenSequenceIndexes = WriteLocalChunkToCache(CloneQuery, + CopyData, + ScavengedContents, + ScavengedLookups, + ScavengedPaths, + Context.WriteCache); + bool WritePartsDone = Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount; + if (!m_AbortFlag) + { + if (WritePartsDone) + { + Context.FilteredWrittenBytesPerSecond.Stop(); + } + + // Write tracking, updating this must be done without any files open + std::vector CompletedChunkSequences; + for (uint32_t RemoteSequenceIndex : WrittenSequenceIndexes) + { + if (CompleteSequenceChunk(RemoteSequenceIndex, Context.SequenceIndexChunksLeftToWriteCounters)) + { + CompletedChunkSequences.push_back(RemoteSequenceIndex); + } + } + Context.WriteCache.Close(CompletedChunkSequences); + VerifyAndCompleteChunkSequencesAsync(CompletedChunkSequences, Context.Work); + } + } + }); + } +} + +void +BuildsOperationUpdateFolder::ScheduleCachedBlockWrites(WriteChunksContext& Context, std::span CachedBlockIndexes) +{ + for (uint32_t BlockIndex : CachedBlockIndexes) + { + if (m_AbortFlag) + { + break; + } + + Context.Work.ScheduleWork(m_IOWorkerPool, [this, &Context, BlockIndex](std::atomic&) { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("Async_WriteCachedBlock"); + + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; + Context.FilteredWrittenBytesPerSecond.Start(); + + std::filesystem::path BlockChunkPath = m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString(); + IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Can not read block {} at {}", BlockDescription.BlockHash, BlockChunkPath)); + } + + if (!m_AbortFlag) + { + if (!WriteChunksBlockToCache(BlockDescription, + Context.SequenceIndexChunksLeftToWriteCounters, + Context.Work, + CompositeBuffer(std::move(BlockBuffer)), + Context.RemoteChunkIndexNeedsCopyFromSourceFlags, + Context.WriteCache)) + { + std::error_code DummyEc; + RemoveFile(BlockChunkPath, DummyEc); + throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + } + + std::error_code Ec = TryRemoveFile(BlockChunkPath); + if (Ec) + { + ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", BlockChunkPath, Ec.value(), Ec.message()); + } + + if (Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount) + { + Context.FilteredWrittenBytesPerSecond.Stop(); + } + } + } + }); + } +} + +void +BuildsOperationUpdateFolder::SchedulePartialBlockDownloads(WriteChunksContext& Context, + const ChunkBlockAnalyser::BlockResult& PartialBlocks) +{ + for (size_t BlockRangeIndex = 0; BlockRangeIndex < PartialBlocks.BlockRanges.size();) + { + if (m_AbortFlag) + { + break; + } + + size_t RangeCount = 1; + size_t RangesLeft = PartialBlocks.BlockRanges.size() - BlockRangeIndex; + const ChunkBlockAnalyser::BlockRangeDescriptor& CurrentBlockRange = PartialBlocks.BlockRanges[BlockRangeIndex]; + while (RangeCount < RangesLeft && + CurrentBlockRange.BlockIndex == PartialBlocks.BlockRanges[BlockRangeIndex + RangeCount].BlockIndex) + { + RangeCount++; + } + + Context.Work.ScheduleWork( + m_NetworkPool, + [this, &Context, &PartialBlocks, BlockRangeStartIndex = BlockRangeIndex, RangeCount = RangeCount](std::atomic&) { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("Async_GetPartialBlockRanges"); + + Context.FilteredDownloadedBytesPerSecond.Start(); + + DownloadPartialBlock( + PartialBlocks.BlockRanges, + BlockRangeStartIndex, + RangeCount, + Context.ExistsResult, + Context.TotalRequestCount, + Context.FilteredDownloadedBytesPerSecond, + [this, &Context, &PartialBlocks](IoBuffer&& InMemoryBuffer, + const std::filesystem::path& OnDiskPath, + size_t BlockRangeStartIndex, + std::span> OffsetAndLengths) { + if (!m_AbortFlag) + { + Context.Work.ScheduleWork( + m_IOWorkerPool, + [this, + &Context, + &PartialBlocks, + BlockRangeStartIndex, + BlockChunkPath = std::filesystem::path(OnDiskPath), + BlockPartialBuffer = std::move(InMemoryBuffer), + OffsetAndLengths = + std::vector>(OffsetAndLengths.begin(), OffsetAndLengths.end())]( + std::atomic&) mutable { + if (!m_AbortFlag) + { + WritePartialBlockToCache(Context, + BlockRangeStartIndex, + std::move(BlockPartialBuffer), + BlockChunkPath, + OffsetAndLengths, + PartialBlocks); + } + }, + OnDiskPath.empty() ? WorkerThreadPool::EMode::DisableBacklog : WorkerThreadPool::EMode::EnableBacklog); + } + }); + } + }); + BlockRangeIndex += RangeCount; + } +} + +void +BuildsOperationUpdateFolder::WritePartialBlockToCache(WriteChunksContext& Context, + size_t BlockRangeStartIndex, + IoBuffer BlockPartialBuffer, + const std::filesystem::path& BlockChunkPath, + std::span> OffsetAndLengths, + const ChunkBlockAnalyser::BlockResult& PartialBlocks) +{ + ZEN_TRACE_CPU("Async_WritePartialBlock"); + + const uint32_t BlockIndex = PartialBlocks.BlockRanges[BlockRangeStartIndex].BlockIndex; + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; + + if (BlockChunkPath.empty()) + { + ZEN_ASSERT(BlockPartialBuffer); + } + else + { + ZEN_ASSERT(!BlockPartialBuffer); + BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockPartialBuffer) + { + throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}", BlockDescription.BlockHash, BlockChunkPath)); + } + } + + Context.FilteredWrittenBytesPerSecond.Start(); + + const size_t RangeCount = OffsetAndLengths.size(); + + for (size_t PartialRangeIndex = 0; PartialRangeIndex < RangeCount; PartialRangeIndex++) + { + const std::pair& OffsetAndLength = OffsetAndLengths[PartialRangeIndex]; + IoBuffer BlockRangeBuffer(BlockPartialBuffer, OffsetAndLength.first, OffsetAndLength.second); + + const ChunkBlockAnalyser::BlockRangeDescriptor& RangeDescriptor = + PartialBlocks.BlockRanges[BlockRangeStartIndex + PartialRangeIndex]; + + if (!WritePartialBlockChunksToCache(BlockDescription, + Context.SequenceIndexChunksLeftToWriteCounters, + Context.Work, + CompositeBuffer(std::move(BlockRangeBuffer)), + RangeDescriptor.ChunkBlockIndexStart, + RangeDescriptor.ChunkBlockIndexStart + RangeDescriptor.ChunkBlockIndexCount - 1, + Context.RemoteChunkIndexNeedsCopyFromSourceFlags, + Context.WriteCache)) + { + std::error_code DummyEc; + RemoveFile(BlockChunkPath, DummyEc); + throw std::runtime_error(fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); + } + + if (Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount) + { + Context.FilteredWrittenBytesPerSecond.Stop(); + } + } + std::error_code Ec = TryRemoveFile(BlockChunkPath); + if (Ec) + { + ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", BlockChunkPath, Ec.value(), Ec.message()); + } +} + +void +BuildsOperationUpdateFolder::ScheduleFullBlockDownloads(WriteChunksContext& Context, std::span FullBlockIndexes) +{ + for (uint32_t BlockIndex : FullBlockIndexes) + { + if (m_AbortFlag) + { + break; + } + + Context.Work.ScheduleWork(m_NetworkPool, [this, &Context, BlockIndex](std::atomic&) { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("Async_GetFullBlock"); + + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; + + Context.FilteredDownloadedBytesPerSecond.Start(); + + IoBuffer BlockBuffer; + const bool ExistsInCache = + m_Storage.CacheStorage && Context.ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); + if (ExistsInCache) + { + BlockBuffer = m_Storage.CacheStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash); + } + if (!BlockBuffer) + { + try + { + BlockBuffer = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash); + } + catch (const std::exception&) + { + // Silence http errors due to abort + if (!m_AbortFlag) + { + throw; + } + } + } + if (!m_AbortFlag) + { + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + } + + uint64_t BlockSize = BlockBuffer.GetSize(); + m_DownloadStats.DownloadedBlockCount++; + m_DownloadStats.DownloadedBlockByteCount += BlockSize; + if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == Context.TotalRequestCount) + { + Context.FilteredDownloadedBytesPerSecond.Stop(); + } + + const bool PutInCache = !ExistsInCache && m_Storage.CacheStorage && m_Options.PopulateCache; + + std::filesystem::path BlockChunkPath = + TryMoveDownloadedChunk(BlockBuffer, + m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString(), + /* ForceDiskBased */ PutInCache || (BlockSize > m_Options.MaximumInMemoryPayloadSize)); + + if (PutInCache) + { + ZEN_ASSERT(!BlockChunkPath.empty()); + IoBuffer CacheBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (CacheBuffer) + { + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, + BlockDescription.BlockHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(CacheBuffer))); + } + } + + if (!m_AbortFlag) + { + Context.Work.ScheduleWork( + m_IOWorkerPool, + [this, &Context, BlockIndex, BlockChunkPath, BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { + if (!m_AbortFlag) + { + WriteFullBlockToCache(Context, BlockIndex, std::move(BlockBuffer), BlockChunkPath); + } + }, + BlockChunkPath.empty() ? WorkerThreadPool::EMode::DisableBacklog : WorkerThreadPool::EMode::EnableBacklog); + } + } + } + }); + } +} + +void +BuildsOperationUpdateFolder::WriteFullBlockToCache(WriteChunksContext& Context, + uint32_t BlockIndex, + IoBuffer BlockBuffer, + const std::filesystem::path& BlockChunkPath) +{ + ZEN_TRACE_CPU("Async_WriteFullBlock"); + + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; + + if (BlockChunkPath.empty()) + { + ZEN_ASSERT(BlockBuffer); + } + else + { + ZEN_ASSERT(!BlockBuffer); + BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Could not open dowloaded block {} from {}", BlockDescription.BlockHash, BlockChunkPath)); + } + } + + Context.FilteredWrittenBytesPerSecond.Start(); + if (!WriteChunksBlockToCache(BlockDescription, + Context.SequenceIndexChunksLeftToWriteCounters, + Context.Work, + CompositeBuffer(std::move(BlockBuffer)), + Context.RemoteChunkIndexNeedsCopyFromSourceFlags, + Context.WriteCache)) + { + std::error_code DummyEc; + RemoveFile(BlockChunkPath, DummyEc); + throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + } + + if (!BlockChunkPath.empty()) + { + std::error_code Ec = TryRemoveFile(BlockChunkPath); + if (Ec) + { + ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", BlockChunkPath, Ec.value(), Ec.message()); + } + } + + if (Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount) + { + Context.FilteredWrittenBytesPerSecond.Stop(); + } +} + +void +BuildsOperationUpdateFolder::ScheduleLocalFileRemovals(ParallelWork& Work, + std::span RemoveLocalPathIndexes, + std::atomic& DeletedCount) +{ + for (uint32_t LocalPathIndex : RemoveLocalPathIndexes) + { + if (m_AbortFlag) + { + break; + } + Work.ScheduleWork(m_IOWorkerPool, [this, &DeletedCount, LocalPathIndex](std::atomic&) { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("Async_RemoveFile"); + + const std::filesystem::path LocalFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred(); + SetFileReadOnlyWithRetry(LocalFilePath, false); + RemoveFileWithRetry(LocalFilePath); + DeletedCount++; + } + }); + } +} + +void +BuildsOperationUpdateFolder::ScheduleTargetFinalization( + ParallelWork& Work, + std::span Targets, + const tsl::robin_map& SequenceHashToLocalPathIndex, + const tsl::robin_map& RemotePathIndexToLocalPathIndex, + FolderContent& OutLocalFolderState, + std::atomic& TargetsComplete) +{ + size_t TargetOffset = 0; + while (TargetOffset < Targets.size()) + { + if (m_AbortFlag) + { + break; + } + + size_t TargetCount = 1; + while ((TargetOffset + TargetCount) < Targets.size() && + (Targets[TargetOffset + TargetCount].RawHash == Targets[TargetOffset].RawHash)) + { + TargetCount++; + } + + Work.ScheduleWork(m_IOWorkerPool, + [this, + &SequenceHashToLocalPathIndex, + Targets, + &RemotePathIndexToLocalPathIndex, + &OutLocalFolderState, + BaseTargetOffset = TargetOffset, + TargetCount, + &TargetsComplete](std::atomic&) { + if (!m_AbortFlag) + { + FinalizeTargetGroup(BaseTargetOffset, + TargetCount, + Targets, + SequenceHashToLocalPathIndex, + RemotePathIndexToLocalPathIndex, + OutLocalFolderState, + TargetsComplete); + } + }); + + TargetOffset += TargetCount; + } +} + +void +BuildsOperationUpdateFolder::FinalizeTargetGroup(size_t BaseOffset, + size_t Count, + std::span Targets, + const tsl::robin_map& SequenceHashToLocalPathIndex, + const tsl::robin_map& RemotePathIndexToLocalPathIndex, + FolderContent& OutLocalFolderState, + std::atomic& TargetsComplete) +{ + ZEN_TRACE_CPU("Async_FinalizeChunkSequence"); + + size_t TargetOffset = BaseOffset; + const IoHash& RawHash = Targets[TargetOffset].RawHash; + + if (RawHash == IoHash::Zero) + { + ZEN_TRACE_CPU("CreateEmptyFiles"); + while (TargetOffset < (BaseOffset + Count)) + { + const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; + ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); + const std::filesystem::path& TargetPath = m_RemoteContent.Paths[RemotePathIndex]; + std::filesystem::path TargetFilePath = (m_Path / TargetPath).make_preferred(); + auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); + if (InPlaceIt == RemotePathIndexToLocalPathIndex.end() || InPlaceIt->second == 0) + { + if (IsFileWithRetry(TargetFilePath)) + { + SetFileReadOnlyWithRetry(TargetFilePath, false); + } + else + { + CreateDirectories(TargetFilePath.parent_path()); + } + BasicFile OutputFile; + OutputFile.Open(TargetFilePath, BasicFile::Mode::kTruncate); + } + OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; + OutLocalFolderState.RawSizes[RemotePathIndex] = m_RemoteContent.RawSizes[RemotePathIndex]; + + OutLocalFolderState.Attributes[RemotePathIndex] = + m_RemoteContent.Attributes.empty() + ? GetNativeFileAttributes(TargetFilePath) + : SetNativeFileAttributes(TargetFilePath, m_RemoteContent.Platform, m_RemoteContent.Attributes[RemotePathIndex]); + OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); + + TargetOffset++; + TargetsComplete++; + } + } + else + { + ZEN_TRACE_CPU("FinalizeFile"); + ZEN_ASSERT(m_RemoteLookup.RawHashToSequenceIndex.contains(RawHash)); + const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex; + const std::filesystem::path& FirstTargetPath = m_RemoteContent.Paths[FirstRemotePathIndex]; + std::filesystem::path FirstTargetFilePath = (m_Path / FirstTargetPath).make_preferred(); + + if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end()) + { + ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); + } + else + { + if (IsFileWithRetry(FirstTargetFilePath)) + { + SetFileReadOnlyWithRetry(FirstTargetFilePath, false); + } + else + { + CreateDirectories(FirstTargetFilePath.parent_path()); + } + + if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash); InplaceIt != SequenceHashToLocalPathIndex.end()) + { + ZEN_TRACE_CPU("Copy"); + const uint32_t LocalPathIndex = InplaceIt->second; + const std::filesystem::path& SourcePath = m_LocalContent.Paths[LocalPathIndex]; + std::filesystem::path SourceFilePath = (m_Path / SourcePath).make_preferred(); + ZEN_ASSERT_SLOW(IsFileWithRetry(SourceFilePath)); + + ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath); + const uint64_t RawSize = m_LocalContent.RawSizes[LocalPathIndex]; + FastCopyFile(m_Options.AllowFileClone, + m_Options.UseSparseFiles, + SourceFilePath, + FirstTargetFilePath, + RawSize, + m_DiskStats.WriteCount, + m_DiskStats.WriteByteCount, + m_DiskStats.CloneCount, + m_DiskStats.CloneByteCount); + + m_RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + } + else + { + ZEN_TRACE_CPU("Rename"); + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RawHash); + ZEN_ASSERT_SLOW(IsFileWithRetry(CacheFilePath)); + + std::error_code Ec = RenameFileWithRetry(CacheFilePath, FirstTargetFilePath); + if (Ec) + { + ZEN_WARN("Failed to move file from '{}' to '{}', reason: ({}) {}, retrying...", + CacheFilePath, + FirstTargetFilePath, + Ec.value(), + Ec.message()); + Ec = RenameFileWithRetry(CacheFilePath, FirstTargetFilePath); + if (Ec) + { + throw std::system_error(std::error_code(Ec.value(), std::system_category()), + fmt::format("Failed to move file from '{}' to '{}', reason: ({}) {}", + CacheFilePath, + FirstTargetFilePath, + Ec.value(), + Ec.message())); + } + } + + m_RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; + } + } + + OutLocalFolderState.Paths[FirstRemotePathIndex] = FirstTargetPath; + OutLocalFolderState.RawSizes[FirstRemotePathIndex] = m_RemoteContent.RawSizes[FirstRemotePathIndex]; + + OutLocalFolderState.Attributes[FirstRemotePathIndex] = + m_RemoteContent.Attributes.empty() + ? GetNativeFileAttributes(FirstTargetFilePath) + : SetNativeFileAttributes(FirstTargetFilePath, m_RemoteContent.Platform, m_RemoteContent.Attributes[FirstRemotePathIndex]); + OutLocalFolderState.ModificationTicks[FirstRemotePathIndex] = GetModificationTickFromPath(FirstTargetFilePath); + + TargetOffset++; + TargetsComplete++; + + while (TargetOffset < (BaseOffset + Count)) + { + const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; + ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); + const std::filesystem::path& TargetPath = m_RemoteContent.Paths[RemotePathIndex]; + std::filesystem::path TargetFilePath = (m_Path / TargetPath).make_preferred(); + + if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end()) + { + ZEN_ASSERT_SLOW(IsFileWithRetry(TargetFilePath)); + } + else + { + ZEN_TRACE_CPU("Copy"); + if (IsFileWithRetry(TargetFilePath)) + { + SetFileReadOnlyWithRetry(TargetFilePath, false); + } + else + { + CreateDirectories(TargetFilePath.parent_path()); + } + + ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); + ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath); + const uint64_t RawSize = m_RemoteContent.RawSizes[RemotePathIndex]; + FastCopyFile(m_Options.AllowFileClone, + m_Options.UseSparseFiles, + FirstTargetFilePath, + TargetFilePath, + RawSize, + m_DiskStats.WriteCount, + m_DiskStats.WriteByteCount, + m_DiskStats.CloneCount, + m_DiskStats.CloneByteCount); + + m_RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + } + + OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; + OutLocalFolderState.RawSizes[RemotePathIndex] = m_RemoteContent.RawSizes[RemotePathIndex]; + + OutLocalFolderState.Attributes[RemotePathIndex] = + m_RemoteContent.Attributes.empty() + ? GetNativeFileAttributes(TargetFilePath) + : SetNativeFileAttributes(TargetFilePath, m_RemoteContent.Platform, m_RemoteContent.Attributes[RemotePathIndex]); + OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); + + TargetOffset++; + TargetsComplete++; + } + } +} + +std::vector +BuildsOperationUpdateFolder::FindScavengeSources() +{ + ZEN_TRACE_CPU("FindScavengeSources"); + + const bool TargetPathExists = IsDir(m_Path); + + std::vector StatePaths = GetDownloadedStatePaths(m_Options.SystemRootDir); + + std::vector Result; + for (const std::filesystem::path& EntryPath : StatePaths) + { + if (IsFile(EntryPath)) + { + bool DeleteEntry = false; + + try + { + BuildsDownloadInfo Info = ReadDownloadedInfoFile(EntryPath); + const bool LocalPathExists = !Info.LocalPath.empty() && IsDir(Info.LocalPath); + const bool LocalStateFileExists = IsFile(Info.StateFilePath); + if (LocalPathExists && LocalStateFileExists) + { + if (TargetPathExists && std::filesystem::equivalent(Info.LocalPath, m_Path)) + { + DeleteEntry = true; + } + else + { + Result.push_back({.StateFilePath = std::move(Info.StateFilePath), .Path = std::move(Info.LocalPath)}); + } + } + else + { + DeleteEntry = true; + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("{}", Ex.what()); + DeleteEntry = true; + } + + if (DeleteEntry) + { + std::error_code DummyEc; + std::filesystem::remove(EntryPath, DummyEc); + } + } + } + return Result; +} + +std::vector +BuildsOperationUpdateFolder::ScanTargetFolder(const tsl::robin_map& CachedChunkHashesFound, + const tsl::robin_map& CachedSequenceHashesFound) +{ + ZEN_TRACE_CPU("ScanTargetFolder"); + + Stopwatch LocalTimer; + + std::vector MissingSequenceIndexes; + + for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < m_RemoteContent.ChunkedContent.SequenceRawHashes.size(); + RemoteSequenceIndex++) + { + const IoHash& RemoteSequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(m_RemoteLookup, RemoteSequenceIndex); + const uint64_t RemoteRawSize = m_RemoteContent.RawSizes[RemotePathIndex]; + if (auto CacheSequenceIt = CachedSequenceHashesFound.find(RemoteSequenceRawHash); + CacheSequenceIt != CachedSequenceHashesFound.end()) + { + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); + if (m_Options.IsVerbose) + { + ZEN_INFO("Found sequence {} at {} ({})", RemoteSequenceRawHash, CacheFilePath, NiceBytes(RemoteRawSize)); + } + } + else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash); CacheChunkIt != CachedChunkHashesFound.end()) + { + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); + if (m_Options.IsVerbose) + { + ZEN_INFO("Found chunk {} at {} ({})", RemoteSequenceRawHash, CacheFilePath, NiceBytes(RemoteRawSize)); + } + } + else if (auto It = m_LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash); + It != m_LocalLookup.RawHashToSequenceIndex.end()) + { + const uint32_t LocalSequenceIndex = It->second; + const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(m_LocalLookup, LocalSequenceIndex); + const std::filesystem::path LocalFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred(); + ZEN_ASSERT_SLOW(IsFile(LocalFilePath)); + m_CacheMappingStats.LocalPathsMatchingSequencesCount++; + m_CacheMappingStats.LocalPathsMatchingSequencesByteCount += RemoteRawSize; + if (m_Options.IsVerbose) + { + ZEN_INFO("Found sequence {} at {} ({})", RemoteSequenceRawHash, LocalFilePath, NiceBytes(RemoteRawSize)); + } + } + else + { + MissingSequenceIndexes.push_back(RemoteSequenceIndex); + } + } + + m_CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs(); + return MissingSequenceIndexes; +} + +bool +BuildsOperationUpdateFolder::FindScavengeContent(const ScavengeSource& Source, + ChunkedFolderContent& OutScavengedLocalContent, + ChunkedContentLookup& OutScavengedLookup) +{ + ZEN_TRACE_CPU("FindScavengeContent"); + + FolderContent LocalFolderState; + try + { + BuildSaveState SavedState = ReadBuildSaveStateFile(Source.StateFilePath); + if (SavedState.Version == BuildSaveState::NoVersion) + { + ZEN_DEBUG("Skipping old build state at '{}', state files before version {} can not be trusted during scavenge", + Source.StateFilePath, + BuildSaveState::kVersion1); + return false; + } + OutScavengedLocalContent = std::move(SavedState.State.ChunkedContent); + LocalFolderState = std::move(SavedState.FolderState); + } + catch (const std::exception& Ex) + { + ZEN_DEBUG("Skipping invalid build state at '{}', reason: {}", Source.StateFilePath, Ex.what()); + return false; + } + + tsl::robin_set PathIndexesToScavenge; + PathIndexesToScavenge.reserve(OutScavengedLocalContent.Paths.size()); + std::vector ChunkOrderOffsets = BuildChunkOrderOffset(OutScavengedLocalContent.ChunkedContent.ChunkCounts); + + { + tsl::robin_map RawHashToPathIndex; + + RawHashToPathIndex.reserve(OutScavengedLocalContent.Paths.size()); + for (uint32_t ScavengedPathIndex = 0; ScavengedPathIndex < OutScavengedLocalContent.RawHashes.size(); ScavengedPathIndex++) + { + if (!RawHashToPathIndex.contains(OutScavengedLocalContent.RawHashes[ScavengedPathIndex])) + { + RawHashToPathIndex.insert_or_assign(OutScavengedLocalContent.RawHashes[ScavengedPathIndex], ScavengedPathIndex); + } + } + + for (uint32_t ScavengeSequenceIndex = 0; ScavengeSequenceIndex < OutScavengedLocalContent.ChunkedContent.SequenceRawHashes.size(); + ScavengeSequenceIndex++) + { + const IoHash& SequenceHash = OutScavengedLocalContent.ChunkedContent.SequenceRawHashes[ScavengeSequenceIndex]; + if (auto It = RawHashToPathIndex.find(SequenceHash); It != RawHashToPathIndex.end()) + { + uint32_t PathIndex = It->second; + if (!PathIndexesToScavenge.contains(PathIndex)) + { + if (m_RemoteLookup.RawHashToSequenceIndex.contains(SequenceHash)) + { + PathIndexesToScavenge.insert(PathIndex); + } + else + { + uint32_t ChunkOrderIndexStart = ChunkOrderOffsets[ScavengeSequenceIndex]; + const uint32_t ChunkCount = OutScavengedLocalContent.ChunkedContent.ChunkCounts[ScavengeSequenceIndex]; + for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < ChunkCount; ChunkOrderIndex++) + { + const uint32_t ChunkIndex = + OutScavengedLocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndexStart + ChunkOrderIndex]; + const IoHash& ChunkHash = OutScavengedLocalContent.ChunkedContent.ChunkHashes[ChunkIndex]; + if (m_RemoteLookup.ChunkHashToChunkIndex.contains(ChunkHash)) + { + PathIndexesToScavenge.insert(PathIndex); + break; + } + } + } + } + } + else + { + ZEN_WARN("Scavenged state file at '{}' for '{}' is invalid, skipping scavenging for sequence {}", + Source.StateFilePath, + Source.Path, + SequenceHash); + } + } + } + + if (PathIndexesToScavenge.empty()) + { + OutScavengedLocalContent = {}; + return false; + } + + std::vector PathsToScavenge; + PathsToScavenge.reserve(PathIndexesToScavenge.size()); + for (uint32_t ScavengedStatePathIndex : PathIndexesToScavenge) + { + PathsToScavenge.push_back(OutScavengedLocalContent.Paths[ScavengedStatePathIndex]); + } + + FolderContent ValidFolderContent = + GetValidFolderContent(m_IOWorkerPool, m_ScavengedFolderScanStats, Source.Path, PathsToScavenge, {}, 0, m_AbortFlag, m_PauseFlag); + + if (!LocalFolderState.AreKnownFilesEqual(ValidFolderContent)) + { + std::vector DeletedPaths; + FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, ValidFolderContent, DeletedPaths); + + // If the files are modified since the state was saved we ignore the files since we don't + // want to incur the cost of scanning/hashing scavenged files + DeletedPaths.insert(DeletedPaths.end(), UpdatedContent.Paths.begin(), UpdatedContent.Paths.end()); + if (!DeletedPaths.empty()) + { + OutScavengedLocalContent = + DeletePathsFromChunkedContent(OutScavengedLocalContent, + BuildHashLookup(OutScavengedLocalContent.ChunkedContent.SequenceRawHashes), + ChunkOrderOffsets, + DeletedPaths); + } + } + + if (OutScavengedLocalContent.Paths.empty()) + { + OutScavengedLocalContent = {}; + return false; + } + + OutScavengedLookup = BuildChunkedContentLookup(OutScavengedLocalContent); + + return true; +} + +void +BuildsOperationUpdateFolder::ScavengeSourceForChunks(uint32_t& InOutRemainingChunkCount, + std::vector& InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags, + tsl::robin_map& InOutRawHashToCopyChunkDataIndex, + const std::vector>& SequenceIndexChunksLeftToWriteCounters, + const ChunkedFolderContent& ScavengedContent, + const ChunkedContentLookup& ScavengedLookup, + std::vector& InOutCopyChunkDatas, + uint32_t ScavengedContentIndex, + uint64_t& InOutChunkMatchingRemoteCount, + uint64_t& InOutChunkMatchingRemoteByteCount) +{ + for (uint32_t RemoteChunkIndex = 0; + RemoteChunkIndex < m_RemoteContent.ChunkedContent.ChunkHashes.size() && (InOutRemainingChunkCount > 0); + RemoteChunkIndex++) + { + if (!InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) + { + const IoHash& RemoteChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + if (auto It = ScavengedLookup.ChunkHashToChunkIndex.find(RemoteChunkHash); It != ScavengedLookup.ChunkHashToChunkIndex.end()) + { + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex); + + if (!ChunkTargetPtrs.empty()) + { + const uint32_t ScavengedChunkIndex = It->second; + const uint64_t ScavengedChunkRawSize = ScavengedContent.ChunkedContent.ChunkRawSizes[ScavengedChunkIndex]; + const size_t ChunkSequenceLocationOffset = ScavengedLookup.ChunkSequenceLocationOffset[ScavengedChunkIndex]; + const ChunkedContentLookup::ChunkSequenceLocation& ScavengeLocation = + ScavengedLookup.ChunkSequenceLocations[ChunkSequenceLocationOffset]; + const IoHash& ScavengedSequenceRawHash = + ScavengedContent.ChunkedContent.SequenceRawHashes[ScavengeLocation.SequenceIndex]; + + CopyChunkData::ChunkTarget Target = {.TargetChunkLocationCount = gsl::narrow(ChunkTargetPtrs.size()), + .RemoteChunkIndex = RemoteChunkIndex, + .CacheFileOffset = ScavengeLocation.Offset}; + if (auto CopySourceIt = InOutRawHashToCopyChunkDataIndex.find(ScavengedSequenceRawHash); + CopySourceIt != InOutRawHashToCopyChunkDataIndex.end()) + { + CopyChunkData& Data = InOutCopyChunkDatas[CopySourceIt->second]; + if (Data.TargetChunkLocationPtrs.size() > 1024) + { + InOutRawHashToCopyChunkDataIndex.insert_or_assign(ScavengedSequenceRawHash, InOutCopyChunkDatas.size()); + InOutCopyChunkDatas.push_back(CopyChunkData{.ScavengeSourceIndex = ScavengedContentIndex, + .SourceSequenceIndex = ScavengeLocation.SequenceIndex, + .TargetChunkLocationPtrs = ChunkTargetPtrs, + .ChunkTargets = std::vector{Target}}); + } + else + { + Data.TargetChunkLocationPtrs.insert(Data.TargetChunkLocationPtrs.end(), + ChunkTargetPtrs.begin(), + ChunkTargetPtrs.end()); + Data.ChunkTargets.push_back(Target); + } + } + else + { + InOutRawHashToCopyChunkDataIndex.insert_or_assign(ScavengedSequenceRawHash, InOutCopyChunkDatas.size()); + InOutCopyChunkDatas.push_back(CopyChunkData{.ScavengeSourceIndex = ScavengedContentIndex, + .SourceSequenceIndex = ScavengeLocation.SequenceIndex, + .TargetChunkLocationPtrs = ChunkTargetPtrs, + .ChunkTargets = std::vector{Target}}); + } + InOutChunkMatchingRemoteCount++; + InOutChunkMatchingRemoteByteCount += ScavengedChunkRawSize; + InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true; + InOutRemainingChunkCount--; + } + } + } + } +} + +std::filesystem::path +BuildsOperationUpdateFolder::FindDownloadedChunk(const IoHash& ChunkHash) +{ + ZEN_TRACE_CPU("FindDownloadedChunk"); + + std::filesystem::path CompressedChunkPath = m_TempDownloadFolderPath / ChunkHash.ToHexString(); + if (IsFile(CompressedChunkPath)) + { + IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); + if (ExistingCompressedPart) + { + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart, + RawHash, + RawSize, + /*OutOptionalTotalCompressedSize*/ nullptr)) + { + return CompressedChunkPath; + } + else + { + std::error_code DummyEc; + RemoveFile(CompressedChunkPath, DummyEc); + } + } + } + return {}; +} + +std::vector +BuildsOperationUpdateFolder::GetRemainingChunkTargets(std::span> SequenceIndexChunksLeftToWriteCounters, + uint32_t ChunkIndex) +{ + ZEN_TRACE_CPU("GetRemainingChunkTargets"); + + std::span ChunkSources = GetChunkSequenceLocations(m_RemoteLookup, ChunkIndex); + std::vector ChunkTargetPtrs; + if (!ChunkSources.empty()) + { + ChunkTargetPtrs.reserve(ChunkSources.size()); + for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources) + { + if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0) + { + ChunkTargetPtrs.push_back(&Source); + } + } + } + return ChunkTargetPtrs; +}; + +uint64_t +BuildsOperationUpdateFolder::GetChunkWriteCount(std::span> SequenceIndexChunksLeftToWriteCounters, + uint32_t ChunkIndex) +{ + ZEN_TRACE_CPU("GetChunkWriteCount"); + + uint64_t WriteCount = 0; + std::span ChunkSources = GetChunkSequenceLocations(m_RemoteLookup, ChunkIndex); + for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources) + { + if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0) + { + WriteCount++; + } + } + return WriteCount; +}; + +void +BuildsOperationUpdateFolder::CheckRequiredDiskSpace(const tsl::robin_map& RemotePathToRemoteIndex) +{ + tsl::robin_set ExistingRemotePaths; + + if (m_Options.EnableTargetFolderScavenging) + { + for (uint32_t LocalPathIndex = 0; LocalPathIndex < m_LocalContent.Paths.size(); LocalPathIndex++) + { + const IoHash& RawHash = m_LocalContent.RawHashes[LocalPathIndex]; + const std::filesystem::path& LocalPath = m_LocalContent.Paths[LocalPathIndex]; + + if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string()); RemotePathIt != RemotePathToRemoteIndex.end()) + { + const uint32_t RemotePathIndex = RemotePathIt->second; + if (m_RemoteContent.RawHashes[RemotePathIndex] == RawHash) + { + ExistingRemotePaths.insert(RemotePathIndex); + } + } + } + } + + uint64_t RequiredSpace = 0; + for (uint32_t RemotePathIndex = 0; RemotePathIndex < m_RemoteContent.Paths.size(); RemotePathIndex++) + { + if (!ExistingRemotePaths.contains(RemotePathIndex)) + { + RequiredSpace += m_RemoteContent.RawSizes[RemotePathIndex]; + } + } + + std::error_code Ec; + DiskSpace Space = DiskSpaceInfo(m_Path, Ec); + if (Ec) + { + throw std::runtime_error(fmt::format("Get free disk space for target path '{}' FAILED, reason: {}", m_Path, Ec.message())); + } + if (Space.Free < (RequiredSpace + 16u * 1024u * 1024u)) + { + throw std::runtime_error( + fmt::format("Not enough free space for target path '{}', {} of free space is needed but only {} is available", + m_Path, + NiceBytes(RequiredSpace), + NiceBytes(Space.Free))); + } +} + +void +BuildsOperationUpdateFolder::WriteScavengedSequenceToCache(const std::filesystem::path& ScavengeRootPath, + const ChunkedFolderContent& ScavengedContent, + const ScavengedSequenceCopyOperation& ScavengeOp) +{ + ZEN_TRACE_CPU("WriteScavengedSequenceToCache"); + + const std::filesystem::path ScavengedPath = ScavengedContent.Paths[ScavengeOp.ScavengedPathIndex]; + const std::filesystem::path ScavengedFilePath = (ScavengeRootPath / ScavengedPath).make_preferred(); + ZEN_ASSERT_SLOW(FileSizeFromPath(ScavengedFilePath) == ScavengeOp.RawSize); + + const IoHash& RemoteSequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[ScavengeOp.RemoteSequenceIndex]; + const std::filesystem::path TempFilePath = GetTempChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash); + + const uint64_t RawSize = ScavengedContent.RawSizes[ScavengeOp.ScavengedPathIndex]; + FastCopyFile(m_Options.AllowFileClone, + m_Options.UseSparseFiles, + ScavengedFilePath, + TempFilePath, + RawSize, + m_DiskStats.WriteCount, + m_DiskStats.WriteByteCount, + m_DiskStats.CloneCount, + m_DiskStats.CloneByteCount); + + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash); + RenameFile(TempFilePath, CacheFilePath); + + m_WrittenChunkByteCount += RawSize; + if (m_Options.ValidateCompletedSequences) + { + m_ValidatedChunkByteCount += RawSize; + } +} + +void +BuildsOperationUpdateFolder::WriteLooseChunk(const uint32_t RemoteChunkIndex, + const BlobsExistsResult& ExistsResult, + std::span> SequenceIndexChunksLeftToWriteCounters, + std::atomic& WritePartsComplete, + std::vector&& ChunkTargetPtrs, + BufferedWriteFileCache& WriteCache, + ParallelWork& Work, + uint64_t TotalRequestCount, + uint64_t TotalPartWriteCount, + FilteredRate& FilteredDownloadedBytesPerSecond, + FilteredRate& FilteredWrittenBytesPerSecond) +{ + const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + std::filesystem::path ExistingCompressedChunkPath = FindDownloadedChunk(ChunkHash); + if (!ExistingCompressedChunkPath.empty()) + { + if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + } + if (!m_AbortFlag) + { + if (!ExistingCompressedChunkPath.empty()) + { + Work.ScheduleWork( + m_IOWorkerPool, + [this, + SequenceIndexChunksLeftToWriteCounters, + &WriteCache, + &Work, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + RemoteChunkIndex, + ChunkTargetPtrs = std::move(ChunkTargetPtrs), + CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic& AbortFlag) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("Async_WritePreDownloadedChunk"); + + FilteredWrittenBytesPerSecond.Start(); + + const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + + IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); + if (!CompressedPart) + { + throw std::runtime_error( + fmt::format("Could not open dowloaded compressed chunk {} from {}", ChunkHash, CompressedChunkPath)); + } + + bool NeedHashVerify = + WriteCompressedChunkToCache(ChunkHash, ChunkTargetPtrs, WriteCache, std::move(CompressedPart)); + bool WritePartsDone = WritePartsComplete.fetch_add(1) + 1 == TotalPartWriteCount; + + if (!AbortFlag) + { + if (WritePartsDone) + { + FilteredWrittenBytesPerSecond.Stop(); + } + + std::error_code Ec = TryRemoveFile(CompressedChunkPath); + if (Ec) + { + ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", CompressedChunkPath, Ec.value(), Ec.message()); + } + + std::vector CompletedSequences = + CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); + WriteCache.Close(CompletedSequences); + if (NeedHashVerify) + { + VerifyAndCompleteChunkSequencesAsync(CompletedSequences, Work); + } + else + { + FinalizeChunkSequences(CompletedSequences); + } + } + } + }); + } + else + { + Work.ScheduleWork(m_NetworkPool, + [this, + &ExistsResult, + SequenceIndexChunksLeftToWriteCounters, + &WriteCache, + &Work, + &WritePartsComplete, + TotalPartWriteCount, + TotalRequestCount, + &FilteredDownloadedBytesPerSecond, + &FilteredWrittenBytesPerSecond, + RemoteChunkIndex, + ChunkTargetPtrs = std::vector( + std::move(ChunkTargetPtrs))](std::atomic&) mutable { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("Async_DownloadChunk"); + + FilteredDownloadedBytesPerSecond.Start(); + DownloadBuildBlob(RemoteChunkIndex, + ExistsResult, + Work, + TotalRequestCount, + FilteredDownloadedBytesPerSecond, + [this, + &ExistsResult, + SequenceIndexChunksLeftToWriteCounters, + &WriteCache, + &Work, + &WritePartsComplete, + TotalPartWriteCount, + RemoteChunkIndex, + &FilteredWrittenBytesPerSecond, + ChunkTargetPtrs = std::move(ChunkTargetPtrs)](IoBuffer&& Payload) mutable { + AsyncWriteDownloadedChunk(RemoteChunkIndex, + ExistsResult, + std::move(ChunkTargetPtrs), + WriteCache, + Work, + std::move(Payload), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond); + }); + } + }); + } + } +} + +void +BuildsOperationUpdateFolder::DownloadBuildBlob(uint32_t RemoteChunkIndex, + const BlobsExistsResult& ExistsResult, + ParallelWork& Work, + uint64_t TotalRequestCount, + FilteredRate& FilteredDownloadedBytesPerSecond, + std::function&& OnDownloaded) +{ + const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + // FilteredDownloadedBytesPerSecond.Start(); + IoBuffer BuildBlob; + const bool ExistsInCache = m_Storage.CacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash); + if (ExistsInCache) + { + BuildBlob = m_Storage.CacheStorage->GetBuildBlob(m_BuildId, ChunkHash); + } + if (BuildBlob) + { + uint64_t BlobSize = BuildBlob.GetSize(); + m_DownloadStats.DownloadedChunkCount++; + m_DownloadStats.DownloadedChunkByteCount += BlobSize; + if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + OnDownloaded(std::move(BuildBlob)); + } + else + { + if (m_RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= m_Options.LargeAttachmentSize) + { + DownloadLargeBlob( + *m_Storage.BuildStorage, + m_TempDownloadFolderPath, + m_BuildId, + ChunkHash, + m_Options.PreferredMultipartChunkSize, + Work, + m_NetworkPool, + m_DownloadStats.DownloadedChunkByteCount, + m_DownloadStats.MultipartAttachmentCount, + [this, &FilteredDownloadedBytesPerSecond, TotalRequestCount, OnDownloaded = std::move(OnDownloaded)](IoBuffer&& Payload) { + m_DownloadStats.DownloadedChunkCount++; + if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + + OnDownloaded(std::move(Payload)); + }); + } + else + { + try + { + BuildBlob = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, ChunkHash); + } + catch (const std::exception&) + { + // Silence http errors due to abort + if (!m_AbortFlag) + { + throw; + } + } + if (!m_AbortFlag) + { + if (!BuildBlob) + { + throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); + } + + if (!m_AbortFlag) + { + uint64_t BlobSize = BuildBlob.GetSize(); + m_DownloadStats.DownloadedChunkCount++; + m_DownloadStats.DownloadedChunkByteCount += BlobSize; + if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + + OnDownloaded(std::move(BuildBlob)); + } + } + } + } +} + +void +BuildsOperationUpdateFolder::DownloadPartialBlock( + std::span BlockRanges, + size_t BlockRangeStartIndex, + size_t BlockRangeCount, + const BlobsExistsResult& ExistsResult, + uint64_t TotalRequestCount, + FilteredRate& FilteredDownloadedBytesPerSecond, + std::function> OffsetAndLengths)>&& OnDownloaded) +{ + const uint32_t BlockIndex = BlockRanges[BlockRangeStartIndex].BlockIndex; + + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; + + auto ProcessDownload = [this]( + const ChunkBlockDescription& BlockDescription, + IoBuffer&& BlockRangeBuffer, + size_t BlockRangeStartIndex, + std::span> BlockOffsetAndLengths, + uint64_t TotalRequestCount, + FilteredRate& FilteredDownloadedBytesPerSecond, + const std::function> OffsetAndLengths)>& OnDownloaded) { + uint64_t BlockRangeBufferSize = BlockRangeBuffer.GetSize(); + m_DownloadStats.DownloadedBlockCount++; + m_DownloadStats.DownloadedBlockByteCount += BlockRangeBufferSize; + if (m_DownloadStats.RequestsCompleteCount.fetch_add(BlockOffsetAndLengths.size()) + BlockOffsetAndLengths.size() == + TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + + IoHashStream RangeId; + for (const std::pair& Range : BlockOffsetAndLengths) + { + RangeId.Append(&Range.first, sizeof(uint64_t)); + RangeId.Append(&Range.second, sizeof(uint64_t)); + } + std::filesystem::path BlockChunkPath = + TryMoveDownloadedChunk(BlockRangeBuffer, + m_TempBlockFolderPath / fmt::format("{}_{}", BlockDescription.BlockHash, RangeId.GetHash()), + /* ForceDiskBased */ BlockRangeBufferSize > m_Options.MaximumInMemoryPayloadSize); + + if (!m_AbortFlag) + { + OnDownloaded(std::move(BlockRangeBuffer), std::move(BlockChunkPath), BlockRangeStartIndex, BlockOffsetAndLengths); + } + }; + + std::vector> Ranges; + Ranges.reserve(BlockRangeCount); + for (size_t BlockRangeIndex = BlockRangeStartIndex; BlockRangeIndex < BlockRangeStartIndex + BlockRangeCount; BlockRangeIndex++) + { + const ChunkBlockAnalyser::BlockRangeDescriptor& BlockRange = BlockRanges[BlockRangeIndex]; + Ranges.push_back(std::make_pair(BlockRange.RangeStart, BlockRange.RangeLength)); + } + + const bool ExistsInCache = m_Storage.CacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); + + size_t SubBlockRangeCount = BlockRangeCount; + size_t SubRangeCountComplete = 0; + std::span> RangesSpan(Ranges); + while (SubRangeCountComplete < SubBlockRangeCount) + { + if (m_AbortFlag) + { + break; + } + + // First try to get subrange from cache. + // If not successful, try to get the ranges from the build store and adapt SubRangeCount... + + size_t SubRangeStartIndex = BlockRangeStartIndex + SubRangeCountComplete; + if (ExistsInCache) + { + size_t SubRangeCount = Min(BlockRangeCount - SubRangeCountComplete, m_Storage.CacheHost.Caps.MaxRangeCountPerRequest); + + if (SubRangeCount == 1) + { + // Legacy single-range path, prefer that for max compatibility + + const std::pair SubRange = RangesSpan[SubRangeCountComplete]; + IoBuffer PayloadBuffer = + m_Storage.CacheStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash, SubRange.first, SubRange.second); + if (m_AbortFlag) + { + break; + } + if (PayloadBuffer) + { + ProcessDownload(BlockDescription, + std::move(PayloadBuffer), + SubRangeStartIndex, + std::vector>{std::make_pair(0u, SubRange.second)}, + TotalRequestCount, + FilteredDownloadedBytesPerSecond, + OnDownloaded); + SubRangeCountComplete += SubRangeCount; + continue; + } + } + else + { + auto SubRanges = RangesSpan.subspan(SubRangeCountComplete, SubRangeCount); + + BuildStorageCache::BuildBlobRanges RangeBuffers = + m_Storage.CacheStorage->GetBuildBlobRanges(m_BuildId, BlockDescription.BlockHash, SubRanges); + if (m_AbortFlag) + { + break; + } + if (RangeBuffers.PayloadBuffer) + { + if (RangeBuffers.Ranges.empty()) + { + SubRangeCount = Ranges.size() - SubRangeCountComplete; + ProcessDownload(BlockDescription, + std::move(RangeBuffers.PayloadBuffer), + SubRangeStartIndex, + RangesSpan.subspan(SubRangeCountComplete, SubRangeCount), + TotalRequestCount, + FilteredDownloadedBytesPerSecond, + OnDownloaded); + SubRangeCountComplete += SubRangeCount; + continue; + } + else if (RangeBuffers.Ranges.size() == SubRangeCount) + { + ProcessDownload(BlockDescription, + std::move(RangeBuffers.PayloadBuffer), + SubRangeStartIndex, + RangeBuffers.Ranges, + TotalRequestCount, + FilteredDownloadedBytesPerSecond, + OnDownloaded); + SubRangeCountComplete += SubRangeCount; + continue; + } + } + } + } + + size_t SubRangeCount = Min(BlockRangeCount - SubRangeCountComplete, m_Storage.BuildStorageHost.Caps.MaxRangeCountPerRequest); + + auto SubRanges = RangesSpan.subspan(SubRangeCountComplete, SubRangeCount); + + BuildStorageBase::BuildBlobRanges RangeBuffers; + + try + { + RangeBuffers = m_Storage.BuildStorage->GetBuildBlobRanges(m_BuildId, BlockDescription.BlockHash, SubRanges); + } + catch (const std::exception&) + { + // Silence http errors due to abort + if (!m_AbortFlag) + { + throw; + } + } + + if (!m_AbortFlag) + { + if (RangeBuffers.PayloadBuffer) + { + if (RangeBuffers.Ranges.empty()) + { + // Jupiter will ignore the ranges and send the whole payload if it fetches the payload from S3 + // Upload to cache (if enabled) and use the whole payload for the remaining ranges + + const uint64_t Size = RangeBuffers.PayloadBuffer.GetSize(); + + const bool PopulateCache = !ExistsInCache && m_Storage.CacheStorage && m_Options.PopulateCache; + + std::filesystem::path BlockPath = + TryMoveDownloadedChunk(RangeBuffers.PayloadBuffer, + m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString(), + /* ForceDiskBased */ PopulateCache || Size > m_Options.MaximumInMemoryPayloadSize); + if (!BlockPath.empty()) + { + RangeBuffers.PayloadBuffer = IoBufferBuilder::MakeFromFile(BlockPath); + if (!RangeBuffers.PayloadBuffer) + { + throw std::runtime_error( + fmt::format("Failed to read block {} from temporary path '{}'", BlockDescription.BlockHash, BlockPath)); + } + RangeBuffers.PayloadBuffer.SetDeleteOnClose(true); + } + + if (PopulateCache) + { + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, + BlockDescription.BlockHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(RangeBuffers.PayloadBuffer))); + } + + if (m_AbortFlag) + { + break; + } + + SubRangeCount = Ranges.size() - SubRangeCountComplete; + ProcessDownload(BlockDescription, + std::move(RangeBuffers.PayloadBuffer), + SubRangeStartIndex, + RangesSpan.subspan(SubRangeCountComplete, SubRangeCount), + TotalRequestCount, + FilteredDownloadedBytesPerSecond, + OnDownloaded); + } + else + { + if (RangeBuffers.Ranges.size() != SubRanges.size()) + { + throw std::runtime_error(fmt::format("Fetching {} ranges from {} resulted in {} ranges", + SubRanges.size(), + BlockDescription.BlockHash, + RangeBuffers.Ranges.size())); + } + ProcessDownload(BlockDescription, + std::move(RangeBuffers.PayloadBuffer), + SubRangeStartIndex, + RangeBuffers.Ranges, + TotalRequestCount, + FilteredDownloadedBytesPerSecond, + OnDownloaded); + } + } + else + { + throw std::runtime_error( + fmt::format("Block {} is missing when fetching {} ranges", BlockDescription.BlockHash, SubRangeCount)); + } + + SubRangeCountComplete += SubRangeCount; + } + } +} + +std::vector +BuildsOperationUpdateFolder::WriteLocalChunkToCache(CloneQueryInterface* CloneQuery, + const CopyChunkData& CopyData, + const std::vector& ScavengedContents, + const std::vector& ScavengedLookups, + const std::vector& ScavengedPaths, + BufferedWriteFileCache& WriteCache) +{ + ZEN_TRACE_CPU("WriteLocalChunkToCache"); + + std::filesystem::path SourceFilePath; + + if (CopyData.ScavengeSourceIndex == (uint32_t)-1) + { + const uint32_t LocalPathIndex = m_LocalLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; + SourceFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred(); + } + else + { + const ChunkedFolderContent& ScavengedContent = ScavengedContents[CopyData.ScavengeSourceIndex]; + const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[CopyData.ScavengeSourceIndex]; + const std::filesystem::path ScavengedPath = ScavengedPaths[CopyData.ScavengeSourceIndex]; + const uint32_t ScavengedPathIndex = ScavengedLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; + SourceFilePath = (ScavengedPath / ScavengedContent.Paths[ScavengedPathIndex]).make_preferred(); + } + ZEN_ASSERT_SLOW(IsFile(SourceFilePath)); + ZEN_ASSERT(!CopyData.TargetChunkLocationPtrs.empty()); + + uint64_t CacheLocalFileBytesRead = 0; + + size_t TargetStart = 0; + const std::span AllTargets(CopyData.TargetChunkLocationPtrs); + + struct WriteOp + { + const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr; + uint64_t CacheFileOffset = (uint64_t)-1; + uint32_t ChunkIndex = (uint32_t)-1; + }; + + std::vector WriteOps; + + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("Sort"); + WriteOps.reserve(AllTargets.size()); + for (const CopyChunkData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) + { + std::span TargetRange = + AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); + for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange) + { + WriteOps.push_back( + WriteOp{.Target = Target, .CacheFileOffset = ChunkTarget.CacheFileOffset, .ChunkIndex = ChunkTarget.RemoteChunkIndex}); + } + TargetStart += ChunkTarget.TargetChunkLocationCount; + } + + std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { + if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) + { + return true; + } + else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) + { + return false; + } + if (Lhs.Target->Offset < Rhs.Target->Offset) + { + return true; + } + return false; + }); + } + + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("Write"); + + tsl::robin_set ChunkIndexesWritten; + + BufferedOpenFile SourceFile(SourceFilePath, + m_DiskStats.OpenReadCount, + m_DiskStats.CurrentOpenFileCount, + m_DiskStats.ReadCount, + m_DiskStats.ReadByteCount); + + bool CanCloneSource = CloneQuery && CloneQuery->CanClone(SourceFile.Handle()); + + BufferedWriteFileCache::Local LocalWriter(WriteCache); + + for (size_t WriteOpIndex = 0; WriteOpIndex < WriteOps.size();) + { + if (m_AbortFlag) + { + break; + } + const WriteOp& Op = WriteOps[WriteOpIndex]; + + const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; + const uint32_t RemotePathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; + const uint64_t TargetSize = m_RemoteContent.RawSizes[RemotePathIndex]; + const uint64_t ChunkSize = m_RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex]; + + uint64_t ReadLength = ChunkSize; + size_t WriteCount = 1; + uint64_t OpSourceEnd = Op.CacheFileOffset + ChunkSize; + uint64_t OpTargetEnd = Op.Target->Offset + ChunkSize; + while ((WriteOpIndex + WriteCount) < WriteOps.size()) + { + const WriteOp& NextOp = WriteOps[WriteOpIndex + WriteCount]; + if (NextOp.Target->SequenceIndex != Op.Target->SequenceIndex) + { + break; + } + if (NextOp.Target->Offset != OpTargetEnd) + { + break; + } + if (NextOp.CacheFileOffset != OpSourceEnd) + { + break; + } + const uint64_t NextChunkLength = m_RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex]; + if (ReadLength + NextChunkLength > BufferedOpenFile::BlockSize) + { + break; + } + ReadLength += NextChunkLength; + OpSourceEnd += NextChunkLength; + OpTargetEnd += NextChunkLength; + WriteCount++; + } + + { + bool DidClone = false; + + if (CanCloneSource) + { + uint64_t PreBytes = 0; + uint64_t PostBytes = 0; + uint64_t ClonableBytes = + CloneQuery->GetClonableRange(Op.CacheFileOffset, Op.Target->Offset, ReadLength, PreBytes, PostBytes); + if (ClonableBytes > 0) + { + // We need to open the file... + BufferedWriteFileCache::Local::Writer* Writer = LocalWriter.GetWriter(RemoteSequenceIndex); + if (!Writer) + { + Writer = LocalWriter.PutWriter(RemoteSequenceIndex, std::make_unique()); + + Writer->File = std::make_unique(); + + const std::filesystem::path FileName = + GetTempChunkedSequenceFileName(m_CacheFolderPath, + m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]); + Writer->File->Open(FileName, BasicFile::Mode::kWrite); + if (m_Options.UseSparseFiles) + { + PrepareFileForScatteredWrite(Writer->File->Handle(), TargetSize); + } + } + DidClone = CloneQuery->TryClone(SourceFile.Handle(), + Writer->File->Handle(), + Op.CacheFileOffset + PreBytes, + Op.Target->Offset + PreBytes, + ClonableBytes, + TargetSize); + if (DidClone) + { + m_DiskStats.WriteCount++; + m_DiskStats.WriteByteCount += ClonableBytes; + + m_DiskStats.CloneCount++; + m_DiskStats.CloneByteCount += ClonableBytes; + + m_WrittenChunkByteCount += ClonableBytes; + + if (PreBytes > 0) + { + CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, PreBytes); + const uint64_t FileOffset = Op.Target->Offset; + + WriteSequenceChunkToCache(LocalWriter, ChunkSource, RemoteSequenceIndex, FileOffset, RemotePathIndex); + } + if (PostBytes > 0) + { + CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset + ReadLength - PostBytes, PostBytes); + const uint64_t FileOffset = Op.Target->Offset + ReadLength - PostBytes; + + WriteSequenceChunkToCache(LocalWriter, ChunkSource, RemoteSequenceIndex, FileOffset, RemotePathIndex); + } + } + } + } + + if (!DidClone) + { + CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ReadLength); + + const uint64_t FileOffset = Op.Target->Offset; + + WriteSequenceChunkToCache(LocalWriter, ChunkSource, RemoteSequenceIndex, FileOffset, RemotePathIndex); + } + } + + CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes? + + WriteOpIndex += WriteCount; + } + } + + if (m_Options.IsVerbose) + { + ZEN_INFO("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), SourceFilePath); + } + + std::vector Result; + Result.reserve(WriteOps.size()); + + for (const WriteOp& Op : WriteOps) + { + Result.push_back(Op.Target->SequenceIndex); + } + return Result; +} + +bool +BuildsOperationUpdateFolder::WriteCompressedChunkToCache( + const IoHash& ChunkHash, + const std::vector& ChunkTargetPtrs, + BufferedWriteFileCache& WriteCache, + IoBuffer&& CompressedPart) +{ + ZEN_TRACE_CPU("WriteCompressedChunkToCache"); + + auto ChunkHashToChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); + ZEN_ASSERT(ChunkHashToChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end()); + if (IsSingleFileChunk(m_RemoteContent, ChunkTargetPtrs)) + { + const std::uint32_t SequenceIndex = ChunkTargetPtrs.front()->SequenceIndex; + const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]; + StreamDecompress(SequenceRawHash, CompositeBuffer(std::move(CompressedPart))); + return false; + } + else + { + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompositeBuffer(std::move(CompressedPart)), RawHash, RawSize); + if (!Compressed) + { + throw std::runtime_error(fmt::format("Failed to parse header of compressed large blob {}", ChunkHash)); + } + if (RawHash != ChunkHash) + { + throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, ChunkHash)); + } + + BufferedWriteFileCache::Local LocalWriter(WriteCache); + + IoHashStream Hash; + bool CouldDecompress = Compressed.DecompressToStream( + 0, + (uint64_t)-1, + [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { + ZEN_UNUSED(SourceOffset); + ZEN_TRACE_CPU("Async_StreamDecompress_Write"); + m_DiskStats.ReadByteCount += SourceSize; + if (!m_AbortFlag) + { + for (const ChunkedContentLookup::ChunkSequenceLocation* TargetPtr : ChunkTargetPtrs) + { + const auto& Target = *TargetPtr; + const uint64_t FileOffset = Target.Offset + Offset; + const uint32_t SequenceIndex = Target.SequenceIndex; + const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + + WriteSequenceChunkToCache(LocalWriter, RangeBuffer, SequenceIndex, FileOffset, PathIndex); + } + + return true; + } + return false; + }); + + if (m_AbortFlag) + { + return false; + } + + if (!CouldDecompress) + { + throw std::runtime_error(fmt::format("Failed to decompress large chunk {}", ChunkHash)); + } + + return true; + } +} + +void +BuildsOperationUpdateFolder::StreamDecompress(const IoHash& SequenceRawHash, CompositeBuffer&& CompressedPart) +{ + ZEN_TRACE_CPU("StreamDecompress"); + const std::filesystem::path TempChunkSequenceFileName = GetTempChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash); + TemporaryFile DecompressedTemp; + std::error_code Ec; + DecompressedTemp.CreateTemporary(TempChunkSequenceFileName.parent_path(), Ec); + if (Ec) + { + throw std::runtime_error(fmt::format("Failed creating temporary file for decompressing large blob {}, reason: ({}) {}", + SequenceRawHash, + Ec.value(), + Ec.message())); + } + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompressedPart, RawHash, RawSize); + if (!Compressed) + { + throw std::runtime_error(fmt::format("Failed to parse header of compressed large blob {}", SequenceRawHash)); + } + if (RawHash != SequenceRawHash) + { + throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, SequenceRawHash)); + } + PrepareFileForScatteredWrite(DecompressedTemp.Handle(), RawSize); + + IoHashStream Hash; + bool CouldDecompress = + Compressed.DecompressToStream(0, + (uint64_t)-1, + [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { + ZEN_UNUSED(SourceOffset); + ZEN_TRACE_CPU("StreamDecompress_Write"); + m_DiskStats.ReadCount++; + m_DiskStats.ReadByteCount += SourceSize; + if (!m_AbortFlag) + { + for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) + { + if (m_Options.ValidateCompletedSequences) + { + Hash.Append(Segment.GetView()); + m_ValidatedChunkByteCount += Segment.GetSize(); + } + DecompressedTemp.Write(Segment, Offset); + Offset += Segment.GetSize(); + m_DiskStats.WriteByteCount += Segment.GetSize(); + m_DiskStats.WriteCount++; + m_WrittenChunkByteCount += Segment.GetSize(); + } + return true; + } + return false; + }); + + if (m_AbortFlag) + { + return; + } + + if (!CouldDecompress) + { + throw std::runtime_error(fmt::format("Failed to decompress large blob {}", SequenceRawHash)); + } + if (m_Options.ValidateCompletedSequences) + { + const IoHash VerifyHash = Hash.GetHash(); + if (VerifyHash != SequenceRawHash) + { + throw std::runtime_error( + fmt::format("Decompressed blob payload hash {} does not match expected hash {}", VerifyHash, SequenceRawHash)); + } + } + DecompressedTemp.MoveTemporaryIntoPlace(TempChunkSequenceFileName, Ec); + if (Ec) + { + throw std::runtime_error(fmt::format("Failed moving temporary file for decompressing large blob {}, reason: ({}) {}", + SequenceRawHash, + Ec.value(), + Ec.message())); + } + // WriteChunkStats.ChunkCountWritten++; +} + +void +BuildsOperationUpdateFolder::WriteSequenceChunkToCache(BufferedWriteFileCache::Local& LocalWriter, + const CompositeBuffer& Chunk, + const uint32_t SequenceIndex, + const uint64_t FileOffset, + const uint32_t PathIndex) +{ + ZEN_TRACE_CPU("WriteSequenceChunkToCache"); + + const uint64_t SequenceSize = m_RemoteContent.RawSizes[PathIndex]; + + auto OpenFile = [&](BasicFile& File) { + const std::filesystem::path FileName = + GetTempChunkedSequenceFileName(m_CacheFolderPath, m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + File.Open(FileName, BasicFile::Mode::kWrite); + if (m_Options.UseSparseFiles) + { + PrepareFileForScatteredWrite(File.Handle(), SequenceSize); + } + }; + + const uint64_t ChunkSize = Chunk.GetSize(); + ZEN_ASSERT(FileOffset + ChunkSize <= SequenceSize); + if (ChunkSize == SequenceSize) + { + BasicFile SingleChunkFile; + OpenFile(SingleChunkFile); + + m_DiskStats.CurrentOpenFileCount++; + auto _ = MakeGuard([this]() { m_DiskStats.CurrentOpenFileCount--; }); + SingleChunkFile.Write(Chunk, FileOffset); + } + else + { + const uint64_t MaxWriterBufferSize = 256u * 1025u; + + BufferedWriteFileCache::Local::Writer* Writer = LocalWriter.GetWriter(SequenceIndex); + if (Writer) + { + if ((!Writer->Writer) && (ChunkSize < MaxWriterBufferSize)) + { + Writer->Writer = std::make_unique(*Writer->File, Min(SequenceSize, MaxWriterBufferSize)); + } + Writer->Write(Chunk, FileOffset); + } + else + { + Writer = LocalWriter.PutWriter(SequenceIndex, std::make_unique()); + + Writer->File = std::make_unique(); + OpenFile(*Writer->File); + if (ChunkSize < MaxWriterBufferSize) + { + Writer->Writer = std::make_unique(*Writer->File, Min(SequenceSize, MaxWriterBufferSize)); + } + Writer->Write(Chunk, FileOffset); + } + } + m_DiskStats.WriteCount++; + m_DiskStats.WriteByteCount += ChunkSize; + m_WrittenChunkByteCount += ChunkSize; +} + +bool +BuildsOperationUpdateFolder::GetBlockWriteOps(const IoHash& BlockRawHash, + std::span ChunkRawHashes, + std::span ChunkCompressedLengths, + std::span> SequenceIndexChunksLeftToWriteCounters, + std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, + const MemoryView BlockView, + uint32_t FirstIncludedBlockChunkIndex, + uint32_t LastIncludedBlockChunkIndex, + BlockWriteOps& OutOps) +{ + ZEN_TRACE_CPU("GetBlockWriteOps"); + + uint32_t OffsetInBlock = 0; + for (uint32_t ChunkBlockIndex = FirstIncludedBlockChunkIndex; ChunkBlockIndex <= LastIncludedBlockChunkIndex; ChunkBlockIndex++) + { + const uint32_t ChunkCompressedSize = ChunkCompressedLengths[ChunkBlockIndex]; + const IoHash& ChunkHash = ChunkRawHashes[ChunkBlockIndex]; + if (auto It = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != m_RemoteLookup.ChunkHashToChunkIndex.end()) + { + const uint32_t ChunkIndex = It->second; + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, ChunkIndex); + + if (!ChunkTargetPtrs.empty()) + { + bool NeedsWrite = true; + if (RemoteChunkIndexNeedsCopyFromSourceFlags[ChunkIndex].compare_exchange_strong(NeedsWrite, false)) + { + MemoryView ChunkMemoryView = BlockView.Mid(OffsetInBlock, ChunkCompressedSize); + IoHash VerifyChunkHash; + uint64_t VerifyChunkSize; + CompressedBuffer CompressedChunk = + CompressedBuffer::FromCompressed(SharedBuffer::MakeView(ChunkMemoryView), VerifyChunkHash, VerifyChunkSize); + if (!CompressedChunk) + { + throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} is not a valid compressed buffer", + ChunkHash, + OffsetInBlock, + ChunkCompressedSize, + BlockRawHash)); + } + if (VerifyChunkHash != ChunkHash) + { + throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} has a mismatching content hash {}", + ChunkHash, + OffsetInBlock, + ChunkCompressedSize, + BlockRawHash, + VerifyChunkHash)); + } + if (VerifyChunkSize != m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]) + { + throw std::runtime_error( + fmt::format("Chunk {} at {}, size {} in block {} has a mismatching raw size {}, expected {}", + ChunkHash, + OffsetInBlock, + ChunkCompressedSize, + BlockRawHash, + VerifyChunkSize, + m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex])); + } + + OodleCompressor ChunkCompressor; + OodleCompressionLevel ChunkCompressionLevel; + uint64_t ChunkBlockSize; + + bool GetCompressParametersSuccess = + CompressedChunk.TryGetCompressParameters(ChunkCompressor, ChunkCompressionLevel, ChunkBlockSize); + ZEN_ASSERT(GetCompressParametersSuccess); + + IoBuffer Decompressed; + if (ChunkCompressionLevel == OodleCompressionLevel::None) + { + MemoryView ChunkDecompressedMemoryView = ChunkMemoryView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder()); + Decompressed = + IoBuffer(IoBuffer::Wrap, ChunkDecompressedMemoryView.GetData(), ChunkDecompressedMemoryView.GetSize()); + } + else + { + Decompressed = CompressedChunk.Decompress().AsIoBuffer(); + } + + if (Decompressed.GetSize() != m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]) + { + throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} decompressed to size {}, expected {}", + ChunkHash, + OffsetInBlock, + ChunkCompressedSize, + BlockRawHash, + Decompressed.GetSize(), + m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex])); + } + + ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed)); + for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs) + { + OutOps.WriteOps.push_back( + BlockWriteOps::WriteOpData{.Target = Target, .ChunkBufferIndex = OutOps.ChunkBuffers.size()}); + } + OutOps.ChunkBuffers.emplace_back(std::move(Decompressed)); + } + } + } + + OffsetInBlock += ChunkCompressedSize; + } + { + ZEN_TRACE_CPU("Sort"); + std::sort(OutOps.WriteOps.begin(), + OutOps.WriteOps.end(), + [](const BlockWriteOps::WriteOpData& Lhs, const BlockWriteOps::WriteOpData& Rhs) { + if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) + { + return true; + } + if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) + { + return false; + } + return Lhs.Target->Offset < Rhs.Target->Offset; + }); + } + return true; +} + +void +BuildsOperationUpdateFolder::WriteBlockChunkOpsToCache(std::span> SequenceIndexChunksLeftToWriteCounters, + const BlockWriteOps& Ops, + BufferedWriteFileCache& WriteCache, + ParallelWork& Work) +{ + ZEN_TRACE_CPU("WriteBlockChunkOpsToCache"); + + { + BufferedWriteFileCache::Local LocalWriter(WriteCache); + for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps) + { + if (Work.IsAborted()) + { + break; + } + const CompositeBuffer& Chunk = Ops.ChunkBuffers[WriteOp.ChunkBufferIndex]; + const uint32_t SequenceIndex = WriteOp.Target->SequenceIndex; + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() <= + m_RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]); + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() > 0); + const uint64_t FileOffset = WriteOp.Target->Offset; + const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + + WriteSequenceChunkToCache(LocalWriter, Chunk, SequenceIndex, FileOffset, PathIndex); + } + } + if (!Work.IsAborted()) + { + // Write tracking, updating this must be done without any files open (BufferedWriteFileCache::Local) + std::vector CompletedChunkSequences; + for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps) + { + const uint32_t RemoteSequenceIndex = WriteOp.Target->SequenceIndex; + if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters)) + { + CompletedChunkSequences.push_back(RemoteSequenceIndex); + } + } + WriteCache.Close(CompletedChunkSequences); + VerifyAndCompleteChunkSequencesAsync(CompletedChunkSequences, Work); + } +} + +bool +BuildsOperationUpdateFolder::WriteChunksBlockToCache(const ChunkBlockDescription& BlockDescription, + std::span> SequenceIndexChunksLeftToWriteCounters, + ParallelWork& Work, + CompositeBuffer&& BlockBuffer, + std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, + BufferedWriteFileCache& WriteCache) +{ + ZEN_TRACE_CPU("WriteChunksBlockToCache"); + + IoBuffer BlockMemoryBuffer = MakeBufferMemoryBased(BlockBuffer); + const MemoryView BlockView = BlockMemoryBuffer.GetView(); + + BlockWriteOps Ops; + if ((BlockDescription.HeaderSize == 0) || BlockDescription.ChunkCompressedLengths.empty()) + { + ZEN_TRACE_CPU("WriteChunksBlockToCache_Legacy"); + + uint64_t HeaderSize; + const std::vector ChunkCompressedLengths = + ReadChunkBlockHeader(BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder()), HeaderSize); + + if (GetBlockWriteOps(BlockDescription.BlockHash, + BlockDescription.ChunkRawHashes, + ChunkCompressedLengths, + SequenceIndexChunksLeftToWriteCounters, + RemoteChunkIndexNeedsCopyFromSourceFlags, + BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + HeaderSize), + 0, + gsl::narrow(BlockDescription.ChunkRawHashes.size() - 1), + Ops)) + { + WriteBlockChunkOpsToCache(SequenceIndexChunksLeftToWriteCounters, Ops, WriteCache, Work); + return true; + } + return false; + } + + if (GetBlockWriteOps(BlockDescription.BlockHash, + BlockDescription.ChunkRawHashes, + BlockDescription.ChunkCompressedLengths, + SequenceIndexChunksLeftToWriteCounters, + RemoteChunkIndexNeedsCopyFromSourceFlags, + BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize), + 0, + gsl::narrow(BlockDescription.ChunkRawHashes.size() - 1), + Ops)) + { + WriteBlockChunkOpsToCache(SequenceIndexChunksLeftToWriteCounters, Ops, WriteCache, Work); + return true; + } + return false; +} + +bool +BuildsOperationUpdateFolder::WritePartialBlockChunksToCache(const ChunkBlockDescription& BlockDescription, + std::span> SequenceIndexChunksLeftToWriteCounters, + ParallelWork& Work, + CompositeBuffer&& PartialBlockBuffer, + uint32_t FirstIncludedBlockChunkIndex, + uint32_t LastIncludedBlockChunkIndex, + std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, + BufferedWriteFileCache& WriteCache) +{ + ZEN_TRACE_CPU("WritePartialBlockChunksToCache"); + + IoBuffer BlockMemoryBuffer = MakeBufferMemoryBased(PartialBlockBuffer); + const MemoryView BlockView = BlockMemoryBuffer.GetView(); + + BlockWriteOps Ops; + if (GetBlockWriteOps(BlockDescription.BlockHash, + BlockDescription.ChunkRawHashes, + BlockDescription.ChunkCompressedLengths, + SequenceIndexChunksLeftToWriteCounters, + RemoteChunkIndexNeedsCopyFromSourceFlags, + BlockView, + FirstIncludedBlockChunkIndex, + LastIncludedBlockChunkIndex, + Ops)) + { + WriteBlockChunkOpsToCache(SequenceIndexChunksLeftToWriteCounters, Ops, WriteCache, Work); + return true; + } + else + { + return false; + } +} + +void +BuildsOperationUpdateFolder::AsyncWriteDownloadedChunk(uint32_t RemoteChunkIndex, + const BlobsExistsResult& ExistsResult, + std::vector&& ChunkTargetPtrs, + BufferedWriteFileCache& WriteCache, + ParallelWork& Work, + IoBuffer&& Payload, + std::span> SequenceIndexChunksLeftToWriteCounters, + std::atomic& WritePartsComplete, + const uint64_t TotalPartWriteCount, + FilteredRate& FilteredWrittenBytesPerSecond) +{ + ZEN_TRACE_CPU("AsyncWriteDownloadedChunk"); + + const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + + const uint64_t Size = Payload.GetSize(); + + const bool ExistsInCache = m_Storage.CacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash); + + const bool PopulateCache = !ExistsInCache && m_Storage.CacheStorage && m_Options.PopulateCache; + + std::filesystem::path CompressedChunkPath = + TryMoveDownloadedChunk(Payload, + m_TempDownloadFolderPath / ChunkHash.ToHexString(), + /* ForceDiskBased */ PopulateCache || Size > m_Options.MaximumInMemoryPayloadSize); + if (PopulateCache) + { + IoBuffer CacheBlob = IoBufferBuilder::MakeFromFile(CompressedChunkPath); + if (CacheBlob) + { + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, + ChunkHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(CacheBlob))); + } + } + + IoBufferFileReference FileRef; + bool EnableBacklog = !CompressedChunkPath.empty() || Payload.GetFileReference(FileRef); + + Work.ScheduleWork( + m_IOWorkerPool, + [this, + SequenceIndexChunksLeftToWriteCounters, + &Work, + CompressedChunkPath, + RemoteChunkIndex, + TotalPartWriteCount, + &WriteCache, + &WritePartsComplete, + &FilteredWrittenBytesPerSecond, + ChunkTargetPtrs = std::move(ChunkTargetPtrs), + CompressedPart = IoBuffer(std::move(Payload))](std::atomic&) mutable { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("Async_WriteChunk"); + + FilteredWrittenBytesPerSecond.Start(); + + const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + if (CompressedChunkPath.empty()) + { + ZEN_ASSERT(CompressedPart); + } + else + { + ZEN_ASSERT(!CompressedPart); + CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); + if (!CompressedPart) + { + throw std::runtime_error( + fmt::format("Could not open dowloaded compressed chunk {} from {}", ChunkHash, CompressedChunkPath)); + } + } + + bool NeedHashVerify = WriteCompressedChunkToCache(ChunkHash, ChunkTargetPtrs, WriteCache, std::move(CompressedPart)); + if (!m_AbortFlag) + { + if (WritePartsComplete.fetch_add(1) + 1 == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } + + if (!CompressedChunkPath.empty()) + { + std::error_code Ec = TryRemoveFile(CompressedChunkPath); + if (Ec) + { + ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", CompressedChunkPath, Ec.value(), Ec.message()); + } + } + + std::vector CompletedSequences = + CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); + WriteCache.Close(CompletedSequences); + if (NeedHashVerify) + { + VerifyAndCompleteChunkSequencesAsync(CompletedSequences, Work); + } + else + { + FinalizeChunkSequences(CompletedSequences); + } + } + } + }, + EnableBacklog ? WorkerThreadPool::EMode::EnableBacklog : WorkerThreadPool::EMode::DisableBacklog); +} + +void +BuildsOperationUpdateFolder::VerifyAndCompleteChunkSequencesAsync(std::span RemoteSequenceIndexes, ParallelWork& Work) +{ + if (RemoteSequenceIndexes.empty()) + { + return; + } + ZEN_TRACE_CPU("VerifyAndCompleteChunkSequence"); + if (m_Options.ValidateCompletedSequences) + { + for (uint32_t RemoteSequenceIndexOffset = 1; RemoteSequenceIndexOffset < RemoteSequenceIndexes.size(); RemoteSequenceIndexOffset++) + { + const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[RemoteSequenceIndexOffset]; + Work.ScheduleWork(m_IOWorkerPool, [this, RemoteSequenceIndex](std::atomic&) { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("Async_VerifyAndFinalizeSequence"); + + VerifySequence(RemoteSequenceIndex); + if (!m_AbortFlag) + { + const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + FinalizeChunkSequence(SequenceRawHash); + } + } + }); + } + const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[0]; + + VerifySequence(RemoteSequenceIndex); + const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + FinalizeChunkSequence(SequenceRawHash); + } + else + { + for (uint32_t RemoteSequenceIndexOffset = 0; RemoteSequenceIndexOffset < RemoteSequenceIndexes.size(); RemoteSequenceIndexOffset++) + { + const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[RemoteSequenceIndexOffset]; + const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + FinalizeChunkSequence(SequenceRawHash); + } + } +} + +bool +BuildsOperationUpdateFolder::CompleteSequenceChunk(uint32_t RemoteSequenceIndex, + std::span> SequenceIndexChunksLeftToWriteCounters) +{ + uint32_t PreviousValue = SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1); + ZEN_ASSERT(PreviousValue >= 1); + ZEN_ASSERT(PreviousValue != (uint32_t)-1); + return PreviousValue == 1; +} + +std::vector +BuildsOperationUpdateFolder::CompleteChunkTargets(const std::vector& ChunkTargetPtrs, + std::span> SequenceIndexChunksLeftToWriteCounters) +{ + ZEN_TRACE_CPU("CompleteChunkTargets"); + + std::vector CompletedSequenceIndexes; + for (const ChunkedContentLookup::ChunkSequenceLocation* Location : ChunkTargetPtrs) + { + const uint32_t RemoteSequenceIndex = Location->SequenceIndex; + if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters)) + { + CompletedSequenceIndexes.push_back(RemoteSequenceIndex); + } + } + return CompletedSequenceIndexes; +} + +void +BuildsOperationUpdateFolder::FinalizeChunkSequence(const IoHash& SequenceRawHash) +{ + ZEN_TRACE_CPU("FinalizeChunkSequence"); + + ZEN_ASSERT_SLOW(!IsFile(GetFinalChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash))); + std::error_code Ec; + RenameFile(GetTempChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash), + GetFinalChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash), + Ec); + if (Ec) + { + throw std::system_error(Ec); + } +} + +void +BuildsOperationUpdateFolder::FinalizeChunkSequences(std::span RemoteSequenceIndexes) +{ + ZEN_TRACE_CPU("FinalizeChunkSequences"); + + for (uint32_t SequenceIndex : RemoteSequenceIndexes) + { + FinalizeChunkSequence(m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + } +} + +void +BuildsOperationUpdateFolder::VerifySequence(uint32_t RemoteSequenceIndex) +{ + ZEN_TRACE_CPU("VerifySequence"); + + ZEN_ASSERT(m_Options.ValidateCompletedSequences); + + const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + { + ZEN_TRACE_CPU("HashSequence"); + const std::uint32_t RemotePathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; + const uint64_t ExpectedSize = m_RemoteContent.RawSizes[RemotePathIndex]; + IoBuffer VerifyBuffer = IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash)); + const uint64_t VerifySize = VerifyBuffer.GetSize(); + if (VerifySize != ExpectedSize) + { + throw std::runtime_error(fmt::format("Written chunk sequence {} size {} does not match expected size {}", + SequenceRawHash, + VerifySize, + ExpectedSize)); + } + + const IoHash VerifyChunkHash = IoHash::HashBuffer(std::move(VerifyBuffer), &m_ValidatedChunkByteCount); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error( + fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash)); + } + } +} + +void +VerifyFolder(ProgressBase& Progress, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + TransferThreadWorkers& Workers, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const std::filesystem::path& Path, + const std::vector& ExcludeFolders, + bool VerifyFileHash, + VerifyFolderStatistics& VerifyFolderStats) +{ + ZEN_TRACE_CPU("VerifyFolder"); + + Stopwatch Timer; + + std::unique_ptr ProgressBar = Progress.CreateProgressBar("Verify Files"); + + WorkerThreadPool& VerifyPool = Workers.GetIOWorkerPool(); + + ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + + const uint32_t PathCount = gsl::narrow(Content.Paths.size()); + + RwLock ErrorLock; + std::vector Errors; + + auto IsAcceptedFolder = [ExcludeFolders = ExcludeFolders](const std::string_view& RelativePath) -> bool { + for (const std::string& ExcludeFolder : ExcludeFolders) + { + if (RelativePath.starts_with(ExcludeFolder)) + { + if (RelativePath.length() == ExcludeFolder.length()) + { + return false; + } + else if (RelativePath[ExcludeFolder.length()] == '/') + { + return false; + } + } + } + return true; + }; + + for (uint32_t PathIndex = 0; PathIndex < PathCount; PathIndex++) + { + if (Work.IsAborted()) + { + break; + } + + Work.ScheduleWork( + VerifyPool, + [&Path, &Content, &Lookup, &ErrorLock, &Errors, &VerifyFolderStats, VerifyFileHash, &IsAcceptedFolder, PathIndex, &AbortFlag]( + std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("VerifyFile_work"); + + // TODO: Convert ScheduleWork body to function + + const std::filesystem::path TargetPath = (Path / Content.Paths[PathIndex]).make_preferred(); + if (IsAcceptedFolder(TargetPath.parent_path().generic_string())) + { + const uint64_t ExpectedSize = Content.RawSizes[PathIndex]; + if (!IsFile(TargetPath)) + { + ErrorLock.WithExclusiveLock([&]() { + Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize)); + }); + VerifyFolderStats.FilesFailed++; + } + else + { + std::error_code Ec; + uint64_t SizeOnDisk = gsl::narrow(FileSizeFromPath(TargetPath, Ec)); + if (Ec) + { + ErrorLock.WithExclusiveLock([&]() { + Errors.push_back( + fmt::format("Failed to get size of file {}: {} ({})", TargetPath, Ec.message(), Ec.value())); + }); + VerifyFolderStats.FilesFailed++; + } + else if (SizeOnDisk < ExpectedSize) + { + ErrorLock.WithExclusiveLock([&]() { + Errors.push_back(fmt::format("Size of file {} is smaller than expected. Expected: {}, Found: {}", + TargetPath, + ExpectedSize, + SizeOnDisk)); + }); + VerifyFolderStats.FilesFailed++; + } + else if (SizeOnDisk > ExpectedSize) + { + ErrorLock.WithExclusiveLock([&]() { + Errors.push_back(fmt::format("Size of file {} is bigger than expected. Expected: {}, Found: {}", + TargetPath, + ExpectedSize, + SizeOnDisk)); + }); + VerifyFolderStats.FilesFailed++; + } + else if (SizeOnDisk > 0 && VerifyFileHash) + { + const IoHash& ExpectedRawHash = Content.RawHashes[PathIndex]; + IoBuffer Buffer = IoBufferBuilder::MakeFromFile(TargetPath); + IoHash RawHash = IoHash::HashBuffer(Buffer); + if (RawHash != ExpectedRawHash) + { + uint64_t FileOffset = 0; + const uint32_t SequenceIndex = Lookup.RawHashToSequenceIndex.at(ExpectedRawHash); + const uint32_t OrderOffset = Lookup.SequenceIndexChunkOrderOffset[SequenceIndex]; + for (uint32_t OrderIndex = OrderOffset; + OrderIndex < OrderOffset + Content.ChunkedContent.ChunkCounts[SequenceIndex]; + OrderIndex++) + { + uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex]; + uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + IoHash ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; + IoBuffer FileChunk = IoBuffer(Buffer, FileOffset, ChunkSize); + if (IoHash::HashBuffer(FileChunk) != ChunkHash) + { + ErrorLock.WithExclusiveLock([&]() { + Errors.push_back(fmt::format( + "WARNING: Hash of file {} does not match expected hash. Expected: {}, Found: {}. " + "Mismatch at chunk {}", + TargetPath, + ExpectedRawHash, + RawHash, + OrderIndex - OrderOffset)); + }); + break; + } + FileOffset += ChunkSize; + } + VerifyFolderStats.FilesFailed++; + } + VerifyFolderStats.ReadBytes += SizeOnDisk; + } + } + } + VerifyFolderStats.FilesVerified++; + } + }, + [&, PathIndex](std::exception_ptr Ex, std::atomic&) { + std::string Description; + try + { + std::rethrow_exception(Ex); + } + catch (const std::exception& Ex) + { + Description = Ex.what(); + } + ErrorLock.WithExclusiveLock([&]() { + Errors.push_back(fmt::format("Failed verifying file '{}'. Reason: {}", + (Path / Content.Paths[PathIndex]).make_preferred(), + Description)); + }); + VerifyFolderStats.FilesFailed++; + }); + } + + Work.Wait(Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); + std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}", + VerifyFolderStats.FilesVerified.load(), + PathCount, + NiceBytes(VerifyFolderStats.ReadBytes.load()), + VerifyFolderStats.FilesFailed.load()); + ProgressBar->UpdateState({.Task = "Verifying files ", + .Details = Details, + .TotalCount = gsl::narrow(PathCount), + .RemainingCount = gsl::narrow(PathCount - VerifyFolderStats.FilesVerified.load()), + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); + }); + VerifyFolderStats.VerifyElapsedWallTimeUs = Timer.GetElapsedTimeUs(); + + ProgressBar->Finish(); + if (AbortFlag) + { + return; + } + + for (const std::string& Error : Errors) + { + ZEN_CONSOLE_ERROR("{}", Error); + } + if (!Errors.empty()) + { + throw std::runtime_error(fmt::format("Verify failed with {} errors", Errors.size())); + } +} + +std::vector +GetNewPaths(const std::span KnownPaths, const std::span Paths) +{ + tsl::robin_set KnownPathsSet; + KnownPathsSet.reserve(KnownPaths.size()); + for (const std::filesystem::path& LocalPath : KnownPaths) + { + KnownPathsSet.insert(LocalPath.generic_string()); + } + + std::vector NewPaths; + for (const std::filesystem::path& UntrackedPath : Paths) + { + if (!KnownPathsSet.contains(UntrackedPath.generic_string())) + { + NewPaths.push_back(UntrackedPath); + } + } + return NewPaths; +} + +BuildSaveState +GetLocalStateFromPaths(ProgressBase& Progress, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + TransferThreadWorkers& Workers, + GetFolderContentStatistics& LocalFolderScanStats, + ChunkingStatistics& ChunkingStats, + const std::filesystem::path& Path, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + std::span PathsToCheck) +{ + FolderContent FolderState = + CheckFolderFiles(Progress, AbortFlag, PauseFlag, "Check Files", Workers, LocalFolderScanStats, Path, PathsToCheck); + + ChunkedFolderContent ChunkedContent; + if (FolderState.Paths.size() > 0) + { + ChunkedContent = ScanFolderFiles(Progress, + AbortFlag, + PauseFlag, + "Scan Files", + Workers, + Path, + FolderState, + ChunkController, + ChunkCache, + ChunkingStats); + } + + return BuildSaveState{.State = BuildState{.ChunkedContent = std::move(ChunkedContent)}, .FolderState = FolderState, .LocalPath = Path}; +} + +BuildSaveState +GetLocalContent(ProgressBase& Progress, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + bool IsQuiet, + TransferThreadWorkers& Workers, + GetFolderContentStatistics& LocalFolderScanStats, + ChunkingStatistics& ChunkingStats, + const std::filesystem::path& Path, + const std::filesystem::path& StateFilePath, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache) +{ + Stopwatch ReadStateTimer; + bool FileExists = IsFile(StateFilePath); + if (!FileExists) + { + ZEN_CONSOLE("No known local state file in {}, falling back to scanning", Path); + return {}; + } + + BuildSaveState SavedLocalState; + try + { + SavedLocalState = ReadBuildSaveStateFile(StateFilePath); + if (!IsQuiet) + { + ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); + } + } + catch (const std::exception& Ex) + { + ZEN_CONSOLE_WARN("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what()); + return {}; + } + + FolderContent CurrentLocalFolderState = CheckFolderFiles(Progress, + AbortFlag, + PauseFlag, + "Check Known Files", + Workers, + LocalFolderScanStats, + Path, + SavedLocalState.FolderState.Paths); + if (AbortFlag) + { + return {}; + } + + if (!SavedLocalState.FolderState.AreKnownFilesEqual(CurrentLocalFolderState)) + { + const size_t LocalStatePathCount = SavedLocalState.FolderState.Paths.size(); + std::vector DeletedPaths; + FolderContent UpdatedContent = GetUpdatedContent(SavedLocalState.FolderState, CurrentLocalFolderState, DeletedPaths); + if (!DeletedPaths.empty()) + { + SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths); + } + + if (!IsQuiet) + { + ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", + DeletedPaths.size(), + UpdatedContent.Paths.size(), + LocalStatePathCount); + } + if (UpdatedContent.Paths.size() > 0) + { + ChunkedFolderContent UpdatedLocalContent = ScanFolderFiles(Progress, + AbortFlag, + PauseFlag, + "Scan Known Files", + Workers, + Path, + UpdatedContent, + ChunkController, + ChunkCache, + ChunkingStats); + if (AbortFlag) + { + return {}; + } + SavedLocalState.State.ChunkedContent = + MergeChunkedFolderContents(SavedLocalState.State.ChunkedContent, {{UpdatedLocalContent}}); + } + } + else + { + // Remove files from LocalContent no longer in LocalFolderState + tsl::robin_set LocalFolderPaths; + LocalFolderPaths.reserve(SavedLocalState.FolderState.Paths.size()); + for (const std::filesystem::path& LocalFolderPath : SavedLocalState.FolderState.Paths) + { + LocalFolderPaths.insert(LocalFolderPath.generic_string()); + } + std::vector DeletedPaths; + for (const std::filesystem::path& LocalContentPath : SavedLocalState.State.ChunkedContent.Paths) + { + if (!LocalFolderPaths.contains(LocalContentPath.generic_string())) + { + DeletedPaths.push_back(LocalContentPath); + } + } + if (!DeletedPaths.empty()) + { + SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths); + } + } + + SavedLocalState.FolderState = CurrentLocalFolderState; + + return SavedLocalState; +} + +void +DownloadFolder(LoggerRef InLog, + ProgressBase& Progress, + TransferThreadWorkers& Workers, + StorageInstance& Storage, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + const BuildStorageCache::Statistics& StorageCacheStats, + const Oid& BuildId, + const std::vector& BuildPartIds, + std::span BuildPartNames, + const std::filesystem::path& DownloadSpecPath, + const std::filesystem::path& Path, + const DownloadOptions& Options) +{ + ZEN_TRACE_CPU("DownloadFolder"); + ZEN_SCOPED_LOG(InLog); + + Progress.SetLogOperationName("Download Folder"); + + enum TaskSteps : uint32_t + { + CheckState, + CompareState, + Download, + Verify, + Cleanup, + StepCount + }; + + auto EndProgress = MakeGuard([&]() { Progress.SetLogOperationProgress(TaskSteps::StepCount, TaskSteps::StepCount); }); + + Stopwatch DownloadTimer; + + Progress.SetLogOperationProgress(TaskSteps::CheckState, TaskSteps::StepCount); + + const std::filesystem::path ZenTempFolder = ZenTempFolderPath(Options.ZenFolderPath); + CreateDirectories(ZenTempFolder); + + std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; + + CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId, Options.IsQuiet); + + std::vector> AllBuildParts = + ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); + + BuildManifest Manifest; + if (!DownloadSpecPath.empty()) + { + const std::filesystem::path AbsoluteDownloadSpecPath = + DownloadSpecPath.is_relative() ? MakeSafeAbsolutePath(Path / DownloadSpecPath) : MakeSafeAbsolutePath(DownloadSpecPath); + Manifest = ParseBuildManifest(DownloadSpecPath); + } + + std::vector PartContents; + + std::unique_ptr ChunkController; + + std::vector BlockDescriptions; + std::vector LooseChunkHashes; + + Progress.SetLogOperationProgress(TaskSteps::CompareState, TaskSteps::StepCount); + + ChunkedFolderContent RemoteContent = GetRemoteContent(InLog, + Storage, + BuildId, + AllBuildParts, + Manifest, + Options.IncludeWildcards, + Options.ExcludeWildcards, + ChunkController, + PartContents, + BlockDescriptions, + LooseChunkHashes, + Options.IsQuiet, + Options.IsVerbose, + Options.DoExtraContentVerify); + + const std::uint64_t LargeAttachmentSize = Options.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; + GetFolderContentStatistics LocalFolderScanStats; + ChunkingStatistics ChunkingStats; + + BuildSaveState LocalState; + + if (IsDir(Path)) + { + if (!ChunkController && !Options.IsQuiet) + { + ZEN_CONSOLE_INFO("Unspecified chunking algorithm, using default"); + ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); + } + std::unique_ptr ChunkCache(CreateNullChunkingCache()); + + LocalState = GetLocalContent(Progress, + AbortFlag, + PauseFlag, + Options.IsQuiet, + Workers, + LocalFolderScanStats, + ChunkingStats, + Path, + ZenStateFilePath(Path / ZenFolderName), + *ChunkController, + *ChunkCache); + + std::vector UntrackedPaths = GetNewPaths(LocalState.State.ChunkedContent.Paths, RemoteContent.Paths); + + BuildSaveState UntrackedLocalContent = GetLocalStateFromPaths(Progress, + AbortFlag, + PauseFlag, + Workers, + LocalFolderScanStats, + ChunkingStats, + Path, + *ChunkController, + *ChunkCache, + UntrackedPaths); + + if (!UntrackedLocalContent.State.ChunkedContent.Paths.empty()) + { + LocalState.State.ChunkedContent = + MergeChunkedFolderContents(LocalState.State.ChunkedContent, + std::vector{UntrackedLocalContent.State.ChunkedContent}); + + // TODO: Helper + LocalState.FolderState.Paths.insert(LocalState.FolderState.Paths.begin(), + UntrackedLocalContent.FolderState.Paths.begin(), + UntrackedLocalContent.FolderState.Paths.end()); + LocalState.FolderState.RawSizes.insert(LocalState.FolderState.RawSizes.begin(), + UntrackedLocalContent.FolderState.RawSizes.begin(), + UntrackedLocalContent.FolderState.RawSizes.end()); + LocalState.FolderState.Attributes.insert(LocalState.FolderState.Attributes.begin(), + UntrackedLocalContent.FolderState.Attributes.begin(), + UntrackedLocalContent.FolderState.Attributes.end()); + LocalState.FolderState.ModificationTicks.insert(LocalState.FolderState.ModificationTicks.begin(), + UntrackedLocalContent.FolderState.ModificationTicks.begin(), + UntrackedLocalContent.FolderState.ModificationTicks.end()); + } + + if (Options.AppendNewContent) + { + RemoteContent = ApplyChunkedContentOverlay(LocalState.State.ChunkedContent, + RemoteContent, + Options.IncludeWildcards, + Options.ExcludeWildcards); + } +#if ZEN_BUILD_DEBUG + ValidateChunkedFolderContent(RemoteContent, + BlockDescriptions, + LooseChunkHashes, + Options.IncludeWildcards, + Options.ExcludeWildcards); +#endif // ZEN_BUILD_DEBUG + } + else + { + CreateDirectories(Path); + } + if (AbortFlag) + { + return; + } + + LocalState.LocalPath = Path; + + { + BuildsSelection::Build RemoteBuildState = {.Id = BuildId, + .IncludeWildcards = Options.IncludeWildcards, + .ExcludeWildcards = Options.ExcludeWildcards}; + RemoteBuildState.Parts.reserve(BuildPartIds.size()); + for (size_t PartIndex = 0; PartIndex < BuildPartIds.size(); PartIndex++) + { + RemoteBuildState.Parts.push_back( + {BuildsSelection::BuildPart{.Id = BuildPartIds[PartIndex], + .Name = PartIndex < BuildPartNames.size() ? BuildPartNames[PartIndex] : ""}}); + } + + if (Options.AppendNewContent) + { + LocalState.State.Selection.Builds.emplace_back(std::move(RemoteBuildState)); + } + else + { + LocalState.State.Selection.Builds = std::vector{std::move(RemoteBuildState)}; + } + } + + if ((Options.EnableTargetFolderScavenging || Options.AppendNewContent) && !Options.CleanTargetFolder && + CompareChunkedContent(RemoteContent, LocalState.State.ChunkedContent)) + { + if (!Options.IsQuiet) + { + ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", + NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); + } + + Stopwatch WriteStateTimer; + + CbObject StateObject = CreateBuildSaveStateObject(LocalState); + CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path()); + TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView()); + if (!Options.IsQuiet) + { + ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + } + + AddDownloadedPath(Options.SystemRootDir, + BuildsDownloadInfo{.Selection = LocalState.State.Selection, + .LocalPath = Path, + .StateFilePath = ZenStateFilePath(Options.ZenFolderPath), + .Iso8601Date = DateTime::Now().ToIso8601()}); + } + else + { + ExtendableStringBuilder<128> BuildPartString; + for (const std::pair& BuildPart : AllBuildParts) + { + BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first)); + } + + uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0)); + + if (!Options.IsQuiet) + { + ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize)); + } + + Stopwatch IndexTimer; + + const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalState.State.ChunkedContent); + const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); + + if (!Options.IsQuiet) + { + ZEN_INFO("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); + } + + Progress.SetLogOperationProgress(TaskSteps::Download, TaskSteps::StepCount); + + BuildsOperationUpdateFolder Updater( + InLog, + Progress, + Storage, + AbortFlag, + PauseFlag, + Workers.GetIOWorkerPool(), + Workers.GetNetworkPool(), + BuildId, + Path, + LocalState.State.ChunkedContent, + LocalLookup, + RemoteContent, + RemoteLookup, + BlockDescriptions, + LooseChunkHashes, + BuildsOperationUpdateFolder::Options{ + .IsQuiet = Options.IsQuiet, + .IsVerbose = Options.IsVerbose, + .AllowFileClone = Options.AllowFileClone, + .UseSparseFiles = Options.UseSparseFiles, + .SystemRootDir = Options.SystemRootDir, + .ZenFolderPath = Options.ZenFolderPath, + .LargeAttachmentSize = LargeAttachmentSize, + .PreferredMultipartChunkSize = PreferredMultipartChunkSize, + .PartialBlockRequestMode = Options.PartialBlockRequestMode, + .WipeTargetFolder = Options.CleanTargetFolder, + .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging, + .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging || Options.AppendNewContent, + .ValidateCompletedSequences = Options.PostDownloadVerify, + .ExcludeFolders = Options.ExcludeFolders, + .MaximumInMemoryPayloadSize = Options.MaximumInMemoryPayloadSize, + .PopulateCache = Options.PopulateCache}); + { + Progress.PushLogOperation("Download"); + auto _ = MakeGuard([&Progress]() { Progress.PopLogOperation(); }); + FolderContent UpdatedLocalFolderState; + Updater.Execute(UpdatedLocalFolderState); + + LocalState.State.ChunkedContent = RemoteContent; + LocalState.FolderState = std::move(UpdatedLocalFolderState); + } + + VerifyFolderStatistics VerifyFolderStats; + if (!AbortFlag) + { + AddDownloadedPath(Options.SystemRootDir, + BuildsDownloadInfo{.Selection = LocalState.State.Selection, + .LocalPath = Path, + .StateFilePath = ZenStateFilePath(Options.ZenFolderPath), + .Iso8601Date = DateTime::Now().ToIso8601()}); + + Progress.SetLogOperationProgress(TaskSteps::Verify, TaskSteps::StepCount); + + VerifyFolder(Progress, + AbortFlag, + PauseFlag, + Workers, + RemoteContent, + RemoteLookup, + Path, + Options.ExcludeFolders, + Options.PostDownloadVerify, + VerifyFolderStats); + + Stopwatch WriteStateTimer; + CbObject StateObject = CreateBuildSaveStateObject(LocalState); + + CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path()); + TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView()); + if (!Options.IsQuiet) + { + ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + } + +#if 0 + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(StateObject, SB); + WriteFile(ZenStateFileJsonPath(Options.ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); +#endif // 0 + const uint64_t DownloadCount = Updater.m_DownloadStats.DownloadedChunkCount.load() + + Updater.m_DownloadStats.DownloadedBlockCount.load() + + Updater.m_DownloadStats.DownloadedPartialBlockCount.load(); + const uint64_t DownloadByteCount = Updater.m_DownloadStats.DownloadedChunkByteCount.load() + + Updater.m_DownloadStats.DownloadedBlockByteCount.load() + + Updater.m_DownloadStats.DownloadedPartialBlockByteCount.load(); + const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs(); + + if (!Options.IsQuiet) + { + std::string CloneInfo; + if (Updater.m_DiskStats.CloneByteCount > 0) + { + CloneInfo = fmt::format(" ({} cloned)", NiceBytes(Updater.m_DiskStats.CloneByteCount.load())); + } + + std::string DownloadDetails; + { + ExtendableStringBuilder<128> SB; + BuildStorageBase::ExtendedStatistics ExtendedDownloadStats; + if (Storage.BuildStorage->GetExtendedStatistics(ExtendedDownloadStats)) + { + if (!ExtendedDownloadStats.ReceivedBytesPerSource.empty()) + { + for (auto& It : ExtendedDownloadStats.ReceivedBytesPerSource) + { + if (SB.Size() > 0) + { + SB.Append(", "sv); + } + SB.Append(It.first); + SB.Append(": "sv); + SB.Append(NiceBytes(It.second)); + } + } + } + if (Storage.CacheStorage) + { + if (SB.Size() > 0) + { + SB.Append(", "sv); + } + SB.Append("Cache: "); + SB.Append(NiceBytes(StorageCacheStats.TotalBytesRead.load())); + } + if (SB.Size() > 0) + { + DownloadDetails = fmt::format(" ({})", SB.ToView()); + } + } + + ZEN_CONSOLE( + "Downloaded build {}, parts:{} in {}\n" + " Scavenge: {} (Target: {}, Cache: {}, Others: {})\n" + " Download: {} ({}) {}bits/s{}\n" + " Write: {} ({}) {}B/s{}\n" + " Clean: {}\n" + " Finalize: {}\n" + " Verify: {}", + BuildId, + BuildPartString.ToView(), + NiceTimeSpanMs(DownloadTimeMs), + + NiceTimeSpanMs((Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs + + Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs + + Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs) / + 1000), + NiceTimeSpanMs(Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000), + NiceTimeSpanMs(Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000), + NiceTimeSpanMs(Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000), + + DownloadCount, + NiceBytes(DownloadByteCount), + NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)), + DownloadDetails, + + Updater.m_DiskStats.WriteCount.load(), + NiceBytes(Updater.m_WrittenChunkByteCount.load()), + NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.WriteTimeUs, Updater.m_DiskStats.WriteByteCount.load())), + CloneInfo, + + NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000), + + NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000), + + NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); + } + } + } + + Progress.SetLogOperationProgress(TaskSteps::Cleanup, TaskSteps::StepCount); + + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag, PauseFlag, ZenTempFolder); +} +} // namespace zen diff --git a/src/zenremotestore/builds/builduploadfolder.cpp b/src/zenremotestore/builds/builduploadfolder.cpp new file mode 100644 index 000000000..b536ae464 --- /dev/null +++ b/src/zenremotestore/builds/builduploadfolder.cpp @@ -0,0 +1,2634 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +using namespace std::literals; + +namespace { + bool IsExtensionHashCompressable(const tsl::robin_set& NonCompressableExtensionHashes, const uint32_t PathHash) + { + return !NonCompressableExtensionHashes.contains(PathHash); + } + + bool IsChunkCompressable(const tsl::robin_set& NonCompressableExtensionHashes, + const ChunkedContentLookup& Lookup, + uint32_t ChunkIndex) + { + const uint32_t ChunkLocationCount = Lookup.ChunkSequenceLocationCounts[ChunkIndex]; + if (ChunkLocationCount == 0) + { + return false; + } + const size_t ChunkLocationOffset = Lookup.ChunkSequenceLocationOffset[ChunkIndex]; + const uint32_t SequenceIndex = Lookup.ChunkSequenceLocations[ChunkLocationOffset].SequenceIndex; + const uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[SequenceIndex]; + const uint32_t ExtensionHash = Lookup.PathExtensionHash[PathIndex]; + + const bool IsCompressable = IsExtensionHashCompressable(NonCompressableExtensionHashes, ExtensionHash); + return IsCompressable; + } + template + std::string FormatArray(std::span Items, std::string_view Prefix) + { + ExtendableStringBuilder<512> SB; + for (const T& Item : Items) + { + SB.Append(fmt::format("{}{}", Prefix, Item)); + } + return SB.ToString(); + } +} // namespace + +class ReadFileCache +{ +public: + // A buffered file reader that provides CompositeBuffer where the buffers are owned and the memory never overwritten + ReadFileCache(std::atomic& OpenReadCount, + std::atomic& CurrentOpenFileCount, + std::atomic& ReadCount, + std::atomic& ReadByteCount, + const std::filesystem::path& Path, + const ChunkedFolderContent& LocalContent, + const ChunkedContentLookup& LocalLookup, + size_t MaxOpenFileCount) + : m_Path(Path) + , m_LocalContent(LocalContent) + , m_LocalLookup(LocalLookup) + , m_OpenReadCount(OpenReadCount) + , m_CurrentOpenFileCount(CurrentOpenFileCount) + , m_ReadCount(ReadCount) + , m_ReadByteCount(ReadByteCount) + { + m_OpenFiles.reserve(MaxOpenFileCount); + } + ~ReadFileCache() { m_OpenFiles.clear(); } + + CompositeBuffer GetRange(uint32_t SequenceIndex, uint64_t Offset, uint64_t Size) + { + ZEN_TRACE_CPU("ReadFileCache::GetRange"); + + auto CacheIt = + std::find_if(m_OpenFiles.begin(), m_OpenFiles.end(), [SequenceIndex](const auto& Lhs) { return Lhs.first == SequenceIndex; }); + if (CacheIt != m_OpenFiles.end()) + { + if (CacheIt != m_OpenFiles.begin()) + { + auto CachedFile(std::move(CacheIt->second)); + m_OpenFiles.erase(CacheIt); + m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(SequenceIndex, std::move(CachedFile))); + } + CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size); + return Result; + } + const uint32_t LocalPathIndex = m_LocalLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + const std::filesystem::path LocalFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred(); + if (Size == m_LocalContent.RawSizes[LocalPathIndex]) + { + IoBuffer Result = IoBufferBuilder::MakeFromFile(LocalFilePath); + return CompositeBuffer(SharedBuffer(Result)); + } + if (m_OpenFiles.size() == m_OpenFiles.capacity()) + { + m_OpenFiles.pop_back(); + } + m_OpenFiles.insert( + m_OpenFiles.begin(), + std::make_pair( + SequenceIndex, + std::make_unique(LocalFilePath, m_OpenReadCount, m_CurrentOpenFileCount, m_ReadCount, m_ReadByteCount))); + CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size); + return Result; + } + +private: + const std::filesystem::path m_Path; + const ChunkedFolderContent& m_LocalContent; + const ChunkedContentLookup& m_LocalLookup; + std::vector>> m_OpenFiles; + std::atomic& m_OpenReadCount; + std::atomic& m_CurrentOpenFileCount; + std::atomic& m_ReadCount; + std::atomic& m_ReadByteCount; +}; + +BuildsOperationUploadFolder::BuildsOperationUploadFolder(LoggerRef Log, + ProgressBase& Progress, + StorageInstance& Storage, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + WorkerThreadPool& IOWorkerPool, + WorkerThreadPool& NetworkPool, + const Oid& BuildId, + const std::filesystem::path& Path, + bool CreateBuild, + const CbObject& MetaData, + const Options& Options) +: m_Log(Log) +, m_Progress(Progress) +, m_Storage(Storage) +, m_AbortFlag(AbortFlag) +, m_PauseFlag(PauseFlag) +, m_IOWorkerPool(IOWorkerPool) +, m_NetworkPool(NetworkPool) +, m_BuildId(BuildId) +, m_Path(Path) +, m_CreateBuild(CreateBuild) +, m_MetaData(MetaData) +, m_Options(Options) +{ + m_NonCompressableExtensionHashes.reserve(Options.NonCompressableExtensions.size()); + for (const std::string& Extension : Options.NonCompressableExtensions) + { + m_NonCompressableExtensionHashes.insert(HashStringAsLowerDjb2(Extension)); + } +} + +BuildsOperationUploadFolder::PrepareBuildResult +BuildsOperationUploadFolder::PrepareBuild() +{ + ZEN_TRACE_CPU("PrepareBuild"); + + PrepareBuildResult Result; + Result.PreferredMultipartChunkSize = m_Options.PreferredMultipartChunkSize; + Stopwatch Timer; + if (m_CreateBuild) + { + ZEN_TRACE_CPU("CreateBuild"); + + Stopwatch PutBuildTimer; + CbObject PutBuildResult = m_Storage.BuildStorage->PutBuild(m_BuildId, m_MetaData); + Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs(); + if (auto ChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(); ChunkSize != 0) + { + Result.PreferredMultipartChunkSize = ChunkSize; + } + Result.PayloadSize = m_MetaData.GetSize(); + } + else + { + ZEN_TRACE_CPU("PutBuild"); + Stopwatch GetBuildTimer; + CbObject Build = m_Storage.BuildStorage->GetBuild(m_BuildId); + Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs(); + Result.PayloadSize = Build.GetSize(); + if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) + { + Result.PreferredMultipartChunkSize = ChunkSize; + } + else if (m_Options.AllowMultiparts) + { + ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", NiceBytes(Result.PreferredMultipartChunkSize)); + } + } + + if (!m_Options.IgnoreExistingBlocks) + { + ZEN_TRACE_CPU("FindBlocks"); + Stopwatch KnownBlocksTimer; + CbObject BlockDescriptionList = m_Storage.BuildStorage->FindBlocks(m_BuildId, m_Options.FindBlockMaxCount); + if (BlockDescriptionList) + { + Result.KnownBlocks = ParseChunkBlockDescriptionList(BlockDescriptionList); + } + Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); + } + Result.ElapsedTimeMs = Timer.GetElapsedTimeMs(); + return Result; +} + +std::vector +BuildsOperationUploadFolder::ReadFolder() +{ + std::vector UploadParts; + std::filesystem::path ExcludeManifestPath = m_Path / m_Options.ZenExcludeManifestName; + tsl::robin_set ExcludeAssetPaths; + if (IsFile(ExcludeManifestPath)) + { + std::filesystem::path AbsoluteExcludeManifestPath = + MakeSafeAbsolutePath(ExcludeManifestPath.is_absolute() ? ExcludeManifestPath : m_Path / ExcludeManifestPath); + BuildManifest Manifest = ParseBuildManifest(AbsoluteExcludeManifestPath); + const std::vector& AssetPaths = Manifest.Parts.front().Files; + ExcludeAssetPaths.reserve(AssetPaths.size()); + for (const std::filesystem::path& AssetPath : AssetPaths) + { + ExcludeAssetPaths.insert(AssetPath.generic_string()); + } + } + + UploadParts.resize(1); + + UploadPart& Part = UploadParts.front(); + GetFolderContentStatistics& LocalFolderScanStats = Part.LocalFolderScanStats; + + Part.Content = GetFolderContent( + Part.LocalFolderScanStats, + m_Path, + [this](const std::string_view& RelativePath) { return IsAcceptedFolder(RelativePath); }, + [this, &ExcludeAssetPaths](const std::string_view& RelativePath, uint64_t Size, uint32_t Attributes) -> bool { + ZEN_UNUSED(Size, Attributes); + if (!IsAcceptedFile(RelativePath)) + { + return false; + } + if (ExcludeAssetPaths.contains(std::filesystem::path(RelativePath).generic_string())) + { + return false; + } + return true; + }, + m_IOWorkerPool, + m_Progress.GetProgressUpdateDelayMS(), + [&](bool, std::ptrdiff_t) { ZEN_INFO("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), m_Path); }, + m_AbortFlag); + Part.TotalRawSize = std::accumulate(Part.Content.RawSizes.begin(), Part.Content.RawSizes.end(), std::uint64_t(0)); + + return UploadParts; +} + +std::vector +BuildsOperationUploadFolder::ReadManifestParts(const std::filesystem::path& ManifestPath) +{ + std::vector UploadParts; + Stopwatch ManifestParseTimer; + std::filesystem::path AbsoluteManifestPath = MakeSafeAbsolutePath(ManifestPath.is_absolute() ? ManifestPath : m_Path / ManifestPath); + BuildManifest Manifest = ParseBuildManifest(AbsoluteManifestPath); + if (Manifest.Parts.empty()) + { + throw std::runtime_error(fmt::format("Manifest file at '{}' is invalid", ManifestPath)); + } + + UploadParts.resize(Manifest.Parts.size()); + for (size_t PartIndex = 0; PartIndex < Manifest.Parts.size(); PartIndex++) + { + BuildManifest::Part& PartManifest = Manifest.Parts[PartIndex]; + if (ManifestPath.is_relative()) + { + PartManifest.Files.push_back(ManifestPath); + } + + UploadPart& Part = UploadParts[PartIndex]; + FolderContent& Content = Part.Content; + + GetFolderContentStatistics& LocalFolderScanStats = Part.LocalFolderScanStats; + + const std::vector& AssetPaths = PartManifest.Files; + Content = GetValidFolderContent( + m_IOWorkerPool, + LocalFolderScanStats, + m_Path, + AssetPaths, + [](uint64_t PathCount, uint64_t CompletedPathCount) { ZEN_UNUSED(PathCount, CompletedPathCount); }, + 1000, + m_AbortFlag, + m_PauseFlag); + + if (Content.Paths.size() != AssetPaths.size()) + { + const tsl::robin_set FoundPaths(Content.Paths.begin(), Content.Paths.end()); + ExtendableStringBuilder<1024> SB; + for (const std::filesystem::path& AssetPath : AssetPaths) + { + if (!FoundPaths.contains(AssetPath)) + { + SB << "\n " << AssetPath.generic_string(); + } + } + throw std::runtime_error( + fmt::format("Manifest file at '{}' references files that does not exist{}", ManifestPath, SB.ToView())); + } + + Part.PartId = PartManifest.PartId; + Part.PartName = PartManifest.PartName; + Part.TotalRawSize = std::accumulate(Part.Content.RawSizes.begin(), Part.Content.RawSizes.end(), std::uint64_t(0)); + } + + return UploadParts; +} + +std::vector> +BuildsOperationUploadFolder::Execute(const Oid& BuildPartId, + const std::string_view BuildPartName, + const std::filesystem::path& ManifestPath, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache) +{ + ZEN_TRACE_CPU("BuildsOperationUploadFolder::Execute"); + try + { + Stopwatch ReadPartsTimer; + std::vector UploadParts = ManifestPath.empty() ? ReadFolder() : ReadManifestParts(ManifestPath); + + for (UploadPart& Part : UploadParts) + { + if (Part.PartId == Oid::Zero) + { + if (UploadParts.size() != 1) + { + throw std::runtime_error(fmt::format("Multi part upload manifest '{}' must contains build part id", ManifestPath)); + } + + if (BuildPartId == Oid::Zero) + { + Part.PartId = Oid::NewOid(); + } + else + { + Part.PartId = BuildPartId; + } + } + if (Part.PartName.empty()) + { + if (UploadParts.size() != 1) + { + throw std::runtime_error(fmt::format("Multi part upload manifest '{}' must contains build part name", ManifestPath)); + } + if (BuildPartName.empty()) + { + throw std::runtime_error("Build part name must be set"); + } + Part.PartName = std::string(BuildPartName); + } + } + + if (!m_Options.IsQuiet) + { + ZEN_INFO("Reading {} parts took {}", UploadParts.size(), NiceTimeSpanMs(ReadPartsTimer.GetElapsedTimeMs())); + } + + const uint32_t PartsUploadStepCount = gsl::narrow(uint32_t(PartTaskSteps::StepCount) * UploadParts.size()); + + const uint32_t PrepareBuildStep = 0; + const uint32_t UploadPartsStep = 1; + const uint32_t FinalizeBuildStep = UploadPartsStep + PartsUploadStepCount; + const uint32_t CleanupStep = FinalizeBuildStep + 1; + const uint32_t StepCount = CleanupStep + 1; + + auto EndProgress = MakeGuard([&]() { m_Progress.SetLogOperationProgress(StepCount, StepCount); }); + + Stopwatch ProcessTimer; + + CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir); + CreateDirectories(m_Options.TempDir); + auto _ = MakeGuard([&]() { CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir); }); + + m_Progress.SetLogOperationProgress(PrepareBuildStep, StepCount); + + m_PrepBuildResultFuture = m_NetworkPool.EnqueueTask(std::packaged_task{[this] { return PrepareBuild(); }}, + WorkerThreadPool::EMode::EnableBacklog); + + for (uint32_t PartIndex = 0; PartIndex < UploadParts.size(); PartIndex++) + { + const uint32_t PartStepOffset = UploadPartsStep + (PartIndex * uint32_t(PartTaskSteps::StepCount)); + + const UploadPart& Part = UploadParts[PartIndex]; + UploadBuildPart(ChunkController, ChunkCache, PartIndex, Part, PartStepOffset, StepCount); + if (m_AbortFlag) + { + return {}; + } + } + + m_Progress.SetLogOperationProgress(FinalizeBuildStep, StepCount); + + if (m_CreateBuild && !m_AbortFlag) + { + Stopwatch FinalizeBuildTimer; + m_Storage.BuildStorage->FinalizeBuild(m_BuildId); + if (!m_Options.IsQuiet) + { + ZEN_INFO("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs())); + } + } + + m_Progress.SetLogOperationProgress(CleanupStep, StepCount); + + std::vector> Result; + Result.reserve(UploadParts.size()); + for (UploadPart& Part : UploadParts) + { + Result.push_back(std::make_pair(Part.PartId, Part.PartName)); + } + return Result; + } + catch (const std::exception&) + { + m_AbortFlag = true; + throw; + } +} + +bool +BuildsOperationUploadFolder::IsAcceptedFolder(const std::string_view& RelativePath) const +{ + for (const std::string& ExcludeFolder : m_Options.ExcludeFolders) + { + if (RelativePath.starts_with(ExcludeFolder)) + { + if (RelativePath.length() == ExcludeFolder.length()) + { + return false; + } + else if (RelativePath[ExcludeFolder.length()] == '/') + { + return false; + } + } + } + return true; +} + +bool +BuildsOperationUploadFolder::IsAcceptedFile(const std::string_view& RelativePath) const +{ + if (RelativePath == m_Options.ZenExcludeManifestName) + { + return false; + } + for (const std::string& ExcludeExtension : m_Options.ExcludeExtensions) + { + if (RelativePath.ends_with(ExcludeExtension)) + { + return false; + } + } + return true; +} + +void +BuildsOperationUploadFolder::ArrangeChunksIntoBlocks(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + std::vector& ChunkIndexes, + std::vector>& OutBlocks) +{ + ZEN_TRACE_CPU("ArrangeChunksIntoBlocks"); + std::sort(ChunkIndexes.begin(), ChunkIndexes.end(), [&Content, &Lookup](uint32_t Lhs, uint32_t Rhs) { + const ChunkedContentLookup::ChunkSequenceLocation& LhsLocation = GetChunkSequenceLocations(Lookup, Lhs)[0]; + const ChunkedContentLookup::ChunkSequenceLocation& RhsLocation = GetChunkSequenceLocations(Lookup, Rhs)[0]; + if (LhsLocation.SequenceIndex < RhsLocation.SequenceIndex) + { + return true; + } + else if (LhsLocation.SequenceIndex > RhsLocation.SequenceIndex) + { + return false; + } + return LhsLocation.Offset < RhsLocation.Offset; + }); + + uint64_t MaxBlockSizeLowThreshold = m_Options.BlockParameters.MaxBlockSize - (m_Options.BlockParameters.MaxBlockSize / 16); + + uint64_t BlockSize = 0; + + uint32_t ChunkIndexStart = 0; + for (uint32_t ChunkIndexOffset = 0; ChunkIndexOffset < ChunkIndexes.size();) + { + const uint32_t ChunkIndex = ChunkIndexes[ChunkIndexOffset]; + const uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + + if (((BlockSize + ChunkSize) > m_Options.BlockParameters.MaxBlockSize) || + (ChunkIndexOffset - ChunkIndexStart) > m_Options.BlockParameters.MaxChunksPerBlock) + { + // Within the span of MaxBlockSizeLowThreshold and MaxBlockSize, see if there is a break + // between source paths for chunks. Break the block at the last such break if any. + ZEN_ASSERT(ChunkIndexOffset > ChunkIndexStart); + + const uint32_t ChunkSequenceIndex = Lookup.ChunkSequenceLocations[Lookup.ChunkSequenceLocationOffset[ChunkIndex]].SequenceIndex; + + uint64_t ScanBlockSize = BlockSize; + + uint32_t ScanChunkIndexOffset = ChunkIndexOffset - 1; + while (ScanChunkIndexOffset > (ChunkIndexStart + 2)) + { + const uint32_t TestChunkIndex = ChunkIndexes[ScanChunkIndexOffset]; + const uint64_t TestChunkSize = Content.ChunkedContent.ChunkRawSizes[TestChunkIndex]; + if ((ScanBlockSize - TestChunkSize) < MaxBlockSizeLowThreshold) + { + break; + } + + const uint32_t TestSequenceIndex = + Lookup.ChunkSequenceLocations[Lookup.ChunkSequenceLocationOffset[TestChunkIndex]].SequenceIndex; + if (ChunkSequenceIndex != TestSequenceIndex) + { + ChunkIndexOffset = ScanChunkIndexOffset + 1; + break; + } + + ScanBlockSize -= TestChunkSize; + ScanChunkIndexOffset--; + } + + std::vector ChunksInBlock; + ChunksInBlock.reserve(ChunkIndexOffset - ChunkIndexStart); + for (uint32_t AddIndexOffset = ChunkIndexStart; AddIndexOffset < ChunkIndexOffset; AddIndexOffset++) + { + const uint32_t AddChunkIndex = ChunkIndexes[AddIndexOffset]; + ChunksInBlock.push_back(AddChunkIndex); + } + OutBlocks.emplace_back(std::move(ChunksInBlock)); + BlockSize = 0; + ChunkIndexStart = ChunkIndexOffset; + } + else + { + ChunkIndexOffset++; + BlockSize += ChunkSize; + } + } + if (ChunkIndexStart < ChunkIndexes.size()) + { + std::vector ChunksInBlock; + ChunksInBlock.reserve(ChunkIndexes.size() - ChunkIndexStart); + for (uint32_t AddIndexOffset = ChunkIndexStart; AddIndexOffset < ChunkIndexes.size(); AddIndexOffset++) + { + const uint32_t AddChunkIndex = ChunkIndexes[AddIndexOffset]; + ChunksInBlock.push_back(AddChunkIndex); + } + OutBlocks.emplace_back(std::move(ChunksInBlock)); + } +} + +void +BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const std::vector>& NewBlockChunks, + GeneratedBlocks& OutBlocks, + GenerateBlocksStatistics& GenerateBlocksStats, + UploadStatistics& UploadStats) +{ + ZEN_TRACE_CPU("GenerateBuildBlocks"); + const std::size_t NewBlockCount = NewBlockChunks.size(); + if (NewBlockCount == 0) + { + return; + } + + std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Generate Blocks"); + + OutBlocks.BlockDescriptions.resize(NewBlockCount); + OutBlocks.BlockSizes.resize(NewBlockCount); + OutBlocks.BlockMetaDatas.resize(NewBlockCount); + OutBlocks.BlockHeaders.resize(NewBlockCount); + OutBlocks.MetaDataHasBeenUploaded.resize(NewBlockCount, 0); + OutBlocks.BlockHashToBlockIndex.reserve(NewBlockCount); + + RwLock Lock; + FilteredRate FilteredGeneratedBytesPerSecond; + FilteredRate FilteredUploadedBytesPerSecond; + ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + std::atomic QueuedPendingBlocksForUpload = 0; + + GenerateBuildBlocksContext Context{.Work = Work, + .GenerateBlobsPool = m_IOWorkerPool, + .UploadBlocksPool = m_NetworkPool, + .FilteredGeneratedBytesPerSecond = FilteredGeneratedBytesPerSecond, + .FilteredUploadedBytesPerSecond = FilteredUploadedBytesPerSecond, + .QueuedPendingBlocksForUpload = QueuedPendingBlocksForUpload, + .Lock = Lock, + .OutBlocks = OutBlocks, + .GenerateBlocksStats = GenerateBlocksStats, + .UploadStats = UploadStats, + .NewBlockCount = NewBlockCount}; + + ScheduleBlockGeneration(Context, Content, Lookup, NewBlockChunks); + + Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); + + FilteredGeneratedBytesPerSecond.Update(GenerateBlocksStats.GeneratedBlockByteCount.load()); + FilteredUploadedBytesPerSecond.Update(UploadStats.BlocksBytes.load()); + + std::string Details = fmt::format("Generated {}/{} ({}, {}B/s). Uploaded {}/{} ({}, {}bits/s)", + GenerateBlocksStats.GeneratedBlockCount.load(), + NewBlockCount, + NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()), + NiceNum(FilteredGeneratedBytesPerSecond.GetCurrent()), + UploadStats.BlockCount.load(), + NewBlockCount, + NiceBytes(UploadStats.BlocksBytes.load()), + NiceNum(FilteredUploadedBytesPerSecond.GetCurrent() * 8)); + + ProgressBar->UpdateState({.Task = "Generating blocks", + .Details = Details, + .TotalCount = gsl::narrow(NewBlockCount), + .RemainingCount = gsl::narrow(NewBlockCount - GenerateBlocksStats.GeneratedBlockCount.load()), + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); + }); + + ZEN_ASSERT(m_AbortFlag || QueuedPendingBlocksForUpload.load() == 0); + + ProgressBar->Finish(); + + GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGeneratedBytesPerSecond.GetElapsedTimeUS(); + UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTimeUS(); +} + +void +BuildsOperationUploadFolder::ScheduleBlockGeneration(GenerateBuildBlocksContext& Context, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const std::vector>& NewBlockChunks) +{ + for (size_t BlockIndex = 0; BlockIndex < Context.NewBlockCount; BlockIndex++) + { + if (Context.Work.IsAborted()) + { + break; + } + const std::vector& ChunksInBlock = NewBlockChunks[BlockIndex]; + Context.Work.ScheduleWork( + Context.GenerateBlobsPool, + [this, &Context, &Content, &Lookup, ChunksInBlock, BlockIndex](std::atomic&) { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("GenerateBuildBlocks_Generate"); + + Context.FilteredGeneratedBytesPerSecond.Start(); + + Stopwatch GenerateTimer; + CompressedBuffer CompressedBlock = + GenerateBlock(Content, Lookup, ChunksInBlock, Context.OutBlocks.BlockDescriptions[BlockIndex]); + if (m_Options.IsVerbose) + { + ZEN_INFO("Generated block {} ({}) containing {} chunks in {}", + Context.OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(CompressedBlock.GetCompressedSize()), + Context.OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(), + NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs())); + } + + Context.OutBlocks.BlockSizes[BlockIndex] = CompressedBlock.GetCompressedSize(); + { + CbObjectWriter Writer; + Writer.AddString("createdBy", "zen"); + Context.OutBlocks.BlockMetaDatas[BlockIndex] = Writer.Save(); + } + Context.GenerateBlocksStats.GeneratedBlockByteCount += Context.OutBlocks.BlockSizes[BlockIndex]; + Context.GenerateBlocksStats.GeneratedBlockCount++; + + Context.Lock.WithExclusiveLock([&]() { + Context.OutBlocks.BlockHashToBlockIndex.insert_or_assign(Context.OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + BlockIndex); + }); + + { + std::span Segments = CompressedBlock.GetCompressed().GetSegments(); + ZEN_ASSERT(Segments.size() >= 2); + Context.OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); + } + + if (Context.GenerateBlocksStats.GeneratedBlockCount == Context.NewBlockCount) + { + Context.FilteredGeneratedBytesPerSecond.Stop(); + } + + if (Context.QueuedPendingBlocksForUpload.load() > 16) + { + std::span Segments = CompressedBlock.GetCompressed().GetSegments(); + ZEN_ASSERT(Segments.size() >= 2); + Context.OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); + } + else + { + if (!m_AbortFlag) + { + Context.QueuedPendingBlocksForUpload++; + Context.Work.ScheduleWork( + Context.UploadBlocksPool, + [this, &Context, BlockIndex, Payload = std::move(CompressedBlock)](std::atomic&) mutable { + UploadGeneratedBlock(Context, BlockIndex, std::move(Payload)); + }); + } + } + } + }); + } +} + +void +BuildsOperationUploadFolder::UploadGeneratedBlock(GenerateBuildBlocksContext& Context, size_t BlockIndex, CompressedBuffer Payload) +{ + auto _ = MakeGuard([&Context] { Context.QueuedPendingBlocksForUpload--; }); + if (m_AbortFlag) + { + return; + } + + if (Context.GenerateBlocksStats.GeneratedBlockCount == Context.NewBlockCount) + { + ZEN_TRACE_CPU("GenerateBuildBlocks_Save"); + + Context.FilteredUploadedBytesPerSecond.Stop(); + std::span Segments = Payload.GetCompressed().GetSegments(); + ZEN_ASSERT(Segments.size() >= 2); + Context.OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); + return; + } + + ZEN_TRACE_CPU("GenerateBuildBlocks_Upload"); + + Context.FilteredUploadedBytesPerSecond.Start(); + + const CbObject BlockMetaData = + BuildChunkBlockDescription(Context.OutBlocks.BlockDescriptions[BlockIndex], Context.OutBlocks.BlockMetaDatas[BlockIndex]); + + const IoHash& BlockHash = Context.OutBlocks.BlockDescriptions[BlockIndex].BlockHash; + const uint64_t CompressedBlockSize = Payload.GetCompressedSize(); + + if (m_Storage.CacheStorage && m_Options.PopulateCache) + { + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload.GetCompressed()); + } + + try + { + m_Storage.BuildStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, std::move(Payload).GetCompressed()); + } + catch (const std::exception&) + { + // Silence http errors due to abort + if (!m_AbortFlag) + { + throw; + } + } + + if (m_AbortFlag) + { + return; + } + + Context.UploadStats.BlocksBytes += CompressedBlockSize; + + if (m_Options.IsVerbose) + { + ZEN_INFO("Uploaded block {} ({}) containing {} chunks", + BlockHash, + NiceBytes(CompressedBlockSize), + Context.OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); + } + + if (m_Storage.CacheStorage && m_Options.PopulateCache) + { + m_Storage.CacheStorage->PutBlobMetadatas(m_BuildId, std::vector({BlockHash}), std::vector({BlockMetaData})); + } + + bool MetadataSucceeded = false; + try + { + MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData); + } + catch (const std::exception&) + { + // Silence http errors due to abort + if (!m_AbortFlag) + { + throw; + } + } + + if (m_AbortFlag) + { + return; + } + + if (MetadataSucceeded) + { + if (m_Options.IsVerbose) + { + ZEN_INFO("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize())); + } + + Context.OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + Context.UploadStats.BlocksBytes += BlockMetaData.GetSize(); + } + + Context.UploadStats.BlockCount++; + if (Context.UploadStats.BlockCount == Context.NewBlockCount) + { + Context.FilteredUploadedBytesPerSecond.Stop(); + } +} + +std::vector +BuildsOperationUploadFolder::CalculateAbsoluteChunkOrders( + const std::span LocalChunkHashes, + const std::span LocalChunkOrder, + const tsl::robin_map& ChunkHashToLocalChunkIndex, + const std::span& LooseChunkIndexes, + const std::span& BlockDescriptions) +{ + ZEN_TRACE_CPU("CalculateAbsoluteChunkOrders"); + + std::vector TmpAbsoluteChunkHashes; + if (m_Options.DoExtraContentValidation) + { + TmpAbsoluteChunkHashes.reserve(LocalChunkHashes.size()); + } + std::vector LocalChunkIndexToAbsoluteChunkIndex; + LocalChunkIndexToAbsoluteChunkIndex.resize(LocalChunkHashes.size(), (uint32_t)-1); + std::uint32_t AbsoluteChunkCount = 0; + for (uint32_t ChunkIndex : LooseChunkIndexes) + { + LocalChunkIndexToAbsoluteChunkIndex[ChunkIndex] = AbsoluteChunkCount; + if (m_Options.DoExtraContentValidation) + { + TmpAbsoluteChunkHashes.push_back(LocalChunkHashes[ChunkIndex]); + } + AbsoluteChunkCount++; + } + for (const ChunkBlockDescription& Block : BlockDescriptions) + { + for (const IoHash& ChunkHash : Block.ChunkRawHashes) + { + if (auto It = ChunkHashToLocalChunkIndex.find(ChunkHash); It != ChunkHashToLocalChunkIndex.end()) + { + const uint32_t LocalChunkIndex = It->second; + ZEN_ASSERT_SLOW(LocalChunkHashes[LocalChunkIndex] == ChunkHash); + LocalChunkIndexToAbsoluteChunkIndex[LocalChunkIndex] = AbsoluteChunkCount; + } + if (m_Options.DoExtraContentValidation) + { + TmpAbsoluteChunkHashes.push_back(ChunkHash); + } + AbsoluteChunkCount++; + } + } + std::vector AbsoluteChunkOrder; + AbsoluteChunkOrder.reserve(LocalChunkHashes.size()); + for (const uint32_t LocalChunkIndex : LocalChunkOrder) + { + const uint32_t AbsoluteChunkIndex = LocalChunkIndexToAbsoluteChunkIndex[LocalChunkIndex]; + if (m_Options.DoExtraContentValidation) + { + ZEN_ASSERT(LocalChunkHashes[LocalChunkIndex] == TmpAbsoluteChunkHashes[AbsoluteChunkIndex]); + } + AbsoluteChunkOrder.push_back(AbsoluteChunkIndex); + } + if (m_Options.DoExtraContentValidation) + { + uint32_t OrderIndex = 0; + while (OrderIndex < LocalChunkOrder.size()) + { + const uint32_t LocalChunkIndex = LocalChunkOrder[OrderIndex]; + const IoHash& LocalChunkHash = LocalChunkHashes[LocalChunkIndex]; + const uint32_t AbsoluteChunkIndex = AbsoluteChunkOrder[OrderIndex]; + const IoHash& AbsoluteChunkHash = TmpAbsoluteChunkHashes[AbsoluteChunkIndex]; + ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash); + OrderIndex++; + } + } + return AbsoluteChunkOrder; +} + +CompositeBuffer +BuildsOperationUploadFolder::FetchChunk(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const IoHash& ChunkHash, + ReadFileCache& OpenFileCache) +{ + ZEN_TRACE_CPU("FetchChunk"); + auto It = Lookup.ChunkHashToChunkIndex.find(ChunkHash); + ZEN_ASSERT(It != Lookup.ChunkHashToChunkIndex.end()); + uint32_t ChunkIndex = It->second; + std::span ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex); + ZEN_ASSERT(!ChunkLocations.empty()); + CompositeBuffer Chunk = + OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, ChunkLocations[0].Offset, Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); + if (!Chunk) + { + throw std::runtime_error(fmt::format("Unable to read chunk at {}, size {} from '{}'", + ChunkLocations[0].Offset, + Content.ChunkedContent.ChunkRawSizes[ChunkIndex], + Content.Paths[Lookup.SequenceIndexFirstPathIndex[ChunkLocations[0].SequenceIndex]])); + } + ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == ChunkHash); + return Chunk; +}; + +CompressedBuffer +BuildsOperationUploadFolder::GenerateBlock(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const std::vector& ChunksInBlock, + ChunkBlockDescription& OutBlockDescription) +{ + ZEN_TRACE_CPU("GenerateBlock"); + ReadFileCache OpenFileCache(m_DiskStats.OpenReadCount, + m_DiskStats.CurrentOpenFileCount, + m_DiskStats.ReadCount, + m_DiskStats.ReadByteCount, + m_Path, + Content, + Lookup, + 4); + + std::vector> BlockContent; + BlockContent.reserve(ChunksInBlock.size()); + for (uint32_t ChunkIndex : ChunksInBlock) + { + BlockContent.emplace_back(std::make_pair( + Content.ChunkedContent.ChunkHashes[ChunkIndex], + [this, &Content, &Lookup, &OpenFileCache, ChunkIndex](const IoHash& ChunkHash) -> std::pair { + CompositeBuffer Chunk = FetchChunk(Content, Lookup, ChunkHash, OpenFileCache); + ZEN_ASSERT(Chunk); + uint64_t RawSize = Chunk.GetSize(); + + const bool ShouldCompressChunk = RawSize >= m_Options.MinimumSizeForCompressInBlock && + IsChunkCompressable(m_NonCompressableExtensionHashes, Lookup, ChunkIndex); + + const OodleCompressionLevel CompressionLevel = + ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None; + return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, CompressionLevel).GetCompressed()}; + })); + } + + return GenerateChunkBlock(std::move(BlockContent), OutBlockDescription); +}; + +CompressedBuffer +BuildsOperationUploadFolder::RebuildBlock(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + CompositeBuffer&& HeaderBuffer, + const std::vector& ChunksInBlock) +{ + ZEN_TRACE_CPU("RebuildBlock"); + ReadFileCache OpenFileCache(m_DiskStats.OpenReadCount, + m_DiskStats.CurrentOpenFileCount, + m_DiskStats.ReadCount, + m_DiskStats.ReadByteCount, + m_Path, + Content, + Lookup, + 4); + + std::vector ResultBuffers; + ResultBuffers.reserve(HeaderBuffer.GetSegments().size() + ChunksInBlock.size()); + ResultBuffers.insert(ResultBuffers.end(), HeaderBuffer.GetSegments().begin(), HeaderBuffer.GetSegments().end()); + for (uint32_t ChunkIndex : ChunksInBlock) + { + std::span ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex); + ZEN_ASSERT(!ChunkLocations.empty()); + CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, + ChunkLocations[0].Offset, + Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); + ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == Content.ChunkedContent.ChunkHashes[ChunkIndex]); + + const uint64_t RawSize = Chunk.GetSize(); + const bool ShouldCompressChunk = + RawSize >= m_Options.MinimumSizeForCompressInBlock && IsChunkCompressable(m_NonCompressableExtensionHashes, Lookup, ChunkIndex); + + const OodleCompressionLevel CompressionLevel = ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None; + + CompositeBuffer CompressedChunk = + CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, CompressionLevel).GetCompressed(); + ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); + } + return CompressedBuffer::FromCompressedNoValidate(CompositeBuffer(std::move(ResultBuffers))); +}; + +void +BuildsOperationUploadFolder::UploadBuildPart(ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + uint32_t PartIndex, + const UploadPart& Part, + uint32_t PartStepOffset, + uint32_t StepCount) +{ + Stopwatch UploadTimer; + + ChunkingStatistics ChunkingStats; + FindBlocksStatistics FindBlocksStats; + ReuseBlocksStatistics ReuseBlocksStats; + UploadStatistics UploadStats; + GenerateBlocksStatistics GenerateBlocksStats; + LooseChunksStatistics LooseChunksStats; + + m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::ChunkPartContent, StepCount); + + ChunkedFolderContent LocalContent = ScanPartContent(Part, ChunkController, ChunkCache, ChunkingStats); + if (m_AbortFlag) + { + return; + } + + const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent); + + if (PartIndex == 0) + { + ConsumePrepareBuildResult(); + } + + ZEN_ASSERT(m_PreferredMultipartChunkSize != 0); + ZEN_ASSERT(m_LargeAttachmentSize != 0); + + m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::CalculateDelta, StepCount); + + Stopwatch BlockArrangeTimer; + + std::vector LooseChunkIndexes; + std::vector NewBlockChunkIndexes; + std::vector ReuseBlockIndexes; + ClassifyChunksByBlockEligibility(LocalContent, + LooseChunkIndexes, + NewBlockChunkIndexes, + ReuseBlockIndexes, + LooseChunksStats, + FindBlocksStats, + ReuseBlocksStats); + + std::vector> NewBlockChunks; + ArrangeChunksIntoBlocks(LocalContent, LocalLookup, NewBlockChunkIndexes, NewBlockChunks); + + FindBlocksStats.NewBlocksCount += NewBlockChunks.size(); + for (uint32_t ChunkIndex : NewBlockChunkIndexes) + { + FindBlocksStats.NewBlocksChunkByteCount += LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + } + FindBlocksStats.NewBlocksChunkCount += NewBlockChunkIndexes.size(); + + const double AcceptedByteCountPercent = FindBlocksStats.PotentialChunkByteCount > 0 + ? (100.0 * ReuseBlocksStats.AcceptedRawByteCount / FindBlocksStats.PotentialChunkByteCount) + : 0.0; + + const double AcceptedReduntantByteCountPercent = + ReuseBlocksStats.AcceptedByteCount > 0 ? (100.0 * ReuseBlocksStats.AcceptedReduntantByteCount) / + (ReuseBlocksStats.AcceptedByteCount + ReuseBlocksStats.AcceptedReduntantByteCount) + : 0.0; + if (!m_Options.IsQuiet) + { + ZEN_INFO( + "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n" + " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n" + " Accepting {} ({}) redundant chunks ({:.1f}%)\n" + " Rejected {} ({}) chunks in {} blocks\n" + " Arranged {} ({}) chunks in {} new blocks\n" + " Keeping {} ({}) chunks as loose chunks\n" + " Discovery completed in {}", + FindBlocksStats.FoundBlockChunkCount, + FindBlocksStats.FoundBlockCount, + NiceBytes(FindBlocksStats.FoundBlockByteCount), + NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS), + + ReuseBlocksStats.AcceptedChunkCount, + NiceBytes(ReuseBlocksStats.AcceptedRawByteCount), + FindBlocksStats.AcceptedBlockCount, + AcceptedByteCountPercent, + + ReuseBlocksStats.AcceptedReduntantChunkCount, + NiceBytes(ReuseBlocksStats.AcceptedReduntantByteCount), + AcceptedReduntantByteCountPercent, + + ReuseBlocksStats.RejectedChunkCount, + NiceBytes(ReuseBlocksStats.RejectedByteCount), + ReuseBlocksStats.RejectedBlockCount, + + FindBlocksStats.NewBlocksChunkCount, + NiceBytes(FindBlocksStats.NewBlocksChunkByteCount), + FindBlocksStats.NewBlocksCount, + + LooseChunksStats.ChunkCount, + NiceBytes(LooseChunksStats.ChunkByteCount), + + NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs())); + } + + m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::GenerateBlocks, StepCount); + GeneratedBlocks NewBlocks; + + if (!NewBlockChunks.empty()) + { + Stopwatch GenerateBuildBlocksTimer; + auto __ = MakeGuard([&]() { + uint64_t BlockGenerateTimeUs = GenerateBuildBlocksTimer.GetElapsedTimeUs(); + if (!m_Options.IsQuiet) + { + ZEN_INFO("Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.", + GenerateBlocksStats.GeneratedBlockCount.load(), + NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount), + UploadStats.BlockCount.load(), + NiceBytes(UploadStats.BlocksBytes.load()), + NiceTimeSpanMs(BlockGenerateTimeUs / 1000), + NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, + GenerateBlocksStats.GeneratedBlockByteCount)), + NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8))); + } + }); + GenerateBuildBlocks(LocalContent, LocalLookup, NewBlockChunks, NewBlocks, GenerateBlocksStats, UploadStats); + } + + m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::BuildPartManifest, StepCount); + + BuiltPartManifest Manifest = + BuildPartManifestObject(LocalContent, LocalLookup, ChunkController, ReuseBlockIndexes, NewBlocks, LooseChunkIndexes); + + m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::UploadBuildPart, StepCount); + + Stopwatch PutBuildPartResultTimer; + std::pair> PutBuildPartResult = + m_Storage.BuildStorage->PutBuildPart(m_BuildId, Part.PartId, Part.PartName, Manifest.PartManifest); + if (!m_Options.IsQuiet) + { + ZEN_INFO("PutBuildPart took {}, payload size {}. {} attachments are needed.", + NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()), + NiceBytes(Manifest.PartManifest.GetSize()), + PutBuildPartResult.second.size()); + } + IoHash PartHash = PutBuildPartResult.first; + + m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::UploadAttachments, StepCount); + + std::vector UnknownChunks; + if (m_Options.IgnoreExistingBlocks) + { + if (m_Options.IsVerbose) + { + ZEN_INFO("PutBuildPart uploading all attachments, needs are: {}", FormatArray(PutBuildPartResult.second, "\n "sv)); + } + + std::vector ForceUploadChunkHashes; + ForceUploadChunkHashes.reserve(LooseChunkIndexes.size()); + + for (uint32_t ChunkIndex : LooseChunkIndexes) + { + ForceUploadChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); + } + + for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockHeaders.size(); BlockIndex++) + { + if (NewBlocks.BlockHeaders[BlockIndex]) + { + // Block was not uploaded during generation + ForceUploadChunkHashes.push_back(NewBlocks.BlockDescriptions[BlockIndex].BlockHash); + } + } + UploadAttachmentBatch(ForceUploadChunkHashes, + UnknownChunks, + LocalContent, + LocalLookup, + NewBlockChunks, + NewBlocks, + LooseChunkIndexes, + UploadStats, + LooseChunksStats); + } + else if (!PutBuildPartResult.second.empty()) + { + if (m_Options.IsVerbose) + { + ZEN_INFO("PutBuildPart needs attachments: {}", FormatArray(PutBuildPartResult.second, "\n "sv)); + } + UploadAttachmentBatch(PutBuildPartResult.second, + UnknownChunks, + LocalContent, + LocalLookup, + NewBlockChunks, + NewBlocks, + LooseChunkIndexes, + UploadStats, + LooseChunksStats); + } + + FinalizeBuildPartWithRetries(Part, + PartHash, + UnknownChunks, + LocalContent, + LocalLookup, + NewBlockChunks, + NewBlocks, + LooseChunkIndexes, + UploadStats, + LooseChunksStats); + + if (!NewBlocks.BlockDescriptions.empty() && !m_AbortFlag) + { + UploadMissingBlockMetadata(NewBlocks, UploadStats); + // The newly generated blocks are now known blocks so the next part upload can use those blocks as well + m_KnownBlocks.insert(m_KnownBlocks.end(), NewBlocks.BlockDescriptions.begin(), NewBlocks.BlockDescriptions.end()); + } + + m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::PutBuildPartStats, StepCount); + + m_Storage.BuildStorage->PutBuildPartStats( + m_BuildId, + Part.PartId, + {{"totalSize", double(Part.LocalFolderScanStats.FoundFileByteCount.load())}, + {"reusedRatio", AcceptedByteCountPercent / 100.0}, + {"reusedBlockCount", double(FindBlocksStats.AcceptedBlockCount)}, + {"reusedBlockByteCount", double(ReuseBlocksStats.AcceptedRawByteCount)}, + {"newBlockCount", double(FindBlocksStats.NewBlocksCount)}, + {"newBlockByteCount", double(FindBlocksStats.NewBlocksChunkByteCount)}, + {"uploadedCount", double(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load())}, + {"uploadedByteCount", double(UploadStats.BlocksBytes.load() + UploadStats.ChunksBytes.load())}, + {"uploadedBytesPerSec", + double(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.ChunksBytes + UploadStats.BlocksBytes))}, + {"elapsedTimeSec", double(UploadTimer.GetElapsedTimeMs() / 1000.0)}}); + + m_LocalFolderScanStats += Part.LocalFolderScanStats; + m_ChunkingStats += ChunkingStats; + m_FindBlocksStats += FindBlocksStats; + m_ReuseBlocksStats += ReuseBlocksStats; + m_UploadStats += UploadStats; + m_GenerateBlocksStats += GenerateBlocksStats; + m_LooseChunksStats += LooseChunksStats; +} + +ChunkedFolderContent +BuildsOperationUploadFolder::ScanPartContent(const UploadPart& Part, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + ChunkingStatistics& ChunkingStats) +{ + Stopwatch ScanTimer; + + std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Scan Folder"); + + FilteredRate FilteredBytesHashed; + FilteredBytesHashed.Start(); + ChunkedFolderContent LocalContent = ChunkFolderContent( + ChunkingStats, + m_IOWorkerPool, + m_Path, + Part.Content, + ChunkController, + ChunkCache, + m_Progress.GetProgressUpdateDelayMS(), + [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { + FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", + ChunkingStats.FilesProcessed.load(), + Part.Content.Paths.size(), + NiceBytes(ChunkingStats.BytesHashed.load()), + NiceBytes(Part.TotalRawSize), + NiceNum(FilteredBytesHashed.GetCurrent()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load())); + ProgressBar->UpdateState({.Task = "Scanning files ", + .Details = Details, + .TotalCount = Part.TotalRawSize, + .RemainingCount = Part.TotalRawSize - ChunkingStats.BytesHashed.load(), + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); + }, + m_AbortFlag, + m_PauseFlag); + FilteredBytesHashed.Stop(); + ProgressBar->Finish(); + if (m_AbortFlag) + { + return LocalContent; + } + + if (!m_Options.IsQuiet) + { + ZEN_INFO("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", + Part.Content.Paths.size(), + NiceBytes(Part.TotalRawSize), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load()), + m_Path, + NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()), + NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + } + + return LocalContent; +} + +void +BuildsOperationUploadFolder::ConsumePrepareBuildResult() +{ + const PrepareBuildResult PrepBuildResult = m_PrepBuildResultFuture.get(); + + m_FindBlocksStats.FindBlockTimeMS = PrepBuildResult.ElapsedTimeMs; + m_FindBlocksStats.FoundBlockCount = PrepBuildResult.KnownBlocks.size(); + + if (!m_Options.IsQuiet) + { + ZEN_INFO("Build prepare took {}. {} took {}, payload size {}{}", + NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs), + m_CreateBuild ? "PutBuild" : "GetBuild", + NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs), + NiceBytes(PrepBuildResult.PayloadSize), + m_Options.IgnoreExistingBlocks ? "" + : fmt::format(". Found {} blocks in {}", + PrepBuildResult.KnownBlocks.size(), + NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs))); + } + + m_PreferredMultipartChunkSize = PrepBuildResult.PreferredMultipartChunkSize; + m_LargeAttachmentSize = m_Options.AllowMultiparts ? m_PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; + m_KnownBlocks = std::move(PrepBuildResult.KnownBlocks); +} + +void +BuildsOperationUploadFolder::ClassifyChunksByBlockEligibility(const ChunkedFolderContent& LocalContent, + std::vector& OutLooseChunkIndexes, + std::vector& OutNewBlockChunkIndexes, + std::vector& OutReuseBlockIndexes, + LooseChunksStatistics& LooseChunksStats, + FindBlocksStatistics& FindBlocksStats, + ReuseBlocksStatistics& ReuseBlocksStats) +{ + const bool EnableBlocks = true; + std::vector BlockChunkIndexes; + for (uint32_t ChunkIndex = 0; ChunkIndex < LocalContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++) + { + const uint64_t ChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + if (!EnableBlocks || ChunkRawSize == 0 || ChunkRawSize > m_Options.BlockParameters.MaxChunkEmbedSize) + { + OutLooseChunkIndexes.push_back(ChunkIndex); + LooseChunksStats.ChunkByteCount += ChunkRawSize; + } + else + { + BlockChunkIndexes.push_back(ChunkIndex); + FindBlocksStats.PotentialChunkByteCount += ChunkRawSize; + } + } + FindBlocksStats.PotentialChunkCount += BlockChunkIndexes.size(); + LooseChunksStats.ChunkCount = OutLooseChunkIndexes.size(); + + if (m_Options.IgnoreExistingBlocks) + { + if (!m_Options.IsQuiet) + { + ZEN_INFO("Ignoring any existing blocks in store"); + } + OutNewBlockChunkIndexes = std::move(BlockChunkIndexes); + return; + } + + OutReuseBlockIndexes = FindReuseBlocks(Log(), + m_Options.BlockReuseMinPercentLimit, + m_Options.IsVerbose, + ReuseBlocksStats, + m_KnownBlocks, + LocalContent.ChunkedContent.ChunkHashes, + BlockChunkIndexes, + OutNewBlockChunkIndexes); + FindBlocksStats.AcceptedBlockCount += OutReuseBlockIndexes.size(); + + for (const ChunkBlockDescription& Description : m_KnownBlocks) + { + for (uint32_t ChunkRawLength : Description.ChunkRawLengths) + { + FindBlocksStats.FoundBlockByteCount += ChunkRawLength; + } + FindBlocksStats.FoundBlockChunkCount += Description.ChunkRawHashes.size(); + } +} + +BuildsOperationUploadFolder::BuiltPartManifest +BuildsOperationUploadFolder::BuildPartManifestObject(const ChunkedFolderContent& LocalContent, + const ChunkedContentLookup& LocalLookup, + ChunkingController& ChunkController, + std::span ReuseBlockIndexes, + const GeneratedBlocks& NewBlocks, + std::span LooseChunkIndexes) +{ + BuiltPartManifest Result; + + CbObjectWriter PartManifestWriter; + Stopwatch ManifestGenerationTimer; + auto __ = MakeGuard([&]() { + if (!m_Options.IsQuiet) + { + ZEN_INFO("Generated build part manifest in {} ({})", + NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()), + NiceBytes(PartManifestWriter.GetSaveSize())); + } + }); + + PartManifestWriter.BeginObject("chunker"sv); + { + PartManifestWriter.AddString("name"sv, ChunkController.GetName()); + PartManifestWriter.AddObject("parameters"sv, ChunkController.GetParameters()); + } + PartManifestWriter.EndObject(); // chunker + + Result.AllChunkBlockHashes.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size()); + Result.AllChunkBlockDescriptions.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size()); + for (size_t ReuseBlockIndex : ReuseBlockIndexes) + { + Result.AllChunkBlockDescriptions.push_back(m_KnownBlocks[ReuseBlockIndex]); + Result.AllChunkBlockHashes.push_back(m_KnownBlocks[ReuseBlockIndex].BlockHash); + } + Result.AllChunkBlockDescriptions.insert(Result.AllChunkBlockDescriptions.end(), + NewBlocks.BlockDescriptions.begin(), + NewBlocks.BlockDescriptions.end()); + for (const ChunkBlockDescription& BlockDescription : NewBlocks.BlockDescriptions) + { + Result.AllChunkBlockHashes.push_back(BlockDescription.BlockHash); + } + + std::vector AbsoluteChunkHashes; + if (m_Options.DoExtraContentValidation) + { + tsl::robin_map ChunkHashToAbsoluteChunkIndex; + AbsoluteChunkHashes.reserve(LocalContent.ChunkedContent.ChunkHashes.size()); + for (uint32_t ChunkIndex : LooseChunkIndexes) + { + ChunkHashToAbsoluteChunkIndex.insert({LocalContent.ChunkedContent.ChunkHashes[ChunkIndex], AbsoluteChunkHashes.size()}); + AbsoluteChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); + } + for (const ChunkBlockDescription& Block : Result.AllChunkBlockDescriptions) + { + for (const IoHash& ChunkHash : Block.ChunkRawHashes) + { + ChunkHashToAbsoluteChunkIndex.insert({ChunkHash, AbsoluteChunkHashes.size()}); + AbsoluteChunkHashes.push_back(ChunkHash); + } + } + for (const IoHash& ChunkHash : LocalContent.ChunkedContent.ChunkHashes) + { + ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(ChunkHash)] == ChunkHash); + ZEN_ASSERT(LocalContent.ChunkedContent.ChunkHashes[LocalLookup.ChunkHashToChunkIndex.at(ChunkHash)] == ChunkHash); + } + for (const uint32_t ChunkIndex : LocalContent.ChunkedContent.ChunkOrders) + { + ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex])] == + LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); + ZEN_ASSERT(LocalLookup.ChunkHashToChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]) == ChunkIndex); + } + } + + std::vector AbsoluteChunkOrders = CalculateAbsoluteChunkOrders(LocalContent.ChunkedContent.ChunkHashes, + LocalContent.ChunkedContent.ChunkOrders, + LocalLookup.ChunkHashToChunkIndex, + LooseChunkIndexes, + Result.AllChunkBlockDescriptions); + + if (m_Options.DoExtraContentValidation) + { + for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); ChunkOrderIndex++) + { + uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndex]; + uint32_t AbsoluteChunkIndex = AbsoluteChunkOrders[ChunkOrderIndex]; + const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; + const IoHash& AbsoluteChunkHash = AbsoluteChunkHashes[AbsoluteChunkIndex]; + ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash); + } + } + + WriteBuildContentToCompactBinary(PartManifestWriter, + LocalContent.Platform, + LocalContent.Paths, + LocalContent.RawHashes, + LocalContent.RawSizes, + LocalContent.Attributes, + LocalContent.ChunkedContent.SequenceRawHashes, + LocalContent.ChunkedContent.ChunkCounts, + LocalContent.ChunkedContent.ChunkHashes, + LocalContent.ChunkedContent.ChunkRawSizes, + AbsoluteChunkOrders, + LooseChunkIndexes, + Result.AllChunkBlockHashes); + + if (m_Options.DoExtraContentValidation) + { + ChunkedFolderContent VerifyFolderContent; + + std::vector OutAbsoluteChunkOrders; + std::vector OutLooseChunkHashes; + std::vector OutLooseChunkRawSizes; + std::vector OutBlockRawHashes; + ReadBuildContentFromCompactBinary(PartManifestWriter.Save(), + VerifyFolderContent.Platform, + VerifyFolderContent.Paths, + VerifyFolderContent.RawHashes, + VerifyFolderContent.RawSizes, + VerifyFolderContent.Attributes, + VerifyFolderContent.ChunkedContent.SequenceRawHashes, + VerifyFolderContent.ChunkedContent.ChunkCounts, + OutAbsoluteChunkOrders, + OutLooseChunkHashes, + OutLooseChunkRawSizes, + OutBlockRawHashes); + ZEN_ASSERT(OutBlockRawHashes == Result.AllChunkBlockHashes); + + for (uint32_t OrderIndex = 0; OrderIndex < OutAbsoluteChunkOrders.size(); OrderIndex++) + { + uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex]; + const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; + + uint32_t VerifyChunkIndex = OutAbsoluteChunkOrders[OrderIndex]; + const IoHash VerifyChunkHash = AbsoluteChunkHashes[VerifyChunkIndex]; + + ZEN_ASSERT(LocalChunkHash == VerifyChunkHash); + } + + CalculateLocalChunkOrders(OutAbsoluteChunkOrders, + OutLooseChunkHashes, + OutLooseChunkRawSizes, + Result.AllChunkBlockDescriptions, + VerifyFolderContent.ChunkedContent.ChunkHashes, + VerifyFolderContent.ChunkedContent.ChunkRawSizes, + VerifyFolderContent.ChunkedContent.ChunkOrders, + m_Options.DoExtraContentValidation); + + ZEN_ASSERT(LocalContent.Paths == VerifyFolderContent.Paths); + ZEN_ASSERT(LocalContent.RawHashes == VerifyFolderContent.RawHashes); + ZEN_ASSERT(LocalContent.RawSizes == VerifyFolderContent.RawSizes); + ZEN_ASSERT(LocalContent.Attributes == VerifyFolderContent.Attributes); + ZEN_ASSERT(LocalContent.ChunkedContent.SequenceRawHashes == VerifyFolderContent.ChunkedContent.SequenceRawHashes); + ZEN_ASSERT(LocalContent.ChunkedContent.ChunkCounts == VerifyFolderContent.ChunkedContent.ChunkCounts); + + for (uint32_t OrderIndex = 0; OrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); OrderIndex++) + { + uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex]; + const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; + uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex]; + + uint32_t VerifyChunkIndex = VerifyFolderContent.ChunkedContent.ChunkOrders[OrderIndex]; + const IoHash VerifyChunkHash = VerifyFolderContent.ChunkedContent.ChunkHashes[VerifyChunkIndex]; + uint64_t VerifyChunkRawSize = VerifyFolderContent.ChunkedContent.ChunkRawSizes[VerifyChunkIndex]; + + ZEN_ASSERT(LocalChunkHash == VerifyChunkHash); + ZEN_ASSERT(LocalChunkRawSize == VerifyChunkRawSize); + } + } + + Result.PartManifest = PartManifestWriter.Save(); + return Result; +} + +void +BuildsOperationUploadFolder::UploadAttachmentBatch(std::span RawHashes, + std::vector& OutUnknownChunks, + const ChunkedFolderContent& LocalContent, + const ChunkedContentLookup& LocalLookup, + const std::vector>& NewBlockChunks, + GeneratedBlocks& NewBlocks, + std::span LooseChunkIndexes, + UploadStatistics& UploadStats, + LooseChunksStatistics& LooseChunksStats) +{ + if (m_AbortFlag) + { + return; + } + + UploadStatistics TempUploadStats; + LooseChunksStatistics TempLooseChunksStats; + + Stopwatch TempUploadTimer; + auto __ = MakeGuard([&]() { + if (!m_Options.IsQuiet) + { + uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs(); + ZEN_INFO( + "Uploaded {} ({}) blocks. " + "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. " + "Transferred {} ({}bits/s) in {}", + TempUploadStats.BlockCount.load(), + NiceBytes(TempUploadStats.BlocksBytes), + + TempLooseChunksStats.CompressedChunkCount.load(), + NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()), + NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, TempLooseChunksStats.ChunkByteCount)), + TempUploadStats.ChunkCount.load(), + NiceBytes(TempUploadStats.ChunksBytes), + + NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes), + NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)), + NiceTimeSpanMs(TempChunkUploadTimeUs / 1000)); + } + }); + UploadPartBlobs(LocalContent, + LocalLookup, + RawHashes, + NewBlockChunks, + NewBlocks, + LooseChunkIndexes, + m_LargeAttachmentSize, + TempUploadStats, + TempLooseChunksStats, + OutUnknownChunks); + UploadStats += TempUploadStats; + LooseChunksStats += TempLooseChunksStats; +} + +void +BuildsOperationUploadFolder::FinalizeBuildPartWithRetries(const UploadPart& Part, + const IoHash& PartHash, + std::vector& InOutUnknownChunks, + const ChunkedFolderContent& LocalContent, + const ChunkedContentLookup& LocalLookup, + const std::vector>& NewBlockChunks, + GeneratedBlocks& NewBlocks, + std::span LooseChunkIndexes, + UploadStatistics& UploadStats, + LooseChunksStatistics& LooseChunksStats) +{ + auto BuildUnkownChunksResponse = [](const std::vector& UnknownChunks, bool WillRetry) { + return fmt::format( + "The following build blobs was reported as needed for upload but was reported as existing at the start of the " + "operation.{}{}", + WillRetry ? " Treating this as a transient inconsistency issue and will attempt to retry finalization."sv : ""sv, + FormatArray(UnknownChunks, "\n "sv)); + }; + + if (!InOutUnknownChunks.empty()) + { + ZEN_WARN("{}", BuildUnkownChunksResponse(InOutUnknownChunks, /*WillRetry*/ true)); + } + + uint32_t FinalizeBuildPartRetryCount = 5; + while (!m_AbortFlag && (FinalizeBuildPartRetryCount--) > 0) + { + Stopwatch FinalizeBuildPartTimer; + std::vector Needs = m_Storage.BuildStorage->FinalizeBuildPart(m_BuildId, Part.PartId, PartHash); + if (!m_Options.IsQuiet) + { + ZEN_INFO("FinalizeBuildPart took {}. {} attachments are missing.", + NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()), + Needs.size()); + } + if (Needs.empty()) + { + break; + } + if (m_Options.IsVerbose) + { + ZEN_INFO("FinalizeBuildPart needs attachments: {}", FormatArray(Needs, "\n "sv)); + } + + std::vector RetryUnknownChunks; + UploadAttachmentBatch(Needs, + RetryUnknownChunks, + LocalContent, + LocalLookup, + NewBlockChunks, + NewBlocks, + LooseChunkIndexes, + UploadStats, + LooseChunksStats); + if (RetryUnknownChunks == InOutUnknownChunks) + { + if (FinalizeBuildPartRetryCount > 0) + { + // Back off a bit + Sleep(1000); + } + } + else + { + InOutUnknownChunks = RetryUnknownChunks; + ZEN_WARN("{}", BuildUnkownChunksResponse(InOutUnknownChunks, /*WillRetry*/ FinalizeBuildPartRetryCount != 0)); + } + } + + if (!InOutUnknownChunks.empty()) + { + throw std::runtime_error(BuildUnkownChunksResponse(InOutUnknownChunks, /*WillRetry*/ false)); + } +} + +void +BuildsOperationUploadFolder::UploadMissingBlockMetadata(GeneratedBlocks& NewBlocks, UploadStatistics& UploadStats) +{ + uint64_t UploadBlockMetadataCount = 0; + Stopwatch UploadBlockMetadataTimer; + + uint32_t FailedMetadataUploadCount = 1; + int32_t MetadataUploadRetryCount = 3; + while ((MetadataUploadRetryCount-- > 0) && (FailedMetadataUploadCount > 0)) + { + FailedMetadataUploadCount = 0; + for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++) + { + if (m_AbortFlag) + { + break; + } + const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; + if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex]) + { + const CbObject BlockMetaData = + BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); + if (m_Storage.CacheStorage && m_Options.PopulateCache) + { + m_Storage.CacheStorage->PutBlobMetadatas(m_BuildId, + std::vector({BlockHash}), + std::vector({BlockMetaData})); + } + bool MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData); + if (MetadataSucceeded) + { + UploadStats.BlocksBytes += BlockMetaData.GetSize(); + NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + UploadBlockMetadataCount++; + } + else + { + FailedMetadataUploadCount++; + } + } + } + } + if (UploadBlockMetadataCount > 0) + { + uint64_t ElapsedUS = UploadBlockMetadataTimer.GetElapsedTimeUs(); + UploadStats.ElapsedWallTimeUS += ElapsedUS; + if (!m_Options.IsQuiet) + { + ZEN_INFO("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000)); + } + } +} + +void +BuildsOperationUploadFolder::UploadPartBlobs(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + std::span RawHashes, + const std::vector>& NewBlockChunks, + GeneratedBlocks& NewBlocks, + std::span LooseChunkIndexes, + const std::uint64_t LargeAttachmentSize, + UploadStatistics& TempUploadStats, + LooseChunksStatistics& TempLooseChunksStats, + std::vector& OutUnknownChunks) +{ + ZEN_TRACE_CPU("UploadPartBlobs"); + + UploadPartClassification Classification = + ClassifyUploadRawHashes(RawHashes, Content, Lookup, NewBlocks, LooseChunkIndexes, OutUnknownChunks); + + if (Classification.BlockIndexes.empty() && Classification.LooseChunkOrderIndexes.empty()) + { + return; + } + + std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Upload Blobs"); + + FilteredRate FilteredGenerateBlockBytesPerSecond; + FilteredRate FilteredCompressedBytesPerSecond; + FilteredRate FilteredUploadedBytesPerSecond; + + ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + + std::atomic UploadedBlockSize = 0; + std::atomic UploadedBlockCount = 0; + std::atomic UploadedRawChunkSize = 0; + std::atomic UploadedCompressedChunkSize = 0; + std::atomic UploadedChunkCount = 0; + std::atomic GeneratedBlockCount = 0; + std::atomic GeneratedBlockByteCount = 0; + std::atomic QueuedPendingInMemoryBlocksForUpload = 0; + + const size_t UploadBlockCount = Classification.BlockIndexes.size(); + const uint32_t UploadChunkCount = gsl::narrow(Classification.LooseChunkOrderIndexes.size()); + const uint64_t TotalRawSize = Classification.TotalLooseChunksSize + Classification.TotalBlocksSize; + + UploadPartBlobsContext Context{.Work = Work, + .ReadChunkPool = m_IOWorkerPool, + .UploadChunkPool = m_NetworkPool, + .FilteredGenerateBlockBytesPerSecond = FilteredGenerateBlockBytesPerSecond, + .FilteredCompressedBytesPerSecond = FilteredCompressedBytesPerSecond, + .FilteredUploadedBytesPerSecond = FilteredUploadedBytesPerSecond, + .UploadedBlockSize = UploadedBlockSize, + .UploadedBlockCount = UploadedBlockCount, + .UploadedRawChunkSize = UploadedRawChunkSize, + .UploadedCompressedChunkSize = UploadedCompressedChunkSize, + .UploadedChunkCount = UploadedChunkCount, + .GeneratedBlockCount = GeneratedBlockCount, + .GeneratedBlockByteCount = GeneratedBlockByteCount, + .QueuedPendingInMemoryBlocksForUpload = QueuedPendingInMemoryBlocksForUpload, + .UploadBlockCount = UploadBlockCount, + .UploadChunkCount = UploadChunkCount, + .LargeAttachmentSize = LargeAttachmentSize, + .NewBlocks = NewBlocks, + .Content = Content, + .Lookup = Lookup, + .NewBlockChunks = NewBlockChunks, + .LooseChunkIndexes = LooseChunkIndexes, + .TempUploadStats = TempUploadStats, + .TempLooseChunksStats = TempLooseChunksStats}; + + ScheduleBlockGenerationAndUpload(Context, Classification.BlockIndexes); + ScheduleLooseChunkCompressionAndUpload(Context, Classification.LooseChunkOrderIndexes); + + Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); + FilteredCompressedBytesPerSecond.Update(TempLooseChunksStats.CompressedChunkRawBytes.load()); + FilteredGenerateBlockBytesPerSecond.Update(GeneratedBlockByteCount.load()); + FilteredUploadedBytesPerSecond.Update(UploadedCompressedChunkSize.load() + UploadedBlockSize.load()); + uint64_t UploadedRawSize = UploadedRawChunkSize.load() + UploadedBlockSize.load(); + uint64_t UploadedCompressedSize = UploadedCompressedChunkSize.load() + UploadedBlockSize.load(); + + std::string Details = fmt::format( + "Compressed {}/{} ({}/{}{}) chunks. " + "Uploaded {}/{} ({}/{}) blobs " + "({}{})", + TempLooseChunksStats.CompressedChunkCount.load(), + Classification.LooseChunkOrderIndexes.size(), + NiceBytes(TempLooseChunksStats.CompressedChunkRawBytes), + NiceBytes(Classification.TotalLooseChunksSize), + (TempLooseChunksStats.CompressedChunkCount == Classification.LooseChunkOrderIndexes.size()) + ? "" + : fmt::format(" {}B/s", NiceNum(FilteredCompressedBytesPerSecond.GetCurrent())), + + UploadedBlockCount.load() + UploadedChunkCount.load(), + UploadBlockCount + UploadChunkCount, + NiceBytes(UploadedRawSize), + NiceBytes(TotalRawSize), + + NiceBytes(UploadedCompressedSize), + (UploadedBlockCount == UploadBlockCount && UploadedChunkCount == UploadChunkCount) + ? "" + : fmt::format(" {}bits/s", NiceNum(FilteredUploadedBytesPerSecond.GetCurrent()))); + + ProgressBar->UpdateState({.Task = "Uploading blobs ", + .Details = Details, + .TotalCount = gsl::narrow(TotalRawSize), + .RemainingCount = gsl::narrow(TotalRawSize - UploadedRawSize), + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); + }); + + ZEN_ASSERT(m_AbortFlag || QueuedPendingInMemoryBlocksForUpload.load() == 0); + + ProgressBar->Finish(); + + TempUploadStats.ElapsedWallTimeUS += FilteredUploadedBytesPerSecond.GetElapsedTimeUS(); + TempLooseChunksStats.CompressChunksElapsedWallTimeUS += FilteredCompressedBytesPerSecond.GetElapsedTimeUS(); +} + +BuildsOperationUploadFolder::UploadPartClassification +BuildsOperationUploadFolder::ClassifyUploadRawHashes(std::span RawHashes, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const GeneratedBlocks& NewBlocks, + std::span LooseChunkIndexes, + std::vector& OutUnknownChunks) +{ + UploadPartClassification Result; + + tsl::robin_map ChunkIndexToLooseChunkOrderIndex; + ChunkIndexToLooseChunkOrderIndex.reserve(LooseChunkIndexes.size()); + for (uint32_t OrderIndex = 0; OrderIndex < LooseChunkIndexes.size(); OrderIndex++) + { + ChunkIndexToLooseChunkOrderIndex.insert_or_assign(LooseChunkIndexes[OrderIndex], OrderIndex); + } + + for (const IoHash& RawHash : RawHashes) + { + if (auto It = NewBlocks.BlockHashToBlockIndex.find(RawHash); It != NewBlocks.BlockHashToBlockIndex.end()) + { + Result.BlockIndexes.push_back(It->second); + Result.TotalBlocksSize += NewBlocks.BlockSizes[It->second]; + } + else if (auto ChunkIndexIt = Lookup.ChunkHashToChunkIndex.find(RawHash); ChunkIndexIt != Lookup.ChunkHashToChunkIndex.end()) + { + const uint32_t ChunkIndex = ChunkIndexIt->second; + if (auto LooseOrderIndexIt = ChunkIndexToLooseChunkOrderIndex.find(ChunkIndex); + LooseOrderIndexIt != ChunkIndexToLooseChunkOrderIndex.end()) + { + Result.LooseChunkOrderIndexes.push_back(LooseOrderIndexIt->second); + Result.TotalLooseChunksSize += Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + } + } + else + { + OutUnknownChunks.push_back(RawHash); + } + } + return Result; +} + +void +BuildsOperationUploadFolder::ScheduleBlockGenerationAndUpload(UploadPartBlobsContext& Context, std::span BlockIndexes) +{ + for (const size_t BlockIndex : BlockIndexes) + { + const IoHash& BlockHash = Context.NewBlocks.BlockDescriptions[BlockIndex].BlockHash; + if (m_AbortFlag) + { + break; + } + Context.Work.ScheduleWork( + Context.ReadChunkPool, + [this, &Context, BlockHash = IoHash(BlockHash), BlockIndex, GenerateBlockCount = BlockIndexes.size()](std::atomic&) { + if (m_AbortFlag) + { + return; + } + ZEN_TRACE_CPU("UploadPartBlobs_GenerateBlock"); + + Context.FilteredGenerateBlockBytesPerSecond.Start(); + + Stopwatch GenerateTimer; + CompositeBuffer Payload; + if (Context.NewBlocks.BlockHeaders[BlockIndex]) + { + Payload = RebuildBlock(Context.Content, + Context.Lookup, + std::move(Context.NewBlocks.BlockHeaders[BlockIndex]), + Context.NewBlockChunks[BlockIndex]) + .GetCompressed(); + } + else + { + ChunkBlockDescription BlockDescription; + CompressedBuffer CompressedBlock = + GenerateBlock(Context.Content, Context.Lookup, Context.NewBlockChunks[BlockIndex], BlockDescription); + if (!CompressedBlock) + { + throw std::runtime_error(fmt::format("Failed generating block {}", BlockHash)); + } + ZEN_ASSERT(BlockDescription.BlockHash == BlockHash); + Payload = std::move(CompressedBlock).GetCompressed(); + } + + Context.GeneratedBlockByteCount += Context.NewBlocks.BlockSizes[BlockIndex]; + if (Context.GeneratedBlockCount.fetch_add(1) + 1 == GenerateBlockCount) + { + Context.FilteredGenerateBlockBytesPerSecond.Stop(); + } + if (m_Options.IsVerbose) + { + ZEN_INFO("{} block {} ({}) containing {} chunks in {}", + Context.NewBlocks.BlockHeaders[BlockIndex] ? "Regenerated" : "Generated", + Context.NewBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(Context.NewBlocks.BlockSizes[BlockIndex]), + Context.NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(), + NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs())); + } + if (!m_AbortFlag) + { + UploadBlockPayload(Context, BlockIndex, BlockHash, std::move(Payload)); + } + }); + } +} + +void +BuildsOperationUploadFolder::UploadBlockPayload(UploadPartBlobsContext& Context, + size_t BlockIndex, + const IoHash& BlockHash, + CompositeBuffer Payload) +{ + bool IsInMemoryBlock = true; + if (Context.QueuedPendingInMemoryBlocksForUpload.load() > 16) + { + ZEN_TRACE_CPU("AsyncUploadBlock_WriteTempBlock"); + std::filesystem::path TempFilePath = m_Options.TempDir / (BlockHash.ToHexString()); + Payload = CompositeBuffer(WriteToTempFile(std::move(Payload), TempFilePath)); + IsInMemoryBlock = false; + } + else + { + Context.QueuedPendingInMemoryBlocksForUpload++; + } + + Context.Work.ScheduleWork( + Context.UploadChunkPool, + [this, &Context, IsInMemoryBlock, BlockIndex, BlockHash = IoHash(BlockHash), Payload = CompositeBuffer(std::move(Payload))]( + std::atomic&) { + auto _ = MakeGuard([IsInMemoryBlock, &Context] { + if (IsInMemoryBlock) + { + Context.QueuedPendingInMemoryBlocksForUpload--; + } + }); + if (m_AbortFlag) + { + return; + } + ZEN_TRACE_CPU("AsyncUploadBlock"); + + const uint64_t PayloadSize = Payload.GetSize(); + + Context.FilteredUploadedBytesPerSecond.Start(); + const CbObject BlockMetaData = + BuildChunkBlockDescription(Context.NewBlocks.BlockDescriptions[BlockIndex], Context.NewBlocks.BlockMetaDatas[BlockIndex]); + + if (m_Storage.CacheStorage && m_Options.PopulateCache) + { + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); + } + + try + { + m_Storage.BuildStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); + } + catch (const std::exception&) + { + // Silence http errors due to abort + if (!m_AbortFlag) + { + throw; + } + } + + if (m_AbortFlag) + { + return; + } + if (m_Options.IsVerbose) + { + ZEN_INFO("Uploaded block {} ({}) containing {} chunks", + BlockHash, + NiceBytes(PayloadSize), + Context.NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); + } + Context.UploadedBlockSize += PayloadSize; + Context.TempUploadStats.BlocksBytes += PayloadSize; + + if (m_Storage.CacheStorage && m_Options.PopulateCache) + { + m_Storage.CacheStorage->PutBlobMetadatas(m_BuildId, + std::vector({BlockHash}), + std::vector({BlockMetaData})); + } + + bool MetadataSucceeded = false; + try + { + MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData); + } + catch (const std::exception&) + { + // Silence http errors due to abort + if (!m_AbortFlag) + { + throw; + } + } + if (m_AbortFlag) + { + return; + } + if (MetadataSucceeded) + { + if (m_Options.IsVerbose) + { + ZEN_INFO("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize())); + } + Context.NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + Context.TempUploadStats.BlocksBytes += BlockMetaData.GetSize(); + } + + Context.TempUploadStats.BlockCount++; + + if (Context.UploadedBlockCount.fetch_add(1) + 1 == Context.UploadBlockCount && + Context.UploadedChunkCount == Context.UploadChunkCount) + { + Context.FilteredUploadedBytesPerSecond.Stop(); + } + }); +} + +void +BuildsOperationUploadFolder::ScheduleLooseChunkCompressionAndUpload(UploadPartBlobsContext& Context, + std::span LooseChunkOrderIndexes) +{ + for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes) + { + const uint32_t ChunkIndex = Context.LooseChunkIndexes[LooseChunkOrderIndex]; + Context.Work.ScheduleWork(Context.ReadChunkPool, + [this, &Context, LooseChunkOrderCount = LooseChunkOrderIndexes.size(), ChunkIndex](std::atomic&) { + if (m_AbortFlag) + { + return; + } + ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk"); + + Context.FilteredCompressedBytesPerSecond.Start(); + Stopwatch CompressTimer; + CompositeBuffer Payload = + CompressChunk(Context.Content, Context.Lookup, ChunkIndex, Context.TempLooseChunksStats); + if (m_Options.IsVerbose) + { + ZEN_INFO("Compressed chunk {} ({} -> {}) in {}", + Context.Content.ChunkedContent.ChunkHashes[ChunkIndex], + NiceBytes(Context.Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), + NiceBytes(Payload.GetSize()), + NiceTimeSpanMs(CompressTimer.GetElapsedTimeMs())); + } + const uint64_t ChunkRawSize = Context.Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + Context.TempUploadStats.ReadFromDiskBytes += ChunkRawSize; + if (Context.TempLooseChunksStats.CompressedChunkCount == LooseChunkOrderCount) + { + Context.FilteredCompressedBytesPerSecond.Stop(); + } + if (!m_AbortFlag) + { + UploadLooseChunkPayload(Context, + Context.Content.ChunkedContent.ChunkHashes[ChunkIndex], + ChunkRawSize, + std::move(Payload)); + } + }); + } +} + +void +BuildsOperationUploadFolder::UploadLooseChunkPayload(UploadPartBlobsContext& Context, + const IoHash& RawHash, + uint64_t RawSize, + CompositeBuffer Payload) +{ + Context.Work.ScheduleWork( + Context.UploadChunkPool, + [this, &Context, RawHash = IoHash(RawHash), RawSize, Payload = CompositeBuffer(std::move(Payload))](std::atomic&) mutable { + if (m_AbortFlag) + { + return; + } + ZEN_TRACE_CPU("AsyncUploadLooseChunk"); + + const uint64_t PayloadSize = Payload.GetSize(); + + if (m_Storage.CacheStorage && m_Options.PopulateCache) + { + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); + } + + if (PayloadSize >= Context.LargeAttachmentSize) + { + ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart"); + Context.TempUploadStats.MultipartAttachmentCount++; + try + { + std::vector> MultipartWork = m_Storage.BuildStorage->PutLargeBuildBlob( + m_BuildId, + RawHash, + ZenContentType::kCompressedBinary, + PayloadSize, + [Payload = std::move(Payload), &Context](uint64_t Offset, uint64_t Size) -> IoBuffer { + Context.FilteredUploadedBytesPerSecond.Start(); + + IoBuffer PartPayload = Payload.Mid(Offset, Size).Flatten().AsIoBuffer(); + PartPayload.SetContentType(ZenContentType::kBinary); + return PartPayload; + }, + [&Context, RawSize](uint64_t SentBytes, bool IsComplete) { + Context.TempUploadStats.ChunksBytes += SentBytes; + Context.UploadedCompressedChunkSize += SentBytes; + if (IsComplete) + { + Context.TempUploadStats.ChunkCount++; + if (Context.UploadedChunkCount.fetch_add(1) + 1 == Context.UploadChunkCount && + Context.UploadedBlockCount == Context.UploadBlockCount) + { + Context.FilteredUploadedBytesPerSecond.Stop(); + } + Context.UploadedRawChunkSize += RawSize; + } + }); + for (auto& WorkPart : MultipartWork) + { + Context.Work.ScheduleWork(Context.UploadChunkPool, [Work = std::move(WorkPart)](std::atomic& AbortFlag) { + ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart_Work"); + if (!AbortFlag) + { + Work(); + } + }); + } + if (m_Options.IsVerbose) + { + ZEN_INFO("Uploaded multipart chunk {} ({})", RawHash, NiceBytes(PayloadSize)); + } + } + catch (const std::exception&) + { + // Silence http errors due to abort + if (!m_AbortFlag) + { + throw; + } + } + return; + } + + ZEN_TRACE_CPU("AsyncUploadLooseChunk_Singlepart"); + try + { + m_Storage.BuildStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); + } + catch (const std::exception&) + { + // Silence http errors due to abort + if (!m_AbortFlag) + { + throw; + } + } + if (m_AbortFlag) + { + return; + } + if (m_Options.IsVerbose) + { + ZEN_INFO("Uploaded chunk {} ({})", RawHash, NiceBytes(PayloadSize)); + } + Context.TempUploadStats.ChunksBytes += Payload.GetSize(); + Context.TempUploadStats.ChunkCount++; + Context.UploadedCompressedChunkSize += Payload.GetSize(); + Context.UploadedRawChunkSize += RawSize; + if (Context.UploadedChunkCount.fetch_add(1) + 1 == Context.UploadChunkCount && + Context.UploadedBlockCount == Context.UploadBlockCount) + { + Context.FilteredUploadedBytesPerSecond.Stop(); + } + }); +} + +CompositeBuffer +BuildsOperationUploadFolder::CompressChunk(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + uint32_t ChunkIndex, + LooseChunksStatistics& TempLooseChunksStats) +{ + ZEN_TRACE_CPU("CompressChunk"); + ZEN_ASSERT(!m_Options.TempDir.empty()); + const IoHash& ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; + const uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + + const ChunkedContentLookup::ChunkSequenceLocation& Source = GetChunkSequenceLocations(Lookup, ChunkIndex)[0]; + const std::uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[Source.SequenceIndex]; + IoBuffer RawSource = IoBufferBuilder::MakeFromFile((m_Path / Content.Paths[PathIndex]).make_preferred(), Source.Offset, ChunkSize); + if (!RawSource) + { + throw std::runtime_error(fmt::format("Failed fetching chunk {}", ChunkHash)); + } + if (RawSource.GetSize() != ChunkSize) + { + throw std::runtime_error(fmt::format("Fetched chunk {} has invalid size", ChunkHash)); + } + + const bool ShouldCompressChunk = IsChunkCompressable(m_NonCompressableExtensionHashes, Lookup, ChunkIndex); + const OodleCompressionLevel CompressionLevel = ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None; + + if (ShouldCompressChunk) + { + std::filesystem::path TempFilePath = m_Options.TempDir / ChunkHash.ToHexString(); + + BasicFile CompressedFile; + std::error_code Ec; + CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncateDelete, Ec); + if (Ec) + { + throw std::runtime_error(fmt::format("Failed creating temporary file for compressing blob {}, reason: ({}) {}", + ChunkHash, + Ec.value(), + Ec.message())); + } + + uint64_t StreamRawBytes = 0; + uint64_t StreamCompressedBytes = 0; + + bool CouldCompress = CompressedBuffer::CompressToStream( + CompositeBuffer(SharedBuffer(RawSource)), + [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { + ZEN_UNUSED(SourceOffset); + TempLooseChunksStats.CompressedChunkRawBytes += SourceSize; + CompressedFile.Write(RangeBuffer, Offset); + TempLooseChunksStats.CompressedChunkBytes += RangeBuffer.GetSize(); + StreamRawBytes += SourceSize; + StreamCompressedBytes += RangeBuffer.GetSize(); + }, + OodleCompressor::Mermaid, + CompressionLevel); + if (CouldCompress) + { + uint64_t CompressedSize = CompressedFile.FileSize(); + void* FileHandle = CompressedFile.Detach(); + IoBuffer TempPayload = IoBuffer(IoBuffer::File, + FileHandle, + 0, + CompressedSize, + /*IsWholeFile*/ true); + ZEN_ASSERT(TempPayload); + TempPayload.SetDeleteOnClose(true); + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(TempPayload), RawHash, RawSize); + ZEN_ASSERT(Compressed); + ZEN_ASSERT(RawHash == ChunkHash); + ZEN_ASSERT(RawSize == ChunkSize); + + TempLooseChunksStats.CompressedChunkCount++; + + return Compressed.GetCompressed(); + } + else + { + TempLooseChunksStats.CompressedChunkRawBytes -= StreamRawBytes; + TempLooseChunksStats.CompressedChunkBytes -= StreamCompressedBytes; + } + CompressedFile.Close(); + RemoveFile(TempFilePath, Ec); + ZEN_UNUSED(Ec); + } + + CompressedBuffer CompressedBlob = + CompressedBuffer::Compress(SharedBuffer(std::move(RawSource)), OodleCompressor::Mermaid, CompressionLevel); + if (!CompressedBlob) + { + throw std::runtime_error(fmt::format("Failed to compress large blob {}", ChunkHash)); + } + ZEN_ASSERT_SLOW(CompressedBlob.DecodeRawHash() == ChunkHash); + ZEN_ASSERT_SLOW(CompressedBlob.DecodeRawSize() == ChunkSize); + + TempLooseChunksStats.CompressedChunkRawBytes += ChunkSize; + TempLooseChunksStats.CompressedChunkBytes += CompressedBlob.GetCompressedSize(); + + // If we use none-compression, the compressed blob references the data and has 64 kb in memory so we don't need to write it to disk + if (ShouldCompressChunk) + { + std::filesystem::path TempFilePath = m_Options.TempDir / (ChunkHash.ToHexString()); + IoBuffer TempPayload = WriteToTempFile(std::move(CompressedBlob).GetCompressed(), TempFilePath); + CompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(TempPayload)); + } + + TempLooseChunksStats.CompressedChunkCount++; + return std::move(CompressedBlob).GetCompressed(); +} + +std::vector> +UploadFolder(LoggerRef Log, + ProgressBase& Progress, + TransferThreadWorkers& Workers, + StorageInstance& Storage, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + const Oid& BuildId, + const Oid& BuildPartId, + std::string_view BuildPartName, + const std::filesystem::path& Path, + const std::filesystem::path& ManifestPath, + const CbObject& MetaData, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + const UploadFolderOptions& Options) +{ + Progress.SetLogOperationName("Upload Folder"); + + Stopwatch UploadTimer; + + BuildsOperationUploadFolder UploadOp( + Log, + Progress, + Storage, + AbortFlag, + PauseFlag, + Workers.GetIOWorkerPool(), + Workers.GetNetworkPool(), + BuildId, + Path, + Options.CreateBuild, + std::move(MetaData), + BuildsOperationUploadFolder::Options{.IsQuiet = Options.IsQuiet, + .IsVerbose = Options.IsVerbose, + .DoExtraContentValidation = Options.DoExtraContentVerify, + .FindBlockMaxCount = Options.FindBlockMaxCount, + .BlockReuseMinPercentLimit = Options.BlockReuseMinPercentLimit, + .AllowMultiparts = Options.AllowMultiparts, + .IgnoreExistingBlocks = Options.IgnoreExistingBlocks, + .TempDir = Options.TempDir, + .ExcludeFolders = Options.ExcludeFolders, + .ExcludeExtensions = Options.ExcludeExtensions, + .NonCompressableExtensions = DefaultSplitOnlyExtensions, + .PopulateCache = Options.UploadToZenCache}); + + std::vector> UploadedParts = + UploadOp.Execute(BuildPartId, BuildPartName, ManifestPath, ChunkController, ChunkCache); + if (AbortFlag) + { + return {}; + } + + if (Options.IsVerbose) + { + ZEN_CONSOLE( + "Folder scanning stats:" + "\n FoundFileCount: {}" + "\n FoundFileByteCount: {}" + "\n AcceptedFileCount: {}" + "\n AcceptedFileByteCount: {}" + "\n ElapsedWallTimeUS: {}", + UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), + NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), + UploadOp.m_LocalFolderScanStats.AcceptedFileCount.load(), + NiceBytes(UploadOp.m_LocalFolderScanStats.AcceptedFileByteCount.load()), + NiceLatencyNs(UploadOp.m_LocalFolderScanStats.ElapsedWallTimeUS * 1000)); + + ZEN_CONSOLE( + "Chunking stats:" + "\n FilesProcessed: {}" + "\n FilesChunked: {}" + "\n BytesHashed: {}" + "\n UniqueChunksFound: {}" + "\n UniqueSequencesFound: {}" + "\n UniqueBytesFound: {}" + "\n FilesFoundInCache: {}" + "\n ChunksFoundInCache: {}" + "\n FilesStoredInCache: {}" + "\n ChunksStoredInCache: {}" + "\n ElapsedWallTimeUS: {}", + UploadOp.m_ChunkingStats.FilesProcessed.load(), + UploadOp.m_ChunkingStats.FilesChunked.load(), + NiceBytes(UploadOp.m_ChunkingStats.BytesHashed.load()), + UploadOp.m_ChunkingStats.UniqueChunksFound.load(), + UploadOp.m_ChunkingStats.UniqueSequencesFound.load(), + NiceBytes(UploadOp.m_ChunkingStats.UniqueBytesFound.load()), + UploadOp.m_ChunkingStats.FilesFoundInCache.load(), + UploadOp.m_ChunkingStats.ChunksFoundInCache.load(), + NiceBytes(UploadOp.m_ChunkingStats.BytesFoundInCache.load()), + UploadOp.m_ChunkingStats.FilesStoredInCache.load(), + UploadOp.m_ChunkingStats.ChunksStoredInCache.load(), + NiceBytes(UploadOp.m_ChunkingStats.BytesStoredInCache.load()), + NiceLatencyNs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS * 1000)); + + ZEN_CONSOLE( + "Find block stats:" + "\n FindBlockTimeMS: {}" + "\n PotentialChunkCount: {}" + "\n PotentialChunkByteCount: {}" + "\n FoundBlockCount: {}" + "\n FoundBlockChunkCount: {}" + "\n FoundBlockByteCount: {}" + "\n AcceptedBlockCount: {}" + "\n NewBlocksCount: {}" + "\n NewBlocksChunkCount: {}" + "\n NewBlocksChunkByteCount: {}", + NiceTimeSpanMs(UploadOp.m_FindBlocksStats.FindBlockTimeMS), + UploadOp.m_FindBlocksStats.PotentialChunkCount, + NiceBytes(UploadOp.m_FindBlocksStats.PotentialChunkByteCount), + UploadOp.m_FindBlocksStats.FoundBlockCount, + UploadOp.m_FindBlocksStats.FoundBlockChunkCount, + NiceBytes(UploadOp.m_FindBlocksStats.FoundBlockByteCount), + UploadOp.m_FindBlocksStats.AcceptedBlockCount, + UploadOp.m_FindBlocksStats.NewBlocksCount, + UploadOp.m_FindBlocksStats.NewBlocksChunkCount, + NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount)); + + ZEN_CONSOLE( + "Reuse block stats:" + "\n AcceptedChunkCount: {}" + "\n AcceptedByteCount: {}" + "\n AcceptedRawByteCount: {}" + "\n RejectedBlockCount: {}" + "\n RejectedChunkCount: {}" + "\n RejectedByteCount: {}" + "\n AcceptedReduntantChunkCount: {}" + "\n AcceptedReduntantByteCount: {}", + UploadOp.m_ReuseBlocksStats.AcceptedChunkCount, + NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedByteCount), + NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedRawByteCount), + UploadOp.m_ReuseBlocksStats.RejectedBlockCount, + UploadOp.m_ReuseBlocksStats.RejectedChunkCount, + NiceBytes(UploadOp.m_ReuseBlocksStats.RejectedByteCount), + UploadOp.m_ReuseBlocksStats.AcceptedReduntantChunkCount, + NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedReduntantByteCount)); + + ZEN_CONSOLE( + "Generate blocks stats:" + "\n GeneratedBlockByteCount: {}" + "\n GeneratedBlockCount: {}" + "\n GenerateBlocksElapsedWallTimeUS: {}", + NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), + UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), + NiceLatencyNs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000)); + + ZEN_CONSOLE( + "Loose chunks stats:" + "\n ChunkCount: {}" + "\n ChunkByteCount: {}" + "\n CompressedChunkCount: {}" + "\n CompressChunksElapsedWallTimeUS: {}", + UploadOp.m_LooseChunksStats.ChunkCount, + NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount), + UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), + NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), + NiceLatencyNs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000)); + + ZEN_CONSOLE( + "Disk stats:" + "\n OpenReadCount: {}" + "\n OpenWriteCount: {}" + "\n ReadCount: {}" + "\n ReadByteCount: {}" + "\n WriteCount: {} ({} cloned)" + "\n WriteByteCount: {} ({} cloned)" + "\n CurrentOpenFileCount: {}", + UploadOp.m_DiskStats.OpenReadCount.load(), + UploadOp.m_DiskStats.OpenWriteCount.load(), + UploadOp.m_DiskStats.ReadCount.load(), + NiceBytes(UploadOp.m_DiskStats.ReadByteCount.load()), + UploadOp.m_DiskStats.WriteCount.load(), + UploadOp.m_DiskStats.CloneCount.load(), + NiceBytes(UploadOp.m_DiskStats.WriteByteCount.load()), + NiceBytes(UploadOp.m_DiskStats.CloneByteCount.load()), + UploadOp.m_DiskStats.CurrentOpenFileCount.load()); + + ZEN_CONSOLE( + "Upload stats:" + "\n BlockCount: {}" + "\n BlocksBytes: {}" + "\n ChunkCount: {}" + "\n ChunksBytes: {}" + "\n ReadFromDiskBytes: {}" + "\n MultipartAttachmentCount: {}" + "\n ElapsedWallTimeUS: {}", + UploadOp.m_UploadStats.BlockCount.load(), + NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), + UploadOp.m_UploadStats.ChunkCount.load(), + NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), + NiceBytes(UploadOp.m_UploadStats.ReadFromDiskBytes.load()), + UploadOp.m_UploadStats.MultipartAttachmentCount.load(), + NiceLatencyNs(UploadOp.m_UploadStats.ElapsedWallTimeUS * 1000)); + } + + const double DeltaByteCountPercent = + UploadOp.m_ChunkingStats.BytesHashed > 0 + ? (100.0 * (UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes)) / + (UploadOp.m_ChunkingStats.BytesHashed) + : 0.0; + + const std::string MultipartAttachmentStats = + Options.AllowMultiparts ? fmt::format(" ({} as multipart)", UploadOp.m_UploadStats.MultipartAttachmentCount.load()) : ""; + + if (!Options.IsQuiet) + { + ZEN_CONSOLE( + "Uploaded part {} ('{}') to build {}, {}\n" + " Scanned files: {:>8} ({}), {}B/sec, {}\n" + " New data: {:>8} ({}) {:.1f}%\n" + " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n" + " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n" + " Uploaded: {:>8} ({}), {}bits/sec, {}\n" + " Blocks: {:>8} ({})\n" + " Chunks: {:>8} ({}){}", + BuildPartId, + BuildPartName, + BuildId, + NiceTimeSpanMs(UploadTimer.GetElapsedTimeMs()), + + UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), + NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), + NiceNum(GetBytesPerSecond(UploadOp.m_ChunkingStats.ElapsedWallTimeUS, UploadOp.m_ChunkingStats.BytesHashed)), + NiceTimeSpanMs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS / 1000), + + UploadOp.m_FindBlocksStats.NewBlocksChunkCount + UploadOp.m_LooseChunksStats.CompressedChunkCount, + NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes), + DeltaByteCountPercent, + + UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), + NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount), + NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), + NiceNum(GetBytesPerSecond(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, + UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount)), + NiceTimeSpanMs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000), + + UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), + NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkRawBytes), + NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), + NiceNum(GetBytesPerSecond(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS, + UploadOp.m_LooseChunksStats.CompressedChunkRawBytes)), + NiceTimeSpanMs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000), + + UploadOp.m_UploadStats.BlockCount.load() + UploadOp.m_UploadStats.ChunkCount.load(), + NiceBytes(UploadOp.m_UploadStats.BlocksBytes + UploadOp.m_UploadStats.ChunksBytes), + NiceNum(GetBytesPerSecond(UploadOp.m_UploadStats.ElapsedWallTimeUS, + (UploadOp.m_UploadStats.ChunksBytes + UploadOp.m_UploadStats.BlocksBytes) * 8)), + NiceTimeSpanMs(UploadOp.m_UploadStats.ElapsedWallTimeUS / 1000), + + UploadOp.m_UploadStats.BlockCount.load(), + NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), + + UploadOp.m_UploadStats.ChunkCount.load(), + NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), + MultipartAttachmentStats); + } + return UploadedParts; +} + +} // namespace zen diff --git a/src/zenremotestore/builds/buildvalidatebuildpart.cpp b/src/zenremotestore/builds/buildvalidatebuildpart.cpp new file mode 100644 index 000000000..a1334e1ef --- /dev/null +++ b/src/zenremotestore/builds/buildvalidatebuildpart.cpp @@ -0,0 +1,371 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace zen { + +using namespace std::literals; + +BuildsOperationValidateBuildPart::BuildsOperationValidateBuildPart(LoggerRef Log, + ProgressBase& Progress, + BuildStorageBase& Storage, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + WorkerThreadPool& IOWorkerPool, + WorkerThreadPool& NetworkPool, + const Oid& BuildId, + const Oid& BuildPartId, + const std::string_view BuildPartName, + const Options& Options) + +: m_Log(Log) +, m_Progress(Progress) +, m_Storage(Storage) +, m_AbortFlag(AbortFlag) +, m_PauseFlag(PauseFlag) +, m_IOWorkerPool(IOWorkerPool) +, m_NetworkPool(NetworkPool) +, m_BuildId(BuildId) +, m_BuildPartId(BuildPartId) +, m_BuildPartName(BuildPartName) +, m_Options(Options) +{ +} + +void +BuildsOperationValidateBuildPart::Execute() +{ + ZEN_TRACE_CPU("ValidateBuildPart"); + try + { + auto EndProgress = + MakeGuard([&]() { m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::StepCount, (uint32_t)TaskSteps::StepCount); }); + + Stopwatch Timer; + auto _ = MakeGuard([&]() { + if (!m_Options.IsQuiet) + { + ZEN_INFO("Validated build part {}/{} ('{}') in {}", + m_BuildId, + m_BuildPartId, + m_BuildPartName, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } + }); + + m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::FetchBuild, (uint32_t)TaskSteps::StepCount); + + ResolvedBuildPart Resolved = ResolveBuildPart(); + + ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + + const std::filesystem::path TempFolder = LegacyZenTempFolderName; + + CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, TempFolder); + CreateDirectories(TempFolder); + auto __ = MakeGuard([this, TempFolder]() { CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, TempFolder); }); + + m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::ValidateBlobs, (uint32_t)TaskSteps::StepCount); + + std::unique_ptr ProgressBar = m_Progress.CreateProgressBar("Validate Blobs"); + + const uint64_t AttachmentsToVerifyCount = Resolved.ChunkAttachments.size() + Resolved.BlockAttachments.size(); + FilteredRate FilteredDownloadedBytesPerSecond; + FilteredRate FilteredVerifiedBytesPerSecond; + + ValidateBlobsContext Context{.Work = Work, + .AttachmentsToVerifyCount = AttachmentsToVerifyCount, + .FilteredDownloadedBytesPerSecond = FilteredDownloadedBytesPerSecond, + .FilteredVerifiedBytesPerSecond = FilteredVerifiedBytesPerSecond}; + + ScheduleChunkAttachmentValidation(Context, Resolved.ChunkAttachments, TempFolder, Resolved.PreferredMultipartChunkSize); + ScheduleBlockAttachmentValidation(Context, Resolved.BlockAttachments); + + Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); + + const uint64_t DownloadedAttachmentCount = m_DownloadStats.DownloadedChunkCount + m_DownloadStats.DownloadedBlockCount; + const uint64_t DownloadedByteCount = m_DownloadStats.DownloadedChunkByteCount + m_DownloadStats.DownloadedBlockByteCount; + + FilteredDownloadedBytesPerSecond.Update(DownloadedByteCount); + FilteredVerifiedBytesPerSecond.Update(m_ValidateStats.VerifiedByteCount); + + std::string Details = fmt::format("Downloaded {}/{} ({}, {}bits/s). Verified {}/{} ({}, {}B/s)", + DownloadedAttachmentCount, + AttachmentsToVerifyCount, + NiceBytes(DownloadedByteCount), + NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8), + m_ValidateStats.VerifiedAttachmentCount.load(), + AttachmentsToVerifyCount, + NiceBytes(m_ValidateStats.VerifiedByteCount.load()), + NiceNum(FilteredVerifiedBytesPerSecond.GetCurrent())); + + ProgressBar->UpdateState( + {.Task = "Validating blobs ", + .Details = Details, + .TotalCount = gsl::narrow(AttachmentsToVerifyCount * 2), + .RemainingCount = gsl::narrow(AttachmentsToVerifyCount * 2 - + (DownloadedAttachmentCount + m_ValidateStats.VerifiedAttachmentCount.load())), + .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); + }); + + ProgressBar->Finish(); + m_ValidateStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); + + m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::Cleanup, (uint32_t)TaskSteps::StepCount); + } + catch (const std::exception&) + { + m_AbortFlag = true; + throw; + } +} + +BuildsOperationValidateBuildPart::ResolvedBuildPart +BuildsOperationValidateBuildPart::ResolveBuildPart() +{ + ResolvedBuildPart Result; + Result.PreferredMultipartChunkSize = 32u * 1024u * 1024u; + + CbObject Build = m_Storage.GetBuild(m_BuildId); + if (!m_BuildPartName.empty()) + { + m_BuildPartId = Build["parts"sv].AsObjectView()[m_BuildPartName].AsObjectId(); + if (m_BuildPartId == Oid::Zero) + { + throw std::runtime_error(fmt::format("Build {} does not have a part named '{}'", m_BuildId, m_BuildPartName)); + } + } + m_ValidateStats.BuildBlobSize = Build.GetSize(); + if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) + { + Result.PreferredMultipartChunkSize = ChunkSize; + } + + m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::FetchBuildPart, (uint32_t)TaskSteps::StepCount); + + CbObject BuildPart = m_Storage.GetBuildPart(m_BuildId, m_BuildPartId); + m_ValidateStats.BuildPartSize = BuildPart.GetSize(); + if (!m_Options.IsQuiet) + { + ZEN_INFO("Validating build part {}/{} ({})", m_BuildId, m_BuildPartId, NiceBytes(BuildPart.GetSize())); + } + if (const CbObjectView ChunkAttachmentsView = BuildPart["chunkAttachments"sv].AsObjectView()) + { + for (CbFieldView LooseFileView : ChunkAttachmentsView["rawHashes"sv]) + { + Result.ChunkAttachments.push_back(LooseFileView.AsBinaryAttachment()); + } + } + m_ValidateStats.ChunkAttachmentCount = Result.ChunkAttachments.size(); + if (const CbObjectView BlockAttachmentsView = BuildPart["blockAttachments"sv].AsObjectView()) + { + for (CbFieldView BlocksView : BlockAttachmentsView["rawHashes"sv]) + { + Result.BlockAttachments.push_back(BlocksView.AsBinaryAttachment()); + } + } + m_ValidateStats.BlockAttachmentCount = Result.BlockAttachments.size(); + + std::vector VerifyBlockDescriptions = + ParseChunkBlockDescriptionList(m_Storage.GetBlockMetadatas(m_BuildId, Result.BlockAttachments)); + if (VerifyBlockDescriptions.size() != Result.BlockAttachments.size()) + { + throw std::runtime_error(fmt::format("Uploaded blocks metadata could not all be found, {} blocks metadata is missing", + Result.BlockAttachments.size() - VerifyBlockDescriptions.size())); + } + + return Result; +} + +void +BuildsOperationValidateBuildPart::ScheduleChunkAttachmentValidation(ValidateBlobsContext& Context, + std::span ChunkAttachments, + const std::filesystem::path& TempFolder, + uint64_t PreferredMultipartChunkSize) +{ + for (const IoHash& ChunkAttachment : ChunkAttachments) + { + Context.Work.ScheduleWork( + m_NetworkPool, + [this, &Context, &TempFolder, PreferredMultipartChunkSize, ChunkAttachment = IoHash(ChunkAttachment)](std::atomic&) { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("ValidateBuildPart_GetChunk"); + + Context.FilteredDownloadedBytesPerSecond.Start(); + DownloadLargeBlob( + m_Storage, + TempFolder, + m_BuildId, + ChunkAttachment, + PreferredMultipartChunkSize, + Context.Work, + m_NetworkPool, + m_DownloadStats.DownloadedChunkByteCount, + m_DownloadStats.MultipartAttachmentCount, + [this, &Context, ChunkHash = IoHash(ChunkAttachment)](IoBuffer&& Payload) { + m_DownloadStats.DownloadedChunkCount++; + Payload.SetContentType(ZenContentType::kCompressedBinary); + if (!m_AbortFlag) + { + Context.Work.ScheduleWork( + m_IOWorkerPool, + [this, &Context, Payload = IoBuffer(std::move(Payload)), ChunkHash](std::atomic&) mutable { + if (!m_AbortFlag) + { + ValidateDownloadedChunk(Context, ChunkHash, std::move(Payload)); + } + }); + } + }); + } + }); + } +} + +void +BuildsOperationValidateBuildPart::ScheduleBlockAttachmentValidation(ValidateBlobsContext& Context, std::span BlockAttachments) +{ + for (const IoHash& BlockAttachment : BlockAttachments) + { + Context.Work.ScheduleWork(m_NetworkPool, [this, &Context, BlockAttachment = IoHash(BlockAttachment)](std::atomic&) { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("ValidateBuildPart_GetBlock"); + + Context.FilteredDownloadedBytesPerSecond.Start(); + IoBuffer Payload = m_Storage.GetBuildBlob(m_BuildId, BlockAttachment); + m_DownloadStats.DownloadedBlockCount++; + m_DownloadStats.DownloadedBlockByteCount += Payload.GetSize(); + if (m_DownloadStats.DownloadedChunkCount + m_DownloadStats.DownloadedBlockCount == Context.AttachmentsToVerifyCount) + { + Context.FilteredDownloadedBytesPerSecond.Stop(); + } + if (!Payload) + { + throw std::runtime_error(fmt::format("Block attachment {} could not be found", BlockAttachment)); + } + if (!m_AbortFlag) + { + Context.Work.ScheduleWork(m_IOWorkerPool, + [this, &Context, Payload = std::move(Payload), BlockAttachment](std::atomic&) mutable { + if (!m_AbortFlag) + { + ValidateDownloadedBlock(Context, BlockAttachment, std::move(Payload)); + } + }); + } + } + }); + } +} + +void +BuildsOperationValidateBuildPart::ValidateDownloadedChunk(ValidateBlobsContext& Context, const IoHash& ChunkHash, IoBuffer Payload) +{ + ZEN_TRACE_CPU("ValidateBuildPart_Validate"); + + if (m_DownloadStats.DownloadedChunkCount + m_DownloadStats.DownloadedBlockCount == Context.AttachmentsToVerifyCount) + { + Context.FilteredDownloadedBytesPerSecond.Stop(); + } + + Context.FilteredVerifiedBytesPerSecond.Start(); + + uint64_t CompressedSize; + uint64_t DecompressedSize; + ValidateBlob(m_AbortFlag, std::move(Payload), ChunkHash, CompressedSize, DecompressedSize); + m_ValidateStats.VerifiedAttachmentCount++; + m_ValidateStats.VerifiedByteCount += DecompressedSize; + if (m_ValidateStats.VerifiedAttachmentCount.load() == Context.AttachmentsToVerifyCount) + { + Context.FilteredVerifiedBytesPerSecond.Stop(); + } +} + +void +BuildsOperationValidateBuildPart::ValidateDownloadedBlock(ValidateBlobsContext& Context, const IoHash& BlockAttachment, IoBuffer Payload) +{ + ZEN_TRACE_CPU("ValidateBuildPart_ValidateBlock"); + + Context.FilteredVerifiedBytesPerSecond.Start(); + + uint64_t CompressedSize; + uint64_t DecompressedSize; + ValidateChunkBlock(std::move(Payload), BlockAttachment, CompressedSize, DecompressedSize); + m_ValidateStats.VerifiedAttachmentCount++; + m_ValidateStats.VerifiedByteCount += DecompressedSize; + if (m_ValidateStats.VerifiedAttachmentCount.load() == Context.AttachmentsToVerifyCount) + { + Context.FilteredVerifiedBytesPerSecond.Stop(); + } +} + +ChunkBlockDescription +BuildsOperationValidateBuildPart::ValidateChunkBlock(IoBuffer&& Payload, + const IoHash& BlobHash, + uint64_t& OutCompressedSize, + uint64_t& OutDecompressedSize) +{ + CompositeBuffer BlockBuffer = ValidateBlob(m_AbortFlag, std::move(Payload), BlobHash, OutCompressedSize, OutDecompressedSize); + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Chunk block blob {} is not compressed using 'None' compression level", BlobHash)); + } + return GetChunkBlockDescription(BlockBuffer.Flatten(), BlobHash); +} + +void +ValidateBuildPart(LoggerRef Log, + ProgressBase& Progress, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + bool IsQuiet, + bool IsVerbose, + TransferThreadWorkers& Workers, + BuildStorageBase& Storage, + const Oid& BuildId, + const Oid& BuildPartId, + std::string_view BuildPartName) +{ + ZEN_TRACE_CPU("ValidateBuildPart"); + + Progress.SetLogOperationName("Validate Part"); + + BuildsOperationValidateBuildPart ValidateOp(Log, + Progress, + Storage, + AbortFlag, + PauseFlag, + Workers.GetIOWorkerPool(), + Workers.GetNetworkPool(), + BuildId, + BuildPartId, + BuildPartName, + BuildsOperationValidateBuildPart::Options{.IsQuiet = IsQuiet, .IsVerbose = IsVerbose}); + + ValidateOp.Execute(); + + const uint64_t DownloadedCount = ValidateOp.m_DownloadStats.DownloadedChunkCount + ValidateOp.m_DownloadStats.DownloadedBlockCount; + const uint64_t DownloadedByteCount = + ValidateOp.m_DownloadStats.DownloadedChunkByteCount + ValidateOp.m_DownloadStats.DownloadedBlockByteCount; + ZEN_CONSOLE("Verified: {:>8} ({}), {}B/sec, {}", + DownloadedCount, + NiceBytes(DownloadedByteCount), + NiceNum(GetBytesPerSecond(ValidateOp.m_ValidateStats.ElapsedWallTimeUS, DownloadedByteCount)), + NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000)); +} + +} // namespace zen diff --git a/src/zenremotestore/builds/jupiterbuildstorage.cpp b/src/zenremotestore/builds/jupiterbuildstorage.cpp index d837ce07f..96b7a5973 100644 --- a/src/zenremotestore/builds/jupiterbuildstorage.cpp +++ b/src/zenremotestore/builds/jupiterbuildstorage.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/src/zenremotestore/include/zenremotestore/builds/buildinspect.h b/src/zenremotestore/include/zenremotestore/builds/buildinspect.h new file mode 100644 index 000000000..7f6c65367 --- /dev/null +++ b/src/zenremotestore/include/zenremotestore/builds/buildinspect.h @@ -0,0 +1,60 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace zen { + +class CbObjectWriter; +class ChunkingCache; +class ChunkingController; +class ProgressBase; +class TransferThreadWorkers; +struct StorageInstance; + +ChunkedFolderContent ScanAndChunkFolder( + ProgressBase& Progress, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + bool IsQuiet, + TransferThreadWorkers& Workers, + GetFolderContentStatistics& GetFolderContentStats, + ChunkingStatistics& ChunkingStats, + const std::filesystem::path& Path, + std::function&& IsAcceptedFolder, + std::function&& IsAcceptedFile, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache); + +////////////////////////////////////////////////////////////////////////// + +void ListBuild(bool IsQuiet, + StorageInstance& Storage, + const Oid& BuildId, + const std::vector& BuildPartIds, + std::span BuildPartNames, + std::span IncludeWildcards, + std::span ExcludeWildcards, + CbObjectWriter* OptionalStructuredOutput); + +void DiffFolders(ProgressBase& Progress, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + bool IsQuiet, + TransferThreadWorkers& Workers, + const std::filesystem::path& BasePath, + const std::filesystem::path& ComparePath, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + const std::vector& ExcludeFolders, + const std::vector& ExcludeExtensions); + +} // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/builds/buildprimecache.h b/src/zenremotestore/include/zenremotestore/builds/buildprimecache.h new file mode 100644 index 000000000..1d04ccbfe --- /dev/null +++ b/src/zenremotestore/include/zenremotestore/builds/buildprimecache.h @@ -0,0 +1,96 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +class FilteredRate; +class ParallelWork; +class ProgressBase; +class WorkerThreadPool; +struct StorageInstance; + +class BuildsOperationPrimeCache +{ +public: + struct Options + { + bool IsQuiet = false; + bool IsVerbose = false; + std::filesystem::path ZenFolderPath; + std::uint64_t LargeAttachmentSize = 32u * 1024u * 1024u * 4u; + std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; + bool ForceUpload = false; + }; + + BuildsOperationPrimeCache(LoggerRef Log, + ProgressBase& Progress, + StorageInstance& Storage, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + WorkerThreadPool& NetworkPool, + const Oid& BuildId, + std::span BuildPartIds, + const Options& Options, + BuildStorageCache::Statistics& StorageCacheStats); + + void Execute(); + + DownloadStatistics m_DownloadStats; + +private: + LoggerRef Log() { return m_Log; } + + void CollectReferencedBlobs(tsl::robin_set& OutBuildBlobs, + tsl::robin_map& OutLooseChunkRawSizes); + + std::vector FilterAlreadyCachedBlobs(const tsl::robin_set& BuildBlobs); + + void ScheduleBlobDownloads(std::span BlobsToDownload, + const tsl::robin_map& LooseChunkRawSizes, + std::atomic& MultipartAttachmentCount, + std::atomic& CompletedDownloadCount, + FilteredRate& FilteredDownloadedBytesPerSecond); + + void DownloadLargeBlobForCache(ParallelWork& Work, + const IoHash& BlobHash, + size_t BlobCount, + std::atomic& CompletedDownloadCount, + std::atomic& MultipartAttachmentCount, + FilteredRate& FilteredDownloadedBytesPerSecond); + + void DownloadSingleBlobForCache(const IoHash& BlobHash, + size_t BlobCount, + std::atomic& CompletedDownloadCount, + FilteredRate& FilteredDownloadedBytesPerSecond); + + LoggerRef m_Log; + ProgressBase& m_Progress; + StorageInstance& m_Storage; + std::atomic& m_AbortFlag; + std::atomic& m_PauseFlag; + WorkerThreadPool& m_NetworkPool; + const Oid m_BuildId; + std::vector m_BuildPartIds; + Options m_Options; + std::filesystem::path m_TempPath; + + BuildStorageCache::Statistics& m_StorageCacheStats; +}; + +} // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorage.h b/src/zenremotestore/include/zenremotestore/builds/buildstorage.h index da8437a58..b933ab95d 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorage.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorage.h @@ -3,7 +3,7 @@ #pragma once #include -#include +#include ZEN_THIRD_PARTY_INCLUDES_START #include diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h b/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h index 24702df0f..4e0bd7243 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h @@ -2,11 +2,9 @@ #pragma once -#include - #include #include -#include +#include namespace zen { diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h deleted file mode 100644 index 2d29e5efd..000000000 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h +++ /dev/null @@ -1,1123 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END - -namespace zen { - -class CloneQueryInterface; - -class ProgressBase; -class BuildStorageBase; -class HttpClient; -class ParallelWork; -class WorkerThreadPool; -class FilteredRate; -class ReadFileCache; -struct StorageInstance; - -class BufferedWriteFileCache; -struct ChunkBlockDescription; -struct ChunkedFolderContent; - -struct DiskStatistics -{ - std::atomic OpenReadCount = 0; - std::atomic OpenWriteCount = 0; - std::atomic ReadCount = 0; - std::atomic ReadByteCount = 0; - std::atomic WriteCount = 0; - std::atomic WriteByteCount = 0; - std::atomic CloneCount = 0; - std::atomic CloneByteCount = 0; - std::atomic CurrentOpenFileCount = 0; -}; - -struct CacheMappingStatistics -{ - uint64_t CacheChunkCount = 0; - uint64_t CacheChunkByteCount = 0; - - uint64_t CacheBlockCount = 0; - uint64_t CacheBlocksByteCount = 0; - - uint64_t CacheSequenceHashesCount = 0; - uint64_t CacheSequenceHashesByteCount = 0; - - uint64_t CacheScanElapsedWallTimeUs = 0; - - uint32_t LocalPathsMatchingSequencesCount = 0; - uint64_t LocalPathsMatchingSequencesByteCount = 0; - - uint64_t LocalChunkMatchingRemoteCount = 0; - uint64_t LocalChunkMatchingRemoteByteCount = 0; - - uint64_t LocalScanElapsedWallTimeUs = 0; - - uint32_t ScavengedPathsMatchingSequencesCount = 0; - uint64_t ScavengedPathsMatchingSequencesByteCount = 0; - - uint64_t ScavengedChunkMatchingRemoteCount = 0; - uint64_t ScavengedChunkMatchingRemoteByteCount = 0; - - uint64_t ScavengeElapsedWallTimeUs = 0; -}; - -struct DownloadStatistics -{ - std::atomic RequestsCompleteCount = 0; - - std::atomic DownloadedChunkCount = 0; - std::atomic DownloadedChunkByteCount = 0; - std::atomic MultipartAttachmentCount = 0; - - std::atomic DownloadedBlockCount = 0; - std::atomic DownloadedBlockByteCount = 0; - - std::atomic DownloadedPartialBlockCount = 0; - std::atomic DownloadedPartialBlockByteCount = 0; -}; - -struct WriteChunkStatistics -{ - uint64_t DownloadTimeUs = 0; - uint64_t WriteTimeUs = 0; - uint64_t WriteChunksElapsedWallTimeUs = 0; -}; - -struct RebuildFolderStateStatistics -{ - uint64_t CleanFolderElapsedWallTimeUs = 0; - std::atomic FinalizeTreeFilesMovedCount = 0; - std::atomic FinalizeTreeFilesCopiedCount = 0; - uint64_t FinalizeTreeElapsedWallTimeUs = 0; -}; - -std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath); -std::filesystem::path ZenTempFolderPath(const std::filesystem::path& ZenFolderPath); - -class BuildsOperationUpdateFolder -{ -public: - struct Options - { - bool IsQuiet = false; - bool IsVerbose = false; - bool AllowFileClone = true; - bool UseSparseFiles = true; - std::filesystem::path SystemRootDir; - std::filesystem::path ZenFolderPath; - std::uint64_t LargeAttachmentSize = 32u * 1024u * 1024u * 4u; - std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - EPartialBlockRequestMode PartialBlockRequestMode = EPartialBlockRequestMode::Mixed; - bool WipeTargetFolder = false; - bool EnableOtherDownloadsScavenging = true; - bool EnableTargetFolderScavenging = true; - bool ValidateCompletedSequences = true; - std::vector ExcludeFolders; - uint64_t MaximumInMemoryPayloadSize = 512u * 1024u; - bool PopulateCache = true; - }; - - BuildsOperationUpdateFolder(LoggerRef Log, - ProgressBase& Progress, - StorageInstance& Storage, - std::atomic& AbortFlag, - std::atomic& PauseFlag, - WorkerThreadPool& IOWorkerPool, - WorkerThreadPool& NetworkPool, - const Oid& BuildId, - const std::filesystem::path& Path, - const ChunkedFolderContent& LocalContent, - const ChunkedContentLookup& LocalLookup, - const ChunkedFolderContent& RemoteContent, - const ChunkedContentLookup& RemoteLookup, - const std::vector& BlockDescriptions, - const std::vector& LooseChunkHashes, - const Options& Options); - - void Execute(FolderContent& OutLocalFolderState); - - DiskStatistics m_DiskStats; - CacheMappingStatistics m_CacheMappingStats; - GetFolderContentStatistics m_ScavengedFolderScanStats; - DownloadStatistics m_DownloadStats; - WriteChunkStatistics m_WriteChunkStats; - RebuildFolderStateStatistics m_RebuildFolderStateStats; - std::atomic m_WrittenChunkByteCount = 0; - -private: - struct BlockWriteOps - { - std::vector ChunkBuffers; - struct WriteOpData - { - const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr; - size_t ChunkBufferIndex = (size_t)-1; - }; - std::vector WriteOps; - }; - - struct ScavengeSource - { - std::filesystem::path StateFilePath; - std::filesystem::path Path; - }; - - struct ScavengedSequenceCopyOperation - { - uint32_t ScavengedContentIndex = (uint32_t)-1; - uint32_t ScavengedPathIndex = (uint32_t)-1; - uint32_t RemoteSequenceIndex = (uint32_t)-1; - uint64_t RawSize = (uint64_t)-1; - }; - - struct CopyChunkData - { - uint32_t ScavengeSourceIndex = (uint32_t)-1; - uint32_t SourceSequenceIndex = (uint32_t)-1; - std::vector TargetChunkLocationPtrs; - struct ChunkTarget - { - uint32_t TargetChunkLocationCount = (uint32_t)-1; - uint32_t RemoteChunkIndex = (uint32_t)-1; - uint64_t CacheFileOffset = (uint64_t)-1; - }; - std::vector ChunkTargets; - }; - - struct BlobsExistsResult - { - tsl::robin_set ExistingBlobs; - uint64_t ElapsedTimeMs = 0; - }; - - struct LooseChunkHashWorkData - { - std::vector ChunkTargetPtrs; - uint32_t RemoteChunkIndex = (uint32_t)-1; - }; - - struct FinalizeTarget - { - IoHash RawHash; - uint32_t RemotePathIndex; - }; - - struct LocalPathCategorization - { - std::vector FilesToCache; - std::vector RemoveLocalPathIndexes; - tsl::robin_map RemotePathIndexToLocalPathIndex; - tsl::robin_map SequenceHashToLocalPathIndex; - uint64_t MatchCount = 0; - uint64_t PathMismatchCount = 0; - uint64_t HashMismatchCount = 0; - uint64_t SkippedCount = 0; - uint64_t DeleteCount = 0; - }; - - struct WriteChunksContext - { - ParallelWork& Work; - BufferedWriteFileCache& WriteCache; - std::span> SequenceIndexChunksLeftToWriteCounters; - std::span> RemoteChunkIndexNeedsCopyFromSourceFlags; - std::atomic& WritePartsComplete; - uint64_t TotalPartWriteCount; - uint64_t TotalRequestCount; - const BlobsExistsResult& ExistsResult; - FilteredRate& FilteredDownloadedBytesPerSecond; - FilteredRate& FilteredWrittenBytesPerSecond; - }; - - void ScanCacheFolder(tsl::robin_map& OutCachedChunkHashesFound, - tsl::robin_map& OutCachedSequenceHashesFound); - void ScanTempBlocksFolder(tsl::robin_map& OutCachedBlocksFound); - std::vector ScanTargetFolder(const tsl::robin_map& CachedChunkHashesFound, - const tsl::robin_map& CachedSequenceHashesFound); - - std::vector FindScavengeSources(); - - bool FindScavengeContent(const ScavengeSource& Source, - ChunkedFolderContent& OutScavengedLocalContent, - ChunkedContentLookup& OutScavengedLookup); - - void ScavengeSourceForChunks(uint32_t& InOutRemainingChunkCount, - std::vector& InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags, - tsl::robin_map& InOutRawHashToCopyChunkDataIndex, - const std::vector>& SequenceIndexChunksLeftToWriteCounters, - const ChunkedFolderContent& ScavengedContent, - const ChunkedContentLookup& ScavengedLookup, - std::vector& InOutCopyChunkDatas, - uint32_t ScavengedContentIndex, - uint64_t& InOutChunkMatchingRemoteCount, - uint64_t& InOutChunkMatchingRemoteByteCount); - - std::filesystem::path FindDownloadedChunk(const IoHash& ChunkHash); - - std::vector GetRemainingChunkTargets( - std::span> SequenceIndexChunksLeftToWriteCounters, - uint32_t ChunkIndex); - - uint64_t GetChunkWriteCount(std::span> SequenceIndexChunksLeftToWriteCounters, uint32_t ChunkIndex); - - void CheckRequiredDiskSpace(const tsl::robin_map& RemotePathToRemoteIndex); - - void WriteScavengedSequenceToCache(const std::filesystem::path& ScavengeRootPath, - const ChunkedFolderContent& ScavengedContent, - const ScavengedSequenceCopyOperation& ScavengeOp); - - void WriteLooseChunk(const uint32_t RemoteChunkIndex, - const BlobsExistsResult& ExistsResult, - std::span> SequenceIndexChunksLeftToWriteCounters, - std::atomic& WritePartsComplete, - std::vector&& ChunkTargetPtrs, - BufferedWriteFileCache& WriteCache, - ParallelWork& Work, - uint64_t TotalRequestCount, - uint64_t TotalPartWriteCount, - FilteredRate& FilteredDownloadedBytesPerSecond, - FilteredRate& FilteredWrittenBytesPerSecond); - - void DownloadBuildBlob(uint32_t RemoteChunkIndex, - const BlobsExistsResult& ExistsResult, - ParallelWork& Work, - uint64_t TotalRequestCount, - FilteredRate& FilteredDownloadedBytesPerSecond, - std::function&& OnDownloaded); - - void DownloadPartialBlock(std::span BlockRanges, - size_t BlockRangeIndex, - size_t BlockRangeCount, - const BlobsExistsResult& ExistsResult, - uint64_t TotalRequestCount, - FilteredRate& FilteredDownloadedBytesPerSecond, - std::function> OffsetAndLengths)>&& OnDownloaded); - - std::vector WriteLocalChunkToCache(CloneQueryInterface* CloneQuery, - const CopyChunkData& CopyData, - const std::vector& ScavengedContents, - const std::vector& ScavengedLookups, - const std::vector& ScavengedPaths, - BufferedWriteFileCache& WriteCache); - - bool WriteCompressedChunkToCache(const IoHash& ChunkHash, - const std::vector& ChunkTargetPtrs, - BufferedWriteFileCache& WriteCache, - IoBuffer&& CompressedPart); - - void StreamDecompress(const IoHash& SequenceRawHash, CompositeBuffer&& CompressedPart); - - void WriteSequenceChunkToCache(BufferedWriteFileCache::Local& LocalWriter, - const CompositeBuffer& Chunk, - const uint32_t SequenceIndex, - const uint64_t FileOffset, - const uint32_t PathIndex); - - bool GetBlockWriteOps(const IoHash& BlockRawHash, - std::span ChunkRawHashes, - std::span ChunkCompressedLengths, - std::span> SequenceIndexChunksLeftToWriteCounters, - std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - const MemoryView BlockView, - uint32_t FirstIncludedBlockChunkIndex, - uint32_t LastIncludedBlockChunkIndex, - BlockWriteOps& OutOps); - - void WriteBlockChunkOpsToCache(std::span> SequenceIndexChunksLeftToWriteCounters, - const BlockWriteOps& Ops, - BufferedWriteFileCache& WriteCache, - ParallelWork& Work); - - bool WriteChunksBlockToCache(const ChunkBlockDescription& BlockDescription, - std::span> SequenceIndexChunksLeftToWriteCounters, - ParallelWork& Work, - CompositeBuffer&& BlockBuffer, - std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - BufferedWriteFileCache& WriteCache); - - bool WritePartialBlockChunksToCache(const ChunkBlockDescription& BlockDescription, - std::span> SequenceIndexChunksLeftToWriteCounters, - ParallelWork& Work, - CompositeBuffer&& PartialBlockBuffer, - uint32_t FirstIncludedBlockChunkIndex, - uint32_t LastIncludedBlockChunkIndex, - std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - BufferedWriteFileCache& WriteCache); - - void AsyncWriteDownloadedChunk(uint32_t RemoteChunkIndex, - const BlobsExistsResult& ExistsResult, - std::vector&& ChunkTargetPtrs, - BufferedWriteFileCache& WriteCache, - ParallelWork& Work, - IoBuffer&& Payload, - std::span> SequenceIndexChunksLeftToWriteCounters, - std::atomic& WritePartsComplete, - const uint64_t TotalPartWriteCount, - FilteredRate& FilteredWrittenBytesPerSecond); - - void VerifyAndCompleteChunkSequencesAsync(std::span RemoteSequenceIndexes, ParallelWork& Work); - bool CompleteSequenceChunk(uint32_t RemoteSequenceIndex, std::span> SequenceIndexChunksLeftToWriteCounters); - std::vector CompleteChunkTargets(const std::vector& ChunkTargetPtrs, - std::span> SequenceIndexChunksLeftToWriteCounters); - void FinalizeChunkSequence(const IoHash& SequenceRawHash); - void FinalizeChunkSequences(std::span RemoteSequenceIndexes); - void VerifySequence(uint32_t RemoteSequenceIndex); - - void InitializeSequenceCounters(std::vector>& OutSequenceCounters, - tsl::robin_map& OutSequencesLeftToFind, - const tsl::robin_map& CachedChunkHashesFound, - const tsl::robin_map& CachedSequenceHashesFound); - - void MatchScavengedSequencesToRemote(std::span Contents, - std::span Lookups, - std::span Paths, - tsl::robin_map& InOutSequencesLeftToFind, - std::vector>& InOutSequenceCounters, - std::vector& OutCopyOperations, - uint64_t& OutScavengedPathsCount); - - uint64_t CalculateBytesToWriteAndFlagNeededChunks(std::span> SequenceCounters, - const std::vector& NeedsCopyFromLocalFileFlags, - std::span> OutNeedsCopyFromSourceFlags); - - void ClassifyCachedAndFetchBlocks(std::span NeededBlocks, - const tsl::robin_map& CachedBlocksFound, - uint64_t& TotalPartWriteCount, - std::vector& OutCachedChunkBlockIndexes, - std::vector& OutFetchBlockIndexes); - - std::vector DetermineNeededLooseChunkIndexes(std::span> SequenceCounters, - const std::vector& NeedsCopyFromLocalFileFlags, - std::span> NeedsCopyFromSourceFlags); - - BlobsExistsResult QueryBlobCacheExists(std::span NeededLooseChunkIndexes, std::span FetchBlockIndexes); - - std::vector DeterminePartialDownloadModes(const BlobsExistsResult& ExistsResult); - - std::vector BuildLooseChunkHashWorks(std::span NeededLooseChunkIndexes, - std::span> SequenceCounters); - - void VerifyWriteChunksComplete(std::span> SequenceCounters, - uint64_t BytesToWrite, - uint64_t BytesToValidate); - - std::vector BuildSortedFinalizeTargets(); - - void ScanScavengeSources(std::span Sources, - std::vector& OutContents, - std::vector& OutLookups, - std::vector& OutPaths); - - LocalPathCategorization CategorizeLocalPaths(const tsl::robin_map& RemotePathToRemoteIndex); - - void ScheduleLocalFileCaching(std::span FilesToCache, - std::atomic& OutCachedCount, - std::atomic& OutCachedByteCount); - - void ScheduleScavengedSequenceWrites(WriteChunksContext& Context, - std::span CopyOperations, - const std::vector& ScavengedContents, - const std::vector& ScavengedPaths); - - void ScheduleLooseChunkWrites(WriteChunksContext& Context, std::vector& LooseChunkHashWorks); - - void ScheduleLocalChunkCopies(WriteChunksContext& Context, - std::span CopyChunkDatas, - CloneQueryInterface* CloneQuery, - const std::vector& ScavengedContents, - const std::vector& ScavengedLookups, - const std::vector& ScavengedPaths); - - void ScheduleCachedBlockWrites(WriteChunksContext& Context, std::span CachedBlockIndexes); - - void SchedulePartialBlockDownloads(WriteChunksContext& Context, const ChunkBlockAnalyser::BlockResult& PartialBlocks); - - void WritePartialBlockToCache(WriteChunksContext& Context, - size_t BlockRangeStartIndex, - IoBuffer BlockPartialBuffer, - const std::filesystem::path& BlockChunkPath, - std::span> OffsetAndLengths, - const ChunkBlockAnalyser::BlockResult& PartialBlocks); - - void ScheduleFullBlockDownloads(WriteChunksContext& Context, std::span FullBlockIndexes); - - void WriteFullBlockToCache(WriteChunksContext& Context, - uint32_t BlockIndex, - IoBuffer BlockBuffer, - const std::filesystem::path& BlockChunkPath); - - void ScheduleLocalFileRemovals(ParallelWork& Work, - std::span RemoveLocalPathIndexes, - std::atomic& DeletedCount); - - void ScheduleTargetFinalization(ParallelWork& Work, - std::span Targets, - const tsl::robin_map& SequenceHashToLocalPathIndex, - const tsl::robin_map& RemotePathIndexToLocalPathIndex, - FolderContent& OutLocalFolderState, - std::atomic& TargetsComplete); - - void FinalizeTargetGroup(size_t BaseOffset, - size_t Count, - std::span Targets, - const tsl::robin_map& SequenceHashToLocalPathIndex, - const tsl::robin_map& RemotePathIndexToLocalPathIndex, - FolderContent& OutLocalFolderState, - std::atomic& TargetsComplete); - - LoggerRef Log() { return m_Log; } - - LoggerRef m_Log; - ProgressBase& m_Progress; - StorageInstance& m_Storage; - std::atomic& m_AbortFlag; - std::atomic& m_PauseFlag; - WorkerThreadPool& m_IOWorkerPool; - WorkerThreadPool& m_NetworkPool; - const Oid m_BuildId; - const std::filesystem::path m_Path; - const ChunkedFolderContent& m_LocalContent; - const ChunkedContentLookup& m_LocalLookup; - const ChunkedFolderContent& m_RemoteContent; - const ChunkedContentLookup& m_RemoteLookup; - const std::vector& m_BlockDescriptions; - const std::vector& m_LooseChunkHashes; - const Options m_Options; - const std::filesystem::path m_CacheFolderPath; - const std::filesystem::path m_TempDownloadFolderPath; - const std::filesystem::path m_TempBlockFolderPath; - - std::atomic m_ValidatedChunkByteCount = 0; -}; - -struct FindBlocksStatistics -{ - uint64_t FindBlockTimeMS = 0; - uint64_t PotentialChunkCount = 0; - uint64_t PotentialChunkByteCount = 0; - uint64_t FoundBlockCount = 0; - uint64_t FoundBlockChunkCount = 0; - uint64_t FoundBlockByteCount = 0; - uint64_t AcceptedBlockCount = 0; - uint64_t NewBlocksCount = 0; - uint64_t NewBlocksChunkCount = 0; - uint64_t NewBlocksChunkByteCount = 0; - - FindBlocksStatistics& operator+=(const FindBlocksStatistics& Rhs) - { - FindBlockTimeMS += Rhs.FindBlockTimeMS; - PotentialChunkCount += Rhs.PotentialChunkCount; - PotentialChunkByteCount += Rhs.PotentialChunkByteCount; - FoundBlockCount += Rhs.FoundBlockCount; - FoundBlockChunkCount += Rhs.FoundBlockChunkCount; - FoundBlockByteCount += Rhs.FoundBlockByteCount; - AcceptedBlockCount += Rhs.AcceptedBlockCount; - NewBlocksCount += Rhs.NewBlocksCount; - NewBlocksChunkCount += Rhs.NewBlocksChunkCount; - NewBlocksChunkByteCount += Rhs.NewBlocksChunkByteCount; - return *this; - } -}; - -struct UploadStatistics -{ - std::atomic BlockCount = 0; - std::atomic BlocksBytes = 0; - std::atomic ChunkCount = 0; - std::atomic ChunksBytes = 0; - std::atomic ReadFromDiskBytes = 0; - std::atomic MultipartAttachmentCount = 0; - uint64_t ElapsedWallTimeUS = 0; - - UploadStatistics& operator+=(const UploadStatistics& Rhs) - { - BlockCount += Rhs.BlockCount; - BlocksBytes += Rhs.BlocksBytes; - ChunkCount += Rhs.ChunkCount; - ChunksBytes += Rhs.ChunksBytes; - ReadFromDiskBytes += Rhs.ReadFromDiskBytes; - MultipartAttachmentCount += Rhs.MultipartAttachmentCount; - ElapsedWallTimeUS += Rhs.ElapsedWallTimeUS; - return *this; - } -}; - -struct LooseChunksStatistics -{ - uint64_t ChunkCount = 0; - uint64_t ChunkByteCount = 0; - std::atomic CompressedChunkCount = 0; - std::atomic CompressedChunkRawBytes = 0; - std::atomic CompressedChunkBytes = 0; - uint64_t CompressChunksElapsedWallTimeUS = 0; - - LooseChunksStatistics& operator+=(const LooseChunksStatistics& Rhs) - { - ChunkCount += Rhs.ChunkCount; - ChunkByteCount += Rhs.ChunkByteCount; - CompressedChunkCount += Rhs.CompressedChunkCount; - CompressedChunkRawBytes += Rhs.CompressedChunkRawBytes; - CompressedChunkBytes += Rhs.CompressedChunkBytes; - CompressChunksElapsedWallTimeUS += Rhs.CompressChunksElapsedWallTimeUS; - return *this; - } -}; - -struct GenerateBlocksStatistics -{ - std::atomic GeneratedBlockByteCount = 0; - std::atomic GeneratedBlockCount = 0; - uint64_t GenerateBlocksElapsedWallTimeUS = 0; - - GenerateBlocksStatistics& operator+=(const GenerateBlocksStatistics& Rhs) - { - GeneratedBlockByteCount += Rhs.GeneratedBlockByteCount; - GeneratedBlockCount += Rhs.GeneratedBlockCount; - GenerateBlocksElapsedWallTimeUS += Rhs.GenerateBlocksElapsedWallTimeUS; - return *this; - } -}; - -static constexpr size_t DefaultMaxChunkBlockSize = 64u * 1024u * 1024u; -static constexpr size_t DefaultMaxChunksPerChunkBlock = 4u * 1000u; -static constexpr size_t DefaultMaxChunkBlockEmbedSize = 3u * 512u * 1024u; - -class BuildsOperationUploadFolder -{ -public: - struct ChunksBlockParameters - { - size_t MaxBlockSize = DefaultMaxChunkBlockSize; - size_t MaxChunksPerBlock = DefaultMaxChunksPerChunkBlock; - size_t MaxChunkEmbedSize = DefaultMaxChunkBlockEmbedSize; - }; - - struct Options - { - bool IsQuiet = false; - bool IsVerbose = false; - bool DoExtraContentValidation = false; - - const uint64_t FindBlockMaxCount = 10000; - const uint8_t BlockReuseMinPercentLimit = 85; - bool AllowMultiparts = true; - bool IgnoreExistingBlocks = false; - ChunksBlockParameters BlockParameters; - - uint32_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - - const uint64_t MinimumSizeForCompressInBlock = 2u * 1024u; - - std::filesystem::path TempDir; - std::vector ExcludeFolders; - std::vector ExcludeExtensions; - std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; - - std::vector NonCompressableExtensions; - - bool PopulateCache = true; - }; - BuildsOperationUploadFolder(LoggerRef Log, - ProgressBase& Progress, - StorageInstance& Storage, - std::atomic& AbortFlag, - std::atomic& PauseFlag, - WorkerThreadPool& IOWorkerPool, - WorkerThreadPool& NetworkPool, - const Oid& BuildId, - const std::filesystem::path& Path, - bool CreateBuild, - const CbObject& MetaData, - const Options& Options); - - std::vector> Execute(const Oid& BuildPartId, - const std::string_view BuildPartName, - const std::filesystem::path& ManifestPath, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache); - - DiskStatistics m_DiskStats; - GetFolderContentStatistics m_LocalFolderScanStats; - ChunkingStatistics m_ChunkingStats; - FindBlocksStatistics m_FindBlocksStats; - ReuseBlocksStatistics m_ReuseBlocksStats; - UploadStatistics m_UploadStats; - GenerateBlocksStatistics m_GenerateBlocksStats; - LooseChunksStatistics m_LooseChunksStats; - -private: - struct PrepareBuildResult - { - std::vector KnownBlocks; - uint64_t PreferredMultipartChunkSize = 0; - uint64_t PayloadSize = 0; - uint64_t PrepareBuildTimeMs = 0; - uint64_t FindBlocksTimeMs = 0; - uint64_t ElapsedTimeMs = 0; - }; - - PrepareBuildResult PrepareBuild(); - - struct UploadPart - { - Oid PartId = Oid::Zero; - std::string PartName; - FolderContent Content; - uint64_t TotalRawSize = 0; - GetFolderContentStatistics LocalFolderScanStats; - }; - - std::vector ReadFolder(); - std::vector ReadManifestParts(const std::filesystem::path& ManifestPath); - - bool IsAcceptedFolder(const std::string_view& RelativePath) const; - bool IsAcceptedFile(const std::string_view& RelativePath) const; - - void ArrangeChunksIntoBlocks(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - std::vector& ChunkIndexes, - std::vector>& OutBlocks); - struct GeneratedBlocks - { - std::vector BlockDescriptions; - std::vector BlockSizes; - std::vector BlockHeaders; - std::vector BlockMetaDatas; - std::vector - MetaDataHasBeenUploaded; // NOTE: Do not use std::vector here as this vector is modified by multiple threads - tsl::robin_map BlockHashToBlockIndex; - }; - - void GenerateBuildBlocks(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const std::vector>& NewBlockChunks, - GeneratedBlocks& OutBlocks, - GenerateBlocksStatistics& GenerateBlocksStats, - UploadStatistics& UploadStats); - - struct GenerateBuildBlocksContext - { - ParallelWork& Work; - WorkerThreadPool& GenerateBlobsPool; - WorkerThreadPool& UploadBlocksPool; - FilteredRate& FilteredGeneratedBytesPerSecond; - FilteredRate& FilteredUploadedBytesPerSecond; - std::atomic& QueuedPendingBlocksForUpload; - RwLock& Lock; - GeneratedBlocks& OutBlocks; - GenerateBlocksStatistics& GenerateBlocksStats; - UploadStatistics& UploadStats; - size_t NewBlockCount; - }; - - void ScheduleBlockGeneration(GenerateBuildBlocksContext& Context, - const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const std::vector>& NewBlockChunks); - - void UploadGeneratedBlock(GenerateBuildBlocksContext& Context, size_t BlockIndex, CompressedBuffer Payload); - - std::vector CalculateAbsoluteChunkOrders(const std::span LocalChunkHashes, - const std::span LocalChunkOrder, - const tsl::robin_map& ChunkHashToLocalChunkIndex, - const std::span& LooseChunkIndexes, - const std::span& BlockDescriptions); - - CompositeBuffer FetchChunk(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const IoHash& ChunkHash, - ReadFileCache& OpenFileCache); - - CompressedBuffer GenerateBlock(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const std::vector& ChunksInBlock, - ChunkBlockDescription& OutBlockDescription); - - CompressedBuffer RebuildBlock(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - CompositeBuffer&& HeaderBuffer, - const std::vector& ChunksInBlock); - - enum class PartTaskSteps : uint32_t - { - ChunkPartContent = 0, - CalculateDelta, - GenerateBlocks, - BuildPartManifest, - UploadBuildPart, - UploadAttachments, - PutBuildPartStats, - StepCount - }; - - void UploadBuildPart(ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - uint32_t PartIndex, - const UploadPart& Part, - uint32_t PartStepOffset, - uint32_t StepCount); - - ChunkedFolderContent ScanPartContent(const UploadPart& Part, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - ChunkingStatistics& ChunkingStats); - - void ConsumePrepareBuildResult(); - - void ClassifyChunksByBlockEligibility(const ChunkedFolderContent& LocalContent, - std::vector& OutLooseChunkIndexes, - std::vector& OutNewBlockChunkIndexes, - std::vector& OutReuseBlockIndexes, - LooseChunksStatistics& LooseChunksStats, - FindBlocksStatistics& FindBlocksStats, - ReuseBlocksStatistics& ReuseBlocksStats); - - struct BuiltPartManifest - { - CbObject PartManifest; - std::vector AllChunkBlockDescriptions; - std::vector AllChunkBlockHashes; - }; - - BuiltPartManifest BuildPartManifestObject(const ChunkedFolderContent& LocalContent, - const ChunkedContentLookup& LocalLookup, - ChunkingController& ChunkController, - std::span ReuseBlockIndexes, - const GeneratedBlocks& NewBlocks, - std::span LooseChunkIndexes); - - void UploadAttachmentBatch(std::span RawHashes, - std::vector& OutUnknownChunks, - const ChunkedFolderContent& LocalContent, - const ChunkedContentLookup& LocalLookup, - const std::vector>& NewBlockChunks, - GeneratedBlocks& NewBlocks, - std::span LooseChunkIndexes, - UploadStatistics& UploadStats, - LooseChunksStatistics& LooseChunksStats); - - void FinalizeBuildPartWithRetries(const UploadPart& Part, - const IoHash& PartHash, - std::vector& InOutUnknownChunks, - const ChunkedFolderContent& LocalContent, - const ChunkedContentLookup& LocalLookup, - const std::vector>& NewBlockChunks, - GeneratedBlocks& NewBlocks, - std::span LooseChunkIndexes, - UploadStatistics& UploadStats, - LooseChunksStatistics& LooseChunksStats); - - void UploadMissingBlockMetadata(GeneratedBlocks& NewBlocks, UploadStatistics& UploadStats); - - void UploadPartBlobs(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - std::span RawHashes, - const std::vector>& NewBlockChunks, - GeneratedBlocks& NewBlocks, - std::span LooseChunkIndexes, - const std::uint64_t LargeAttachmentSize, - UploadStatistics& TempUploadStats, - LooseChunksStatistics& TempLooseChunksStats, - std::vector& OutUnknownChunks); - - struct UploadPartClassification - { - std::vector BlockIndexes; - std::vector LooseChunkOrderIndexes; - uint64_t TotalBlocksSize = 0; - uint64_t TotalLooseChunksSize = 0; - }; - - UploadPartClassification ClassifyUploadRawHashes(std::span RawHashes, - const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const GeneratedBlocks& NewBlocks, - std::span LooseChunkIndexes, - std::vector& OutUnknownChunks); - - struct UploadPartBlobsContext - { - ParallelWork& Work; - WorkerThreadPool& ReadChunkPool; - WorkerThreadPool& UploadChunkPool; - FilteredRate& FilteredGenerateBlockBytesPerSecond; - FilteredRate& FilteredCompressedBytesPerSecond; - FilteredRate& FilteredUploadedBytesPerSecond; - std::atomic& UploadedBlockSize; - std::atomic& UploadedBlockCount; - std::atomic& UploadedRawChunkSize; - std::atomic& UploadedCompressedChunkSize; - std::atomic& UploadedChunkCount; - std::atomic& GeneratedBlockCount; - std::atomic& GeneratedBlockByteCount; - std::atomic& QueuedPendingInMemoryBlocksForUpload; - size_t UploadBlockCount; - uint32_t UploadChunkCount; - uint64_t LargeAttachmentSize; - GeneratedBlocks& NewBlocks; - const ChunkedFolderContent& Content; - const ChunkedContentLookup& Lookup; - const std::vector>& NewBlockChunks; - std::span LooseChunkIndexes; - UploadStatistics& TempUploadStats; - LooseChunksStatistics& TempLooseChunksStats; - }; - - void ScheduleBlockGenerationAndUpload(UploadPartBlobsContext& Context, std::span BlockIndexes); - - void ScheduleLooseChunkCompressionAndUpload(UploadPartBlobsContext& Context, std::span LooseChunkOrderIndexes); - - void UploadBlockPayload(UploadPartBlobsContext& Context, size_t BlockIndex, const IoHash& BlockHash, CompositeBuffer Payload); - - void UploadLooseChunkPayload(UploadPartBlobsContext& Context, const IoHash& RawHash, uint64_t RawSize, CompositeBuffer Payload); - - CompositeBuffer CompressChunk(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - uint32_t ChunkIndex, - LooseChunksStatistics& TempLooseChunksStats); - - LoggerRef Log() { return m_Log; } - - LoggerRef m_Log; - ProgressBase& m_Progress; - StorageInstance& m_Storage; - std::atomic& m_AbortFlag; - std::atomic& m_PauseFlag; - WorkerThreadPool& m_IOWorkerPool; - WorkerThreadPool& m_NetworkPool; - const Oid m_BuildId; - - const std::filesystem::path m_Path; - const bool m_CreateBuild; // ?? Member? - const CbObject m_MetaData; // ?? Member - const Options m_Options; - - tsl::robin_set m_NonCompressableExtensionHashes; - - std::future m_PrepBuildResultFuture; - std::vector m_KnownBlocks; - uint64_t m_PreferredMultipartChunkSize = 0; - uint64_t m_LargeAttachmentSize = 0; -}; - -struct ValidateStatistics -{ - uint64_t BuildBlobSize = 0; - uint64_t BuildPartSize = 0; - uint64_t ChunkAttachmentCount = 0; - uint64_t BlockAttachmentCount = 0; - std::atomic VerifiedAttachmentCount = 0; - std::atomic VerifiedByteCount = 0; - uint64_t ElapsedWallTimeUS = 0; -}; - -class BuildsOperationValidateBuildPart -{ -public: - struct Options - { - bool IsQuiet = false; - bool IsVerbose = false; - }; - BuildsOperationValidateBuildPart(LoggerRef Log, - ProgressBase& Progress, - BuildStorageBase& Storage, - std::atomic& AbortFlag, - std::atomic& PauseFlag, - WorkerThreadPool& IOWorkerPool, - WorkerThreadPool& NetworkPool, - const Oid& BuildId, - const Oid& BuildPartId, - const std::string_view BuildPartName, - const Options& Options); - - void Execute(); - - ValidateStatistics m_ValidateStats; - DownloadStatistics m_DownloadStats; - -private: - enum class TaskSteps : uint32_t - { - FetchBuild, - FetchBuildPart, - ValidateBlobs, - Cleanup, - StepCount - }; - - ChunkBlockDescription ValidateChunkBlock(IoBuffer&& Payload, - const IoHash& BlobHash, - uint64_t& OutCompressedSize, - uint64_t& OutDecompressedSize); - - struct ValidateBlobsContext - { - ParallelWork& Work; - uint64_t AttachmentsToVerifyCount; - FilteredRate& FilteredDownloadedBytesPerSecond; - FilteredRate& FilteredVerifiedBytesPerSecond; - }; - - struct ResolvedBuildPart - { - std::vector ChunkAttachments; - std::vector BlockAttachments; - uint64_t PreferredMultipartChunkSize = 0; - }; - - ResolvedBuildPart ResolveBuildPart(); - - void ScheduleChunkAttachmentValidation(ValidateBlobsContext& Context, - std::span ChunkAttachments, - const std::filesystem::path& TempFolder, - uint64_t PreferredMultipartChunkSize); - - void ScheduleBlockAttachmentValidation(ValidateBlobsContext& Context, std::span BlockAttachments); - - void ValidateDownloadedChunk(ValidateBlobsContext& Context, const IoHash& ChunkHash, IoBuffer Payload); - - void ValidateDownloadedBlock(ValidateBlobsContext& Context, const IoHash& BlockAttachment, IoBuffer Payload); - - LoggerRef Log() { return m_Log; } - - LoggerRef m_Log; - ProgressBase& m_Progress; - BuildStorageBase& m_Storage; - std::atomic& m_AbortFlag; - std::atomic& m_PauseFlag; - WorkerThreadPool& m_IOWorkerPool; - WorkerThreadPool& m_NetworkPool; - const Oid m_BuildId; - Oid m_BuildPartId; - const std::string m_BuildPartName; - const Options m_Options; -}; - -class BuildsOperationPrimeCache -{ -public: - struct Options - { - bool IsQuiet = false; - bool IsVerbose = false; - std::filesystem::path ZenFolderPath; - std::uint64_t LargeAttachmentSize = 32u * 1024u * 1024u * 4u; - std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - bool ForceUpload = false; - }; - - BuildsOperationPrimeCache(LoggerRef Log, - ProgressBase& Progress, - StorageInstance& Storage, - std::atomic& AbortFlag, - std::atomic& PauseFlag, - WorkerThreadPool& NetworkPool, - const Oid& BuildId, - std::span BuildPartIds, - const Options& Options, - BuildStorageCache::Statistics& StorageCacheStats); - - void Execute(); - - DownloadStatistics m_DownloadStats; - -private: - LoggerRef Log() { return m_Log; } - - void CollectReferencedBlobs(tsl::robin_set& OutBuildBlobs, - tsl::robin_map& OutLooseChunkRawSizes); - - std::vector FilterAlreadyCachedBlobs(const tsl::robin_set& BuildBlobs); - - void ScheduleBlobDownloads(std::span BlobsToDownload, - const tsl::robin_map& LooseChunkRawSizes, - std::atomic& MultipartAttachmentCount, - std::atomic& CompletedDownloadCount, - FilteredRate& FilteredDownloadedBytesPerSecond); - - void DownloadLargeBlobForCache(ParallelWork& Work, - const IoHash& BlobHash, - size_t BlobCount, - std::atomic& CompletedDownloadCount, - std::atomic& MultipartAttachmentCount, - FilteredRate& FilteredDownloadedBytesPerSecond); - - void DownloadSingleBlobForCache(const IoHash& BlobHash, - size_t BlobCount, - std::atomic& CompletedDownloadCount, - FilteredRate& FilteredDownloadedBytesPerSecond); - - LoggerRef m_Log; - ProgressBase& m_Progress; - StorageInstance& m_Storage; - std::atomic& m_AbortFlag; - std::atomic& m_PauseFlag; - WorkerThreadPool& m_NetworkPool; - const Oid m_BuildId; - std::vector m_BuildPartIds; - Options m_Options; - std::filesystem::path m_TempPath; - - BuildStorageCache::Statistics& m_StorageCacheStats; -}; - -CompositeBuffer ValidateBlob(std::atomic& AbortFlag, - BuildStorageBase& Storage, - const Oid& BuildId, - const IoHash& BlobHash, - uint64_t& OutCompressedSize, - uint64_t& OutDecompressedSize); - -std::vector> ResolveBuildPartNames(CbObjectView BuildObject, - const Oid& BuildId, - const std::vector& BuildPartIds, - std::span BuildPartNames, - std::uint64_t& OutPreferredMultipartChunkSize); - -struct BuildManifest; - -ChunkedFolderContent GetRemoteContent(LoggerRef InLog, - StorageInstance& Storage, - const Oid& BuildId, - const std::vector>& BuildParts, - const BuildManifest& Manifest, - std::span IncludeWildcards, - std::span ExcludeWildcards, - std::unique_ptr& OutChunkController, - std::vector& OutPartContents, - std::vector& OutBlockDescriptions, - std::vector& OutLooseChunkHashes, - bool IsQuiet, - bool IsVerbose, - bool DoExtraContentVerify); - -std::string GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix); - -#if ZEN_WITH_TESTS -void buildstorageoperations_forcelink(); -#endif // ZEN_WITH_TESTS - -} // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageresolve.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageresolve.h new file mode 100644 index 000000000..c964ad6cc --- /dev/null +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageresolve.h @@ -0,0 +1,46 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen { + +struct BuildStorageResolveResult +{ + struct Capabilities + { + uint64_t MaxRangeCountPerRequest = 1; + }; + struct Host + { + std::string Address; + std::string Name; + bool AssumeHttp2 = false; + double LatencySec = -1.0; + Capabilities Caps; + }; + Host Cloud; + Host Cache; +}; + +////////////////////////////////////////////////////////////////////////// + +enum class ZenCacheResolveMode +{ + Off, + Discovery, + LocalHost, + All +}; + +BuildStorageResolveResult ResolveBuildStorage(LoggerRef InLog, + const HttpClientSettings& ClientSettings, + std::string_view Host, + std::string_view OverrideHost, + std::string_view ZenCacheHost, + ZenCacheResolveMode ZenResolveMode, + bool Verbose); + +} // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstoragestats.h b/src/zenremotestore/include/zenremotestore/builds/buildstoragestats.h new file mode 100644 index 000000000..e0de9ed6b --- /dev/null +++ b/src/zenremotestore/include/zenremotestore/builds/buildstoragestats.h @@ -0,0 +1,182 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen { + +struct DiskStatistics +{ + std::atomic OpenReadCount = 0; + std::atomic OpenWriteCount = 0; + std::atomic ReadCount = 0; + std::atomic ReadByteCount = 0; + std::atomic WriteCount = 0; + std::atomic WriteByteCount = 0; + std::atomic CloneCount = 0; + std::atomic CloneByteCount = 0; + std::atomic CurrentOpenFileCount = 0; +}; + +struct DownloadStatistics +{ + std::atomic RequestsCompleteCount = 0; + + std::atomic DownloadedChunkCount = 0; + std::atomic DownloadedChunkByteCount = 0; + std::atomic MultipartAttachmentCount = 0; + + std::atomic DownloadedBlockCount = 0; + std::atomic DownloadedBlockByteCount = 0; + + std::atomic DownloadedPartialBlockCount = 0; + std::atomic DownloadedPartialBlockByteCount = 0; +}; + +struct CacheMappingStatistics +{ + uint64_t CacheChunkCount = 0; + uint64_t CacheChunkByteCount = 0; + + uint64_t CacheBlockCount = 0; + uint64_t CacheBlocksByteCount = 0; + + uint64_t CacheSequenceHashesCount = 0; + uint64_t CacheSequenceHashesByteCount = 0; + + uint64_t CacheScanElapsedWallTimeUs = 0; + + uint32_t LocalPathsMatchingSequencesCount = 0; + uint64_t LocalPathsMatchingSequencesByteCount = 0; + + uint64_t LocalChunkMatchingRemoteCount = 0; + uint64_t LocalChunkMatchingRemoteByteCount = 0; + + uint64_t LocalScanElapsedWallTimeUs = 0; + + uint32_t ScavengedPathsMatchingSequencesCount = 0; + uint64_t ScavengedPathsMatchingSequencesByteCount = 0; + + uint64_t ScavengedChunkMatchingRemoteCount = 0; + uint64_t ScavengedChunkMatchingRemoteByteCount = 0; + + uint64_t ScavengeElapsedWallTimeUs = 0; +}; + +struct WriteChunkStatistics +{ + uint64_t DownloadTimeUs = 0; + uint64_t WriteTimeUs = 0; + uint64_t WriteChunksElapsedWallTimeUs = 0; +}; + +struct RebuildFolderStateStatistics +{ + uint64_t CleanFolderElapsedWallTimeUs = 0; + std::atomic FinalizeTreeFilesMovedCount = 0; + std::atomic FinalizeTreeFilesCopiedCount = 0; + uint64_t FinalizeTreeElapsedWallTimeUs = 0; +}; + +struct FindBlocksStatistics +{ + uint64_t FindBlockTimeMS = 0; + uint64_t PotentialChunkCount = 0; + uint64_t PotentialChunkByteCount = 0; + uint64_t FoundBlockCount = 0; + uint64_t FoundBlockChunkCount = 0; + uint64_t FoundBlockByteCount = 0; + uint64_t AcceptedBlockCount = 0; + uint64_t NewBlocksCount = 0; + uint64_t NewBlocksChunkCount = 0; + uint64_t NewBlocksChunkByteCount = 0; + + FindBlocksStatistics& operator+=(const FindBlocksStatistics& Rhs) + { + FindBlockTimeMS += Rhs.FindBlockTimeMS; + PotentialChunkCount += Rhs.PotentialChunkCount; + PotentialChunkByteCount += Rhs.PotentialChunkByteCount; + FoundBlockCount += Rhs.FoundBlockCount; + FoundBlockChunkCount += Rhs.FoundBlockChunkCount; + FoundBlockByteCount += Rhs.FoundBlockByteCount; + AcceptedBlockCount += Rhs.AcceptedBlockCount; + NewBlocksCount += Rhs.NewBlocksCount; + NewBlocksChunkCount += Rhs.NewBlocksChunkCount; + NewBlocksChunkByteCount += Rhs.NewBlocksChunkByteCount; + return *this; + } +}; + +struct UploadStatistics +{ + std::atomic BlockCount = 0; + std::atomic BlocksBytes = 0; + std::atomic ChunkCount = 0; + std::atomic ChunksBytes = 0; + std::atomic ReadFromDiskBytes = 0; + std::atomic MultipartAttachmentCount = 0; + uint64_t ElapsedWallTimeUS = 0; + + UploadStatistics& operator+=(const UploadStatistics& Rhs) + { + BlockCount += Rhs.BlockCount; + BlocksBytes += Rhs.BlocksBytes; + ChunkCount += Rhs.ChunkCount; + ChunksBytes += Rhs.ChunksBytes; + ReadFromDiskBytes += Rhs.ReadFromDiskBytes; + MultipartAttachmentCount += Rhs.MultipartAttachmentCount; + ElapsedWallTimeUS += Rhs.ElapsedWallTimeUS; + return *this; + } +}; + +struct LooseChunksStatistics +{ + uint64_t ChunkCount = 0; + uint64_t ChunkByteCount = 0; + std::atomic CompressedChunkCount = 0; + std::atomic CompressedChunkRawBytes = 0; + std::atomic CompressedChunkBytes = 0; + uint64_t CompressChunksElapsedWallTimeUS = 0; + + LooseChunksStatistics& operator+=(const LooseChunksStatistics& Rhs) + { + ChunkCount += Rhs.ChunkCount; + ChunkByteCount += Rhs.ChunkByteCount; + CompressedChunkCount += Rhs.CompressedChunkCount; + CompressedChunkRawBytes += Rhs.CompressedChunkRawBytes; + CompressedChunkBytes += Rhs.CompressedChunkBytes; + CompressChunksElapsedWallTimeUS += Rhs.CompressChunksElapsedWallTimeUS; + return *this; + } +}; + +struct GenerateBlocksStatistics +{ + std::atomic GeneratedBlockByteCount = 0; + std::atomic GeneratedBlockCount = 0; + uint64_t GenerateBlocksElapsedWallTimeUS = 0; + + GenerateBlocksStatistics& operator+=(const GenerateBlocksStatistics& Rhs) + { + GeneratedBlockByteCount += Rhs.GeneratedBlockByteCount; + GeneratedBlockCount += Rhs.GeneratedBlockCount; + GenerateBlocksElapsedWallTimeUS += Rhs.GenerateBlocksElapsedWallTimeUS; + return *this; + } +}; + +struct ValidateStatistics +{ + uint64_t BuildBlobSize = 0; + uint64_t BuildPartSize = 0; + uint64_t ChunkAttachmentCount = 0; + uint64_t BlockAttachmentCount = 0; + std::atomic VerifiedAttachmentCount = 0; + std::atomic VerifiedByteCount = 0; + uint64_t ElapsedWallTimeUS = 0; +}; + +} // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h index c55c930bc..340da8487 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h @@ -5,45 +5,30 @@ #include #include #include +#include +#include + +#include +#include +#include +#include namespace zen { -class BuildStorageBase; class BuildStorageCache; +class ParallelWork; +class WorkerThreadPool; +struct ChunkedFolderContent; +struct BuildManifest; +class ChunkingController; -struct BuildStorageResolveResult -{ - struct Capabilities - { - uint64_t MaxRangeCountPerRequest = 1; - }; - struct Host - { - std::string Address; - std::string Name; - bool AssumeHttp2 = false; - double LatencySec = -1.0; - Capabilities Caps; - }; - Host Cloud; - Host Cache; -}; - -enum class ZenCacheResolveMode -{ - Off, - Discovery, - LocalHost, - All -}; +inline const std::string ZenFolderName = ".zen"; +inline const std::string UnsyncFolderName = ".unsync"; +inline const std::string UGSFolderName = ".ugs"; +inline const std::string LegacyZenTempFolderName = ".zen-tmp"; -BuildStorageResolveResult ResolveBuildStorage(LoggerRef InLog, - const HttpClientSettings& ClientSettings, - std::string_view Host, - std::string_view OverrideHost, - std::string_view ZenCacheHost, - ZenCacheResolveMode ZenResolveMode, - bool Verbose); +inline const std::vector DefaultExcludeFolders{UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName}; +inline const std::vector DefaultExcludeExtensions{}; std::vector GetBlockDescriptions(LoggerRef InLog, BuildStorageBase& Storage, @@ -65,4 +50,62 @@ struct StorageInstance std::unique_ptr CacheStorage; }; +std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath); +std::filesystem::path ZenTempFolderPath(const std::filesystem::path& ZenFolderPath); + +CbObject GetBuild(BuildStorageBase& Storage, const Oid& BuildId, bool IsQuiet); + +uint64_t GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory); + +void DownloadLargeBlob(BuildStorageBase& Storage, + const std::filesystem::path& DownloadFolder, + const Oid& BuildId, + const IoHash& ChunkHash, + const std::uint64_t PreferredMultipartChunkSize, + ParallelWork& Work, + WorkerThreadPool& NetworkPool, + std::atomic& DownloadedChunkByteCount, + std::atomic& MultipartAttachmentCount, + std::function&& OnDownloadComplete); + +CompositeBuffer ValidateBlob(std::atomic& AbortFlag, + IoBuffer&& Payload, + const IoHash& BlobHash, + uint64_t& OutCompressedSize, + uint64_t& OutDecompressedSize); + +CompositeBuffer ValidateBlob(std::atomic& AbortFlag, + BuildStorageBase& Storage, + const Oid& BuildId, + const IoHash& BlobHash, + uint64_t& OutCompressedSize, + uint64_t& OutDecompressedSize); + +std::vector> ResolveBuildPartNames(CbObjectView BuildObject, + const Oid& BuildId, + const std::vector& BuildPartIds, + std::span BuildPartNames, + std::uint64_t& OutPreferredMultipartChunkSize); + +ChunkedFolderContent GetRemoteContent(LoggerRef InLog, + StorageInstance& Storage, + const Oid& BuildId, + const std::vector>& BuildParts, + const BuildManifest& Manifest, + std::span IncludeWildcards, + std::span ExcludeWildcards, + std::unique_ptr& OutChunkController, + std::vector& OutPartContents, + std::vector& OutBlockDescriptions, + std::vector& OutLooseChunkHashes, + bool IsQuiet, + bool IsVerbose, + bool DoExtraContentVerify); + +std::string GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix); + +#if ZEN_WITH_TESTS +void buildstorageutil_forcelink(); +#endif // ZEN_WITH_TESTS + } // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/builds/buildupdatefolder.h b/src/zenremotestore/include/zenremotestore/builds/buildupdatefolder.h new file mode 100644 index 000000000..c820f4dcb --- /dev/null +++ b/src/zenremotestore/include/zenremotestore/builds/buildupdatefolder.h @@ -0,0 +1,529 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +class CloneQueryInterface; +class FilteredRate; +class ParallelWork; +class ProgressBase; +class WorkerThreadPool; + +////////////////////////////////////////////////////////////////////////// + +class BuildsOperationUpdateFolder +{ +public: + struct Options + { + bool IsQuiet = false; + bool IsVerbose = false; + bool AllowFileClone = true; + bool UseSparseFiles = true; + std::filesystem::path SystemRootDir; + std::filesystem::path ZenFolderPath; + std::uint64_t LargeAttachmentSize = 32u * 1024u * 1024u * 4u; + std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; + EPartialBlockRequestMode PartialBlockRequestMode = EPartialBlockRequestMode::Mixed; + bool WipeTargetFolder = false; + bool EnableOtherDownloadsScavenging = true; + bool EnableTargetFolderScavenging = true; + bool ValidateCompletedSequences = true; + std::vector ExcludeFolders; + uint64_t MaximumInMemoryPayloadSize = 512u * 1024u; + bool PopulateCache = true; + }; + + BuildsOperationUpdateFolder(LoggerRef Log, + ProgressBase& Progress, + StorageInstance& Storage, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + WorkerThreadPool& IOWorkerPool, + WorkerThreadPool& NetworkPool, + const Oid& BuildId, + const std::filesystem::path& Path, + const ChunkedFolderContent& LocalContent, + const ChunkedContentLookup& LocalLookup, + const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& RemoteLookup, + const std::vector& BlockDescriptions, + const std::vector& LooseChunkHashes, + const Options& Options); + + void Execute(FolderContent& OutLocalFolderState); + + DiskStatistics m_DiskStats; + CacheMappingStatistics m_CacheMappingStats; + GetFolderContentStatistics m_ScavengedFolderScanStats; + DownloadStatistics m_DownloadStats; + WriteChunkStatistics m_WriteChunkStats; + RebuildFolderStateStatistics m_RebuildFolderStateStats; + std::atomic m_WrittenChunkByteCount = 0; + +private: + struct BlockWriteOps + { + std::vector ChunkBuffers; + struct WriteOpData + { + const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr; + size_t ChunkBufferIndex = (size_t)-1; + }; + std::vector WriteOps; + }; + + struct ScavengeSource + { + std::filesystem::path StateFilePath; + std::filesystem::path Path; + }; + + struct ScavengedSequenceCopyOperation + { + uint32_t ScavengedContentIndex = (uint32_t)-1; + uint32_t ScavengedPathIndex = (uint32_t)-1; + uint32_t RemoteSequenceIndex = (uint32_t)-1; + uint64_t RawSize = (uint64_t)-1; + }; + + struct CopyChunkData + { + uint32_t ScavengeSourceIndex = (uint32_t)-1; + uint32_t SourceSequenceIndex = (uint32_t)-1; + std::vector TargetChunkLocationPtrs; + struct ChunkTarget + { + uint32_t TargetChunkLocationCount = (uint32_t)-1; + uint32_t RemoteChunkIndex = (uint32_t)-1; + uint64_t CacheFileOffset = (uint64_t)-1; + }; + std::vector ChunkTargets; + }; + + struct BlobsExistsResult + { + tsl::robin_set ExistingBlobs; + uint64_t ElapsedTimeMs = 0; + }; + + struct LooseChunkHashWorkData + { + std::vector ChunkTargetPtrs; + uint32_t RemoteChunkIndex = (uint32_t)-1; + }; + + struct FinalizeTarget + { + IoHash RawHash; + uint32_t RemotePathIndex; + }; + + struct LocalPathCategorization + { + std::vector FilesToCache; + std::vector RemoveLocalPathIndexes; + tsl::robin_map RemotePathIndexToLocalPathIndex; + tsl::robin_map SequenceHashToLocalPathIndex; + uint64_t MatchCount = 0; + uint64_t PathMismatchCount = 0; + uint64_t HashMismatchCount = 0; + uint64_t SkippedCount = 0; + uint64_t DeleteCount = 0; + }; + + struct WriteChunksContext + { + ParallelWork& Work; + BufferedWriteFileCache& WriteCache; + std::span> SequenceIndexChunksLeftToWriteCounters; + std::span> RemoteChunkIndexNeedsCopyFromSourceFlags; + std::atomic& WritePartsComplete; + uint64_t TotalPartWriteCount; + uint64_t TotalRequestCount; + const BlobsExistsResult& ExistsResult; + FilteredRate& FilteredDownloadedBytesPerSecond; + FilteredRate& FilteredWrittenBytesPerSecond; + }; + + void ScanCacheFolder(tsl::robin_map& OutCachedChunkHashesFound, + tsl::robin_map& OutCachedSequenceHashesFound); + void ScanTempBlocksFolder(tsl::robin_map& OutCachedBlocksFound); + std::vector ScanTargetFolder(const tsl::robin_map& CachedChunkHashesFound, + const tsl::robin_map& CachedSequenceHashesFound); + + std::vector FindScavengeSources(); + + bool FindScavengeContent(const ScavengeSource& Source, + ChunkedFolderContent& OutScavengedLocalContent, + ChunkedContentLookup& OutScavengedLookup); + + void ScavengeSourceForChunks(uint32_t& InOutRemainingChunkCount, + std::vector& InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags, + tsl::robin_map& InOutRawHashToCopyChunkDataIndex, + const std::vector>& SequenceIndexChunksLeftToWriteCounters, + const ChunkedFolderContent& ScavengedContent, + const ChunkedContentLookup& ScavengedLookup, + std::vector& InOutCopyChunkDatas, + uint32_t ScavengedContentIndex, + uint64_t& InOutChunkMatchingRemoteCount, + uint64_t& InOutChunkMatchingRemoteByteCount); + + std::filesystem::path FindDownloadedChunk(const IoHash& ChunkHash); + + std::vector GetRemainingChunkTargets( + std::span> SequenceIndexChunksLeftToWriteCounters, + uint32_t ChunkIndex); + + uint64_t GetChunkWriteCount(std::span> SequenceIndexChunksLeftToWriteCounters, uint32_t ChunkIndex); + + void CheckRequiredDiskSpace(const tsl::robin_map& RemotePathToRemoteIndex); + + void WriteScavengedSequenceToCache(const std::filesystem::path& ScavengeRootPath, + const ChunkedFolderContent& ScavengedContent, + const ScavengedSequenceCopyOperation& ScavengeOp); + + void WriteLooseChunk(const uint32_t RemoteChunkIndex, + const BlobsExistsResult& ExistsResult, + std::span> SequenceIndexChunksLeftToWriteCounters, + std::atomic& WritePartsComplete, + std::vector&& ChunkTargetPtrs, + BufferedWriteFileCache& WriteCache, + ParallelWork& Work, + uint64_t TotalRequestCount, + uint64_t TotalPartWriteCount, + FilteredRate& FilteredDownloadedBytesPerSecond, + FilteredRate& FilteredWrittenBytesPerSecond); + + void DownloadBuildBlob(uint32_t RemoteChunkIndex, + const BlobsExistsResult& ExistsResult, + ParallelWork& Work, + uint64_t TotalRequestCount, + FilteredRate& FilteredDownloadedBytesPerSecond, + std::function&& OnDownloaded); + + void DownloadPartialBlock(std::span BlockRanges, + size_t BlockRangeIndex, + size_t BlockRangeCount, + const BlobsExistsResult& ExistsResult, + uint64_t TotalRequestCount, + FilteredRate& FilteredDownloadedBytesPerSecond, + std::function> OffsetAndLengths)>&& OnDownloaded); + + std::vector WriteLocalChunkToCache(CloneQueryInterface* CloneQuery, + const CopyChunkData& CopyData, + const std::vector& ScavengedContents, + const std::vector& ScavengedLookups, + const std::vector& ScavengedPaths, + BufferedWriteFileCache& WriteCache); + + bool WriteCompressedChunkToCache(const IoHash& ChunkHash, + const std::vector& ChunkTargetPtrs, + BufferedWriteFileCache& WriteCache, + IoBuffer&& CompressedPart); + + void StreamDecompress(const IoHash& SequenceRawHash, CompositeBuffer&& CompressedPart); + + void WriteSequenceChunkToCache(BufferedWriteFileCache::Local& LocalWriter, + const CompositeBuffer& Chunk, + const uint32_t SequenceIndex, + const uint64_t FileOffset, + const uint32_t PathIndex); + + bool GetBlockWriteOps(const IoHash& BlockRawHash, + std::span ChunkRawHashes, + std::span ChunkCompressedLengths, + std::span> SequenceIndexChunksLeftToWriteCounters, + std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, + const MemoryView BlockView, + uint32_t FirstIncludedBlockChunkIndex, + uint32_t LastIncludedBlockChunkIndex, + BlockWriteOps& OutOps); + + void WriteBlockChunkOpsToCache(std::span> SequenceIndexChunksLeftToWriteCounters, + const BlockWriteOps& Ops, + BufferedWriteFileCache& WriteCache, + ParallelWork& Work); + + bool WriteChunksBlockToCache(const ChunkBlockDescription& BlockDescription, + std::span> SequenceIndexChunksLeftToWriteCounters, + ParallelWork& Work, + CompositeBuffer&& BlockBuffer, + std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, + BufferedWriteFileCache& WriteCache); + + bool WritePartialBlockChunksToCache(const ChunkBlockDescription& BlockDescription, + std::span> SequenceIndexChunksLeftToWriteCounters, + ParallelWork& Work, + CompositeBuffer&& PartialBlockBuffer, + uint32_t FirstIncludedBlockChunkIndex, + uint32_t LastIncludedBlockChunkIndex, + std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, + BufferedWriteFileCache& WriteCache); + + void AsyncWriteDownloadedChunk(uint32_t RemoteChunkIndex, + const BlobsExistsResult& ExistsResult, + std::vector&& ChunkTargetPtrs, + BufferedWriteFileCache& WriteCache, + ParallelWork& Work, + IoBuffer&& Payload, + std::span> SequenceIndexChunksLeftToWriteCounters, + std::atomic& WritePartsComplete, + const uint64_t TotalPartWriteCount, + FilteredRate& FilteredWrittenBytesPerSecond); + + void VerifyAndCompleteChunkSequencesAsync(std::span RemoteSequenceIndexes, ParallelWork& Work); + bool CompleteSequenceChunk(uint32_t RemoteSequenceIndex, std::span> SequenceIndexChunksLeftToWriteCounters); + std::vector CompleteChunkTargets(const std::vector& ChunkTargetPtrs, + std::span> SequenceIndexChunksLeftToWriteCounters); + void FinalizeChunkSequence(const IoHash& SequenceRawHash); + void FinalizeChunkSequences(std::span RemoteSequenceIndexes); + void VerifySequence(uint32_t RemoteSequenceIndex); + + void InitializeSequenceCounters(std::vector>& OutSequenceCounters, + tsl::robin_map& OutSequencesLeftToFind, + const tsl::robin_map& CachedChunkHashesFound, + const tsl::robin_map& CachedSequenceHashesFound); + + void MatchScavengedSequencesToRemote(std::span Contents, + std::span Lookups, + std::span Paths, + tsl::robin_map& InOutSequencesLeftToFind, + std::vector>& InOutSequenceCounters, + std::vector& OutCopyOperations, + uint64_t& OutScavengedPathsCount); + + uint64_t CalculateBytesToWriteAndFlagNeededChunks(std::span> SequenceCounters, + const std::vector& NeedsCopyFromLocalFileFlags, + std::span> OutNeedsCopyFromSourceFlags); + + void ClassifyCachedAndFetchBlocks(std::span NeededBlocks, + const tsl::robin_map& CachedBlocksFound, + uint64_t& TotalPartWriteCount, + std::vector& OutCachedChunkBlockIndexes, + std::vector& OutFetchBlockIndexes); + + std::vector DetermineNeededLooseChunkIndexes(std::span> SequenceCounters, + const std::vector& NeedsCopyFromLocalFileFlags, + std::span> NeedsCopyFromSourceFlags); + + BlobsExistsResult QueryBlobCacheExists(std::span NeededLooseChunkIndexes, std::span FetchBlockIndexes); + + std::vector DeterminePartialDownloadModes(const BlobsExistsResult& ExistsResult); + + std::vector BuildLooseChunkHashWorks(std::span NeededLooseChunkIndexes, + std::span> SequenceCounters); + + void VerifyWriteChunksComplete(std::span> SequenceCounters, + uint64_t BytesToWrite, + uint64_t BytesToValidate); + + std::vector BuildSortedFinalizeTargets(); + + void ScanScavengeSources(std::span Sources, + std::vector& OutContents, + std::vector& OutLookups, + std::vector& OutPaths); + + LocalPathCategorization CategorizeLocalPaths(const tsl::robin_map& RemotePathToRemoteIndex); + + void ScheduleLocalFileCaching(std::span FilesToCache, + std::atomic& OutCachedCount, + std::atomic& OutCachedByteCount); + + void ScheduleScavengedSequenceWrites(WriteChunksContext& Context, + std::span CopyOperations, + const std::vector& ScavengedContents, + const std::vector& ScavengedPaths); + + void ScheduleLooseChunkWrites(WriteChunksContext& Context, std::vector& LooseChunkHashWorks); + + void ScheduleLocalChunkCopies(WriteChunksContext& Context, + std::span CopyChunkDatas, + CloneQueryInterface* CloneQuery, + const std::vector& ScavengedContents, + const std::vector& ScavengedLookups, + const std::vector& ScavengedPaths); + + void ScheduleCachedBlockWrites(WriteChunksContext& Context, std::span CachedBlockIndexes); + + void SchedulePartialBlockDownloads(WriteChunksContext& Context, const ChunkBlockAnalyser::BlockResult& PartialBlocks); + + void WritePartialBlockToCache(WriteChunksContext& Context, + size_t BlockRangeStartIndex, + IoBuffer BlockPartialBuffer, + const std::filesystem::path& BlockChunkPath, + std::span> OffsetAndLengths, + const ChunkBlockAnalyser::BlockResult& PartialBlocks); + + void ScheduleFullBlockDownloads(WriteChunksContext& Context, std::span FullBlockIndexes); + + void WriteFullBlockToCache(WriteChunksContext& Context, + uint32_t BlockIndex, + IoBuffer BlockBuffer, + const std::filesystem::path& BlockChunkPath); + + void ScheduleLocalFileRemovals(ParallelWork& Work, + std::span RemoveLocalPathIndexes, + std::atomic& DeletedCount); + + void ScheduleTargetFinalization(ParallelWork& Work, + std::span Targets, + const tsl::robin_map& SequenceHashToLocalPathIndex, + const tsl::robin_map& RemotePathIndexToLocalPathIndex, + FolderContent& OutLocalFolderState, + std::atomic& TargetsComplete); + + void FinalizeTargetGroup(size_t BaseOffset, + size_t Count, + std::span Targets, + const tsl::robin_map& SequenceHashToLocalPathIndex, + const tsl::robin_map& RemotePathIndexToLocalPathIndex, + FolderContent& OutLocalFolderState, + std::atomic& TargetsComplete); + + LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + ProgressBase& m_Progress; + StorageInstance& m_Storage; + std::atomic& m_AbortFlag; + std::atomic& m_PauseFlag; + WorkerThreadPool& m_IOWorkerPool; + WorkerThreadPool& m_NetworkPool; + const Oid m_BuildId; + const std::filesystem::path m_Path; + const ChunkedFolderContent& m_LocalContent; + const ChunkedContentLookup& m_LocalLookup; + const ChunkedFolderContent& m_RemoteContent; + const ChunkedContentLookup& m_RemoteLookup; + const std::vector& m_BlockDescriptions; + const std::vector& m_LooseChunkHashes; + const Options m_Options; + const std::filesystem::path m_CacheFolderPath; + const std::filesystem::path m_TempDownloadFolderPath; + const std::filesystem::path m_TempBlockFolderPath; + + std::atomic m_ValidatedChunkByteCount = 0; +}; + +////////////////////////////////////////////////////////////////////////// + +class TransferThreadWorkers; + +struct VerifyFolderStatistics +{ + std::atomic FilesVerified = 0; + std::atomic FilesFailed = 0; + std::atomic ReadBytes = 0; + uint64_t VerifyElapsedWallTimeUs = 0; +}; + +////////////////////////////////////////////////////////////////////////// + +std::vector GetNewPaths(std::span KnownPaths, + std::span Paths); + +BuildSaveState GetLocalStateFromPaths(ProgressBase& Progress, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + TransferThreadWorkers& Workers, + GetFolderContentStatistics& LocalFolderScanStats, + ChunkingStatistics& ChunkingStats, + const std::filesystem::path& Path, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + std::span PathsToCheck); + +BuildSaveState GetLocalContent(ProgressBase& Progress, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + bool IsQuiet, + TransferThreadWorkers& Workers, + GetFolderContentStatistics& LocalFolderScanStats, + ChunkingStatistics& ChunkingStats, + const std::filesystem::path& Path, + const std::filesystem::path& StateFilePath, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache); + +void VerifyFolder(ProgressBase& Progress, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + TransferThreadWorkers& Workers, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const std::filesystem::path& Path, + const std::vector& ExcludeFolders, + bool VerifyFileHash, + VerifyFolderStatistics& VerifyFolderStats); + +////////////////////////////////////////////////////////////////////////// + +struct DownloadOptions +{ + std::filesystem::path SystemRootDir; + std::filesystem::path ZenFolderPath; + bool AllowMultiparts = true; + EPartialBlockRequestMode PartialBlockRequestMode = EPartialBlockRequestMode::Mixed; + bool CleanTargetFolder = false; + bool PostDownloadVerify = false; + bool EnableOtherDownloadsScavenging = true; + bool EnableTargetFolderScavenging = true; + bool AllowFileClone = true; + std::vector IncludeWildcards; + std::vector ExcludeWildcards; + uint64_t MaximumInMemoryPayloadSize = 512u * 1024u; + bool PopulateCache = true; + bool AppendNewContent = false; + bool IsQuiet = false; + bool IsVerbose = false; + bool UseSparseFiles = false; + bool DoExtraContentVerify = false; + std::vector ExcludeFolders = DefaultExcludeFolders; +}; + +void DownloadFolder(LoggerRef InLog, + ProgressBase& Progress, + TransferThreadWorkers& Workers, + StorageInstance& Storage, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + const BuildStorageCache::Statistics& StorageCacheStats, + const Oid& BuildId, + const std::vector& BuildPartIds, + std::span BuildPartNames, + const std::filesystem::path& DownloadSpecPath, + const std::filesystem::path& Path, + const DownloadOptions& Options); + +} // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/builds/builduploadfolder.h b/src/zenremotestore/include/zenremotestore/builds/builduploadfolder.h new file mode 100644 index 000000000..9ab80955a --- /dev/null +++ b/src/zenremotestore/include/zenremotestore/builds/builduploadfolder.h @@ -0,0 +1,393 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +class FilteredRate; +class ParallelWork; +class ProgressBase; +class ReadFileCache; +class RwLock; +class TransferThreadWorkers; +class WorkerThreadPool; + +static constexpr size_t DefaultMaxChunkBlockSize = 64u * 1024u * 1024u; +static constexpr size_t DefaultMaxChunksPerChunkBlock = 4u * 1000u; +static constexpr size_t DefaultMaxChunkBlockEmbedSize = 3u * 512u * 1024u; + +////////////////////////////////////////////////////////////////////////// + +class BuildsOperationUploadFolder +{ +public: + struct ChunksBlockParameters + { + size_t MaxBlockSize = DefaultMaxChunkBlockSize; + size_t MaxChunksPerBlock = DefaultMaxChunksPerChunkBlock; + size_t MaxChunkEmbedSize = DefaultMaxChunkBlockEmbedSize; + }; + + struct Options + { + bool IsQuiet = false; + bool IsVerbose = false; + bool DoExtraContentValidation = false; + + const uint64_t FindBlockMaxCount = 10000; + const uint8_t BlockReuseMinPercentLimit = 85; + bool AllowMultiparts = true; + bool IgnoreExistingBlocks = false; + ChunksBlockParameters BlockParameters; + + uint32_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; + + const uint64_t MinimumSizeForCompressInBlock = 2u * 1024u; + + std::filesystem::path TempDir; + std::vector ExcludeFolders; + std::vector ExcludeExtensions; + std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; + + std::vector NonCompressableExtensions; + + bool PopulateCache = true; + }; + BuildsOperationUploadFolder(LoggerRef Log, + ProgressBase& Progress, + StorageInstance& Storage, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + WorkerThreadPool& IOWorkerPool, + WorkerThreadPool& NetworkPool, + const Oid& BuildId, + const std::filesystem::path& Path, + bool CreateBuild, + const CbObject& MetaData, + const Options& Options); + + std::vector> Execute(const Oid& BuildPartId, + const std::string_view BuildPartName, + const std::filesystem::path& ManifestPath, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache); + + DiskStatistics m_DiskStats; + GetFolderContentStatistics m_LocalFolderScanStats; + ChunkingStatistics m_ChunkingStats; + FindBlocksStatistics m_FindBlocksStats; + ReuseBlocksStatistics m_ReuseBlocksStats; + UploadStatistics m_UploadStats; + GenerateBlocksStatistics m_GenerateBlocksStats; + LooseChunksStatistics m_LooseChunksStats; + +private: + struct PrepareBuildResult + { + std::vector KnownBlocks; + uint64_t PreferredMultipartChunkSize = 0; + uint64_t PayloadSize = 0; + uint64_t PrepareBuildTimeMs = 0; + uint64_t FindBlocksTimeMs = 0; + uint64_t ElapsedTimeMs = 0; + }; + + PrepareBuildResult PrepareBuild(); + + struct UploadPart + { + Oid PartId = Oid::Zero; + std::string PartName; + FolderContent Content; + uint64_t TotalRawSize = 0; + GetFolderContentStatistics LocalFolderScanStats; + }; + + std::vector ReadFolder(); + std::vector ReadManifestParts(const std::filesystem::path& ManifestPath); + + bool IsAcceptedFolder(const std::string_view& RelativePath) const; + bool IsAcceptedFile(const std::string_view& RelativePath) const; + + void ArrangeChunksIntoBlocks(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + std::vector& ChunkIndexes, + std::vector>& OutBlocks); + struct GeneratedBlocks + { + std::vector BlockDescriptions; + std::vector BlockSizes; + std::vector BlockHeaders; + std::vector BlockMetaDatas; + std::vector + MetaDataHasBeenUploaded; // NOTE: Do not use std::vector here as this vector is modified by multiple threads + tsl::robin_map BlockHashToBlockIndex; + }; + + void GenerateBuildBlocks(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const std::vector>& NewBlockChunks, + GeneratedBlocks& OutBlocks, + GenerateBlocksStatistics& GenerateBlocksStats, + UploadStatistics& UploadStats); + + struct GenerateBuildBlocksContext + { + ParallelWork& Work; + WorkerThreadPool& GenerateBlobsPool; + WorkerThreadPool& UploadBlocksPool; + FilteredRate& FilteredGeneratedBytesPerSecond; + FilteredRate& FilteredUploadedBytesPerSecond; + std::atomic& QueuedPendingBlocksForUpload; + RwLock& Lock; + GeneratedBlocks& OutBlocks; + GenerateBlocksStatistics& GenerateBlocksStats; + UploadStatistics& UploadStats; + size_t NewBlockCount; + }; + + void ScheduleBlockGeneration(GenerateBuildBlocksContext& Context, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const std::vector>& NewBlockChunks); + + void UploadGeneratedBlock(GenerateBuildBlocksContext& Context, size_t BlockIndex, CompressedBuffer Payload); + + std::vector CalculateAbsoluteChunkOrders(const std::span LocalChunkHashes, + const std::span LocalChunkOrder, + const tsl::robin_map& ChunkHashToLocalChunkIndex, + const std::span& LooseChunkIndexes, + const std::span& BlockDescriptions); + + CompositeBuffer FetchChunk(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const IoHash& ChunkHash, + ReadFileCache& OpenFileCache); + + CompressedBuffer GenerateBlock(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const std::vector& ChunksInBlock, + ChunkBlockDescription& OutBlockDescription); + + CompressedBuffer RebuildBlock(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + CompositeBuffer&& HeaderBuffer, + const std::vector& ChunksInBlock); + + enum class PartTaskSteps : uint32_t + { + ChunkPartContent = 0, + CalculateDelta, + GenerateBlocks, + BuildPartManifest, + UploadBuildPart, + UploadAttachments, + PutBuildPartStats, + StepCount + }; + + void UploadBuildPart(ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + uint32_t PartIndex, + const UploadPart& Part, + uint32_t PartStepOffset, + uint32_t StepCount); + + ChunkedFolderContent ScanPartContent(const UploadPart& Part, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + ChunkingStatistics& ChunkingStats); + + void ConsumePrepareBuildResult(); + + void ClassifyChunksByBlockEligibility(const ChunkedFolderContent& LocalContent, + std::vector& OutLooseChunkIndexes, + std::vector& OutNewBlockChunkIndexes, + std::vector& OutReuseBlockIndexes, + LooseChunksStatistics& LooseChunksStats, + FindBlocksStatistics& FindBlocksStats, + ReuseBlocksStatistics& ReuseBlocksStats); + + struct BuiltPartManifest + { + CbObject PartManifest; + std::vector AllChunkBlockDescriptions; + std::vector AllChunkBlockHashes; + }; + + BuiltPartManifest BuildPartManifestObject(const ChunkedFolderContent& LocalContent, + const ChunkedContentLookup& LocalLookup, + ChunkingController& ChunkController, + std::span ReuseBlockIndexes, + const GeneratedBlocks& NewBlocks, + std::span LooseChunkIndexes); + + void UploadAttachmentBatch(std::span RawHashes, + std::vector& OutUnknownChunks, + const ChunkedFolderContent& LocalContent, + const ChunkedContentLookup& LocalLookup, + const std::vector>& NewBlockChunks, + GeneratedBlocks& NewBlocks, + std::span LooseChunkIndexes, + UploadStatistics& UploadStats, + LooseChunksStatistics& LooseChunksStats); + + void FinalizeBuildPartWithRetries(const UploadPart& Part, + const IoHash& PartHash, + std::vector& InOutUnknownChunks, + const ChunkedFolderContent& LocalContent, + const ChunkedContentLookup& LocalLookup, + const std::vector>& NewBlockChunks, + GeneratedBlocks& NewBlocks, + std::span LooseChunkIndexes, + UploadStatistics& UploadStats, + LooseChunksStatistics& LooseChunksStats); + + void UploadMissingBlockMetadata(GeneratedBlocks& NewBlocks, UploadStatistics& UploadStats); + + void UploadPartBlobs(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + std::span RawHashes, + const std::vector>& NewBlockChunks, + GeneratedBlocks& NewBlocks, + std::span LooseChunkIndexes, + const std::uint64_t LargeAttachmentSize, + UploadStatistics& TempUploadStats, + LooseChunksStatistics& TempLooseChunksStats, + std::vector& OutUnknownChunks); + + struct UploadPartClassification + { + std::vector BlockIndexes; + std::vector LooseChunkOrderIndexes; + uint64_t TotalBlocksSize = 0; + uint64_t TotalLooseChunksSize = 0; + }; + + UploadPartClassification ClassifyUploadRawHashes(std::span RawHashes, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const GeneratedBlocks& NewBlocks, + std::span LooseChunkIndexes, + std::vector& OutUnknownChunks); + + struct UploadPartBlobsContext + { + ParallelWork& Work; + WorkerThreadPool& ReadChunkPool; + WorkerThreadPool& UploadChunkPool; + FilteredRate& FilteredGenerateBlockBytesPerSecond; + FilteredRate& FilteredCompressedBytesPerSecond; + FilteredRate& FilteredUploadedBytesPerSecond; + std::atomic& UploadedBlockSize; + std::atomic& UploadedBlockCount; + std::atomic& UploadedRawChunkSize; + std::atomic& UploadedCompressedChunkSize; + std::atomic& UploadedChunkCount; + std::atomic& GeneratedBlockCount; + std::atomic& GeneratedBlockByteCount; + std::atomic& QueuedPendingInMemoryBlocksForUpload; + size_t UploadBlockCount; + uint32_t UploadChunkCount; + uint64_t LargeAttachmentSize; + GeneratedBlocks& NewBlocks; + const ChunkedFolderContent& Content; + const ChunkedContentLookup& Lookup; + const std::vector>& NewBlockChunks; + std::span LooseChunkIndexes; + UploadStatistics& TempUploadStats; + LooseChunksStatistics& TempLooseChunksStats; + }; + + void ScheduleBlockGenerationAndUpload(UploadPartBlobsContext& Context, std::span BlockIndexes); + + void ScheduleLooseChunkCompressionAndUpload(UploadPartBlobsContext& Context, std::span LooseChunkOrderIndexes); + + void UploadBlockPayload(UploadPartBlobsContext& Context, size_t BlockIndex, const IoHash& BlockHash, CompositeBuffer Payload); + + void UploadLooseChunkPayload(UploadPartBlobsContext& Context, const IoHash& RawHash, uint64_t RawSize, CompositeBuffer Payload); + + CompositeBuffer CompressChunk(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + uint32_t ChunkIndex, + LooseChunksStatistics& TempLooseChunksStats); + + LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + ProgressBase& m_Progress; + StorageInstance& m_Storage; + std::atomic& m_AbortFlag; + std::atomic& m_PauseFlag; + WorkerThreadPool& m_IOWorkerPool; + WorkerThreadPool& m_NetworkPool; + const Oid m_BuildId; + + const std::filesystem::path m_Path; + const bool m_CreateBuild; + const CbObject m_MetaData; + const Options m_Options; + + tsl::robin_set m_NonCompressableExtensionHashes; + + std::future m_PrepBuildResultFuture; + std::vector m_KnownBlocks; + uint64_t m_PreferredMultipartChunkSize = 0; + uint64_t m_LargeAttachmentSize = 0; +}; + +////////////////////////////////////////////////////////////////////////// + +struct UploadFolderOptions +{ + std::filesystem::path TempDir; + uint64_t FindBlockMaxCount; + uint8_t BlockReuseMinPercentLimit; + bool AllowMultiparts; + bool CreateBuild; + bool IgnoreExistingBlocks; + bool UploadToZenCache; + bool IsQuiet = false; + bool IsVerbose = false; + bool DoExtraContentVerify = false; + const std::vector& ExcludeFolders = DefaultExcludeFolders; + const std::vector& ExcludeExtensions = DefaultExcludeExtensions; +}; + +std::vector> UploadFolder(LoggerRef Log, + ProgressBase& Progress, + TransferThreadWorkers& Workers, + StorageInstance& Storage, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + const Oid& BuildId, + const Oid& BuildPartId, + std::string_view BuildPartName, + const std::filesystem::path& Path, + const std::filesystem::path& ManifestPath, + const CbObject& MetaData, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + const UploadFolderOptions& Options); + +} // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/builds/buildvalidatebuildpart.h b/src/zenremotestore/include/zenremotestore/builds/buildvalidatebuildpart.h new file mode 100644 index 000000000..dcb60bca9 --- /dev/null +++ b/src/zenremotestore/include/zenremotestore/builds/buildvalidatebuildpart.h @@ -0,0 +1,121 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace zen { + +class BuildStorageBase; +class FilteredRate; +class ParallelWork; +class ProgressBase; +class TransferThreadWorkers; +class WorkerThreadPool; + +////////////////////////////////////////////////////////////////////////// + +class BuildsOperationValidateBuildPart +{ +public: + struct Options + { + bool IsQuiet = false; + bool IsVerbose = false; + }; + BuildsOperationValidateBuildPart(LoggerRef Log, + ProgressBase& Progress, + BuildStorageBase& Storage, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + WorkerThreadPool& IOWorkerPool, + WorkerThreadPool& NetworkPool, + const Oid& BuildId, + const Oid& BuildPartId, + const std::string_view BuildPartName, + const Options& Options); + + void Execute(); + + ValidateStatistics m_ValidateStats; + DownloadStatistics m_DownloadStats; + +private: + enum class TaskSteps : uint32_t + { + FetchBuild, + FetchBuildPart, + ValidateBlobs, + Cleanup, + StepCount + }; + + ChunkBlockDescription ValidateChunkBlock(IoBuffer&& Payload, + const IoHash& BlobHash, + uint64_t& OutCompressedSize, + uint64_t& OutDecompressedSize); + + struct ValidateBlobsContext + { + ParallelWork& Work; + uint64_t AttachmentsToVerifyCount; + FilteredRate& FilteredDownloadedBytesPerSecond; + FilteredRate& FilteredVerifiedBytesPerSecond; + }; + + struct ResolvedBuildPart + { + std::vector ChunkAttachments; + std::vector BlockAttachments; + uint64_t PreferredMultipartChunkSize = 0; + }; + + ResolvedBuildPart ResolveBuildPart(); + + void ScheduleChunkAttachmentValidation(ValidateBlobsContext& Context, + std::span ChunkAttachments, + const std::filesystem::path& TempFolder, + uint64_t PreferredMultipartChunkSize); + + void ScheduleBlockAttachmentValidation(ValidateBlobsContext& Context, std::span BlockAttachments); + + void ValidateDownloadedChunk(ValidateBlobsContext& Context, const IoHash& ChunkHash, IoBuffer Payload); + + void ValidateDownloadedBlock(ValidateBlobsContext& Context, const IoHash& BlockAttachment, IoBuffer Payload); + + LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + ProgressBase& m_Progress; + BuildStorageBase& m_Storage; + std::atomic& m_AbortFlag; + std::atomic& m_PauseFlag; + WorkerThreadPool& m_IOWorkerPool; + WorkerThreadPool& m_NetworkPool; + const Oid m_BuildId; + Oid m_BuildPartId; + const std::string m_BuildPartName; + const Options m_Options; +}; + +////////////////////////////////////////////////////////////////////////// + +void ValidateBuildPart(LoggerRef Log, + ProgressBase& Progress, + std::atomic& AbortFlag, + std::atomic& PauseFlag, + bool IsQuiet, + bool IsVerbose, + TransferThreadWorkers& Workers, + BuildStorageBase& Storage, + const Oid& BuildId, + const Oid& BuildPartId, + std::string_view BuildPartName); + +} // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h index f44381e42..f374211f2 100644 --- a/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h +++ b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h @@ -3,7 +3,6 @@ #pragma once #include -#include #include #include diff --git a/src/zenremotestore/include/zenremotestore/transferthreadworkers.h b/src/zenremotestore/include/zenremotestore/transferthreadworkers.h index a7faacfd5..6b6584614 100644 --- a/src/zenremotestore/include/zenremotestore/transferthreadworkers.h +++ b/src/zenremotestore/include/zenremotestore/transferthreadworkers.h @@ -3,7 +3,6 @@ #pragma once #include -#include #include #include diff --git a/src/zenremotestore/zenremotestore.cpp b/src/zenremotestore/zenremotestore.cpp index 9642f8470..0b579ebd8 100644 --- a/src/zenremotestore/zenremotestore.cpp +++ b/src/zenremotestore/zenremotestore.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include #include @@ -21,7 +21,7 @@ zenremotestore_forcelinktests() buildmanifest_forcelink(); buildsavedstate_forcelink(); jupiterbuildstorage_forcelink(); - buildstorageoperations_forcelink(); + buildstorageutil_forcelink(); chunkblock_forcelink(); chunkedcontent_forcelink(); chunkedfile_forcelink(); diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp index 2a6c62195..9844d02f0 100644 --- a/src/zenserver/storage/projectstore/httpprojectstore.cpp +++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -633,11 +634,6 @@ namespace { return Result; } - static uint64_t GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory) - { - return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u; - } - } // namespace ////////////////////////////////////////////////////////////////////////// diff --git a/src/zenutil/filteredrate.cpp b/src/zenutil/filteredrate.cpp new file mode 100644 index 000000000..de01af57b --- /dev/null +++ b/src/zenutil/filteredrate.cpp @@ -0,0 +1,92 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +namespace zen { + +void +FilteredRate::Start() +{ + if (StartTimeUS == (uint64_t)-1) + { + uint64_t Expected = (uint64_t)-1; + if (StartTimeUS.compare_exchange_strong(Expected, Timer.GetElapsedTimeUs())) + { + LastTimeUS = StartTimeUS.load(); + } + } +} + +void +FilteredRate::Stop() +{ + if (EndTimeUS == (uint64_t)-1) + { + uint64_t Expected = (uint64_t)-1; + EndTimeUS.compare_exchange_strong(Expected, Timer.GetElapsedTimeUs()); + } +} + +void +FilteredRate::Update(uint64_t Count) +{ + if (LastTimeUS == (uint64_t)-1) + { + return; + } + uint64_t TimeUS = Timer.GetElapsedTimeUs(); + uint64_t TimeDeltaUS = TimeUS - LastTimeUS; + if (TimeDeltaUS >= 2000000) + { + uint64_t Delta = Count - LastCount; + uint64_t PerSecond = (Delta * 1000000) / TimeDeltaUS; + + FilteredPerSecond = (PerSecond + (LastPerSecond * 7)) / 8; + + LastPerSecond = PerSecond; + LastCount = Count; + LastTimeUS = TimeUS; + } +} + +uint64_t +FilteredRate::GetCurrent() const +{ + if (LastTimeUS == (uint64_t)-1) + { + return 0; + } + return FilteredPerSecond; +} + +uint64_t +FilteredRate::GetElapsedTimeUS() const +{ + if (StartTimeUS == (uint64_t)-1) + { + return 0; + } + if (EndTimeUS == (uint64_t)-1) + { + return 0; + } + return EndTimeUS - StartTimeUS; +} + +bool +FilteredRate::IsActive() const +{ + return (StartTimeUS != (uint64_t)-1) && (EndTimeUS == (uint64_t)-1); +} + +uint64_t +GetBytesPerSecond(uint64_t ElapsedWallTimeUS, uint64_t Count) +{ + if (ElapsedWallTimeUS == 0) + { + return 0; + } + return Count * 1000000 / ElapsedWallTimeUS; +} + +} // namespace zen diff --git a/src/zenutil/include/zenutil/filteredrate.h b/src/zenutil/include/zenutil/filteredrate.h new file mode 100644 index 000000000..3349823d0 --- /dev/null +++ b/src/zenutil/include/zenutil/filteredrate.h @@ -0,0 +1,37 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include + +namespace zen { + +class FilteredRate +{ +public: + FilteredRate() {} + + void Start(); + void Stop(); + void Update(uint64_t Count); + + uint64_t GetCurrent() const; + uint64_t GetElapsedTimeUS() const; + bool IsActive() const; + +private: + Stopwatch Timer; + std::atomic StartTimeUS = (uint64_t)-1; + std::atomic EndTimeUS = (uint64_t)-1; + std::atomic LastTimeUS = (uint64_t)-1; + uint64_t LastCount = 0; + uint64_t LastPerSecond = 0; + uint64_t FilteredPerSecond = 0; +}; + +uint64_t GetBytesPerSecond(uint64_t ElapsedWallTimeUS, uint64_t Count); + +} // namespace zen diff --git a/src/zenutil/include/zenutil/progress.h b/src/zenutil/include/zenutil/progress.h index 6a137ae9c..4103723b3 100644 --- a/src/zenutil/include/zenutil/progress.h +++ b/src/zenutil/include/zenutil/progress.h @@ -16,6 +16,8 @@ public: virtual void SetLogOperationName(std::string_view Name) = 0; virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) = 0; + virtual void PushLogOperation(std::string_view Name) = 0; + virtual void PopLogOperation() = 0; virtual uint32_t GetProgressUpdateDelayMS() const = 0; class ProgressBar @@ -54,6 +56,7 @@ public: virtual ~ProgressBar() = default; virtual void UpdateState(const State& NewState, bool DoLinebreak) = 0; + virtual void ForceLinebreak() = 0; virtual void Finish() = 0; }; diff --git a/src/zenutil/progress.cpp b/src/zenutil/progress.cpp index a12076dcc..01f6529f2 100644 --- a/src/zenutil/progress.cpp +++ b/src/zenutil/progress.cpp @@ -14,6 +14,7 @@ public: StandardProgressBar(StandardProgressBase& Owner, std::string_view InSubTask) : m_Owner(Owner), m_SubTask(InSubTask) {} virtual void UpdateState(const State& NewState, bool DoLinebreak) override; + virtual void ForceLinebreak() override {} virtual void Finish() override; private: @@ -38,6 +39,8 @@ public: const size_t PercentDone = StepCount > 0u ? (100u * StepIndex) / StepCount : 0u; ZEN_INFO("{}: {}%", m_LogOperationName, PercentDone); } + virtual void PushLogOperation(std::string_view Name) override { ZEN_UNUSED(Name); } + virtual void PopLogOperation() override {} virtual uint32_t GetProgressUpdateDelayMS() const override { return 2000; } virtual std::unique_ptr CreateProgressBar(std::string_view InSubTask) override { -- cgit v1.2.3