diff options
Diffstat (limited to 'src/zen/cmds/builds_cmd.cpp')
| -rw-r--r-- | src/zen/cmds/builds_cmd.cpp | 3576 |
1 files changed, 889 insertions, 2687 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 3373506f2..e9b443dcb 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -4,68 +4,30 @@ #include <zencore/basicfile.h> #include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinaryfile.h> -#include <zencore/compactbinaryfmt.h> -#include <zencore/compactbinaryvalue.h> -#include <zencore/compress.h> -#include <zencore/except.h> #include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/logging.h> #include <zencore/parallelwork.h> +#include <zencore/process.h> #include <zencore/scopeguard.h> #include <zencore/session.h> -#include <zencore/stream.h> #include <zencore/string.h> -#include <zencore/trace.h> -#include <zencore/uid.h> +#include <zenhttp/auth/authmgr.h> #include <zenhttp/formatters.h> -#include <zenhttp/httpclient.h> -#include <zenhttp/httpclientauth.h> #include <zenhttp/httpcommon.h> -#include <zenremotestore/builds/buildcontent.h> -#include <zenremotestore/builds/buildmanifest.h> -#include <zenremotestore/builds/buildsavedstate.h> -#include <zenremotestore/builds/buildstoragecache.h> -#include <zenremotestore/builds/buildstorageoperations.h> -#include <zenremotestore/builds/buildstorageutil.h> +#include <zenremotestore/builds/buildinspect.h> +#include <zenremotestore/builds/buildprimecache.h> +#include <zenremotestore/builds/buildupdatefolder.h> +#include <zenremotestore/builds/builduploadfolder.h> +#include <zenremotestore/builds/buildvalidatebuildpart.h> #include <zenremotestore/builds/filebuildstorage.h> #include <zenremotestore/builds/jupiterbuildstorage.h> -#include <zenremotestore/chunking/chunkblock.h> -#include <zenremotestore/chunking/chunkedcontent.h> -#include <zenremotestore/chunking/chunkedfile.h> #include <zenremotestore/chunking/chunkingcache.h> #include <zenremotestore/chunking/chunkingcontroller.h> -#include <zenremotestore/jupiter/jupiterhost.h> -#include <zenremotestore/transferthreadworkers.h> #include <zenutil/filesystemutils.h> #include <zenutil/progress.h> -#include <zenutil/wildcard.h> #include <zenutil/workerpools.h> -#include <zenutil/zenserverprocess.h> - -#include "../progressbar.h" #include <signal.h> #include <memory> -#include <numeric> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <tsl/robin_map.h> -#include <tsl/robin_set.h> -#include <json11.hpp> -ZEN_THIRD_PARTY_INCLUDES_END - -#if ZEN_PLATFORM_WINDOWS -# include <zencore/windows.h> -#else -# include <fcntl.h> -# include <sys/file.h> -# include <sys/stat.h> -# include <unistd.h> -#endif - -static const bool DoExtraContentVerify = false; namespace zen { @@ -115,14 +77,6 @@ namespace builds_impl { } }; - struct MemMap - { - void* Handle = nullptr; - void* Data = nullptr; - size_t Size = 0; - std::string Name; - }; - class ZenState { public: @@ -236,11 +190,6 @@ namespace builds_impl { } } - const std::string ZenFolderName = ".zen"; - std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.cbo"; } - // std::filesystem::path ZenStateFileJsonPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.json"; - // } - std::filesystem::path UploadTempDirectory(const std::filesystem::path& Path) { const std::u8string LocalPathString = Path.generic_u8string(); @@ -248,1828 +197,103 @@ namespace builds_impl { return std::filesystem::temp_directory_path() / fmt::format("zen_{}", PathHash); } - const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; - - const std::string UnsyncFolderName = ".unsync"; - - const std::string UGSFolderName = ".ugs"; - const std::string LegacyZenTempFolderName = ".zen-tmp"; - - const std::vector<std::string> DefaultExcludeFolders({UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName}); - const std::vector<std::string> DefaultExcludeExtensions({}); - + // Debugging knobs for file build storage - always 0 in shipped builds const double DefaultLatency = 0; // .0010; const double DefaultDelayPerKBSec = 0; // 0.00005; - const bool SingleThreaded = false; - bool UseSparseFiles = false; - - static bool IsVerbose = false; - static bool IsQuiet = false; - static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; - -#undef ZEN_CONSOLE_VERBOSE -#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ - if (IsVerbose) \ - { \ - ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \ - } - - const std::string DefaultAccessTokenEnvVariableName( -#if ZEN_PLATFORM_WINDOWS - "UE-CloudDataCacheAccessToken"sv -#endif -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - "UE_CloudDataCacheAccessToken"sv -#endif - - ); - - static uint64_t GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory) - { - return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u; - } - - class FilteredRate - { - public: - FilteredRate() {} - - void Start() - { - if (StartTimeUS == (uint64_t)-1) - { - uint64_t Expected = (uint64_t)-1; - if (StartTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs())) - { - LastTimeUS = StartTimeUS.load(); - } - } - } - void Stop() - { - if (EndTimeUS == (uint64_t)-1) - { - uint64_t Expected = (uint64_t)-1; - EndTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs()); - } - } - - void Update(uint64_t Count) - { - if (LastTimeUS == (uint64_t)-1) - { - return; - } - uint64_t TimeUS = Timer.GetElapsedTimeUs(); - uint64_t TimeDeltaUS = TimeUS - LastTimeUS; - if (TimeDeltaUS >= 2000000) - { - uint64_t Delta = Count - LastCount; - uint64_t PerSecond = (Delta * 1000000) / TimeDeltaUS; - - LastPerSecond = PerSecond; - - LastCount = Count; - - FilteredPerSecond = (PerSecond + (LastPerSecond * 7)) / 8; - - LastTimeUS = TimeUS; - } - } - - uint64_t GetCurrent() const // If Stopped - return total count / total time - { - if (LastTimeUS == (uint64_t)-1) - { - return 0; - } - return FilteredPerSecond; - } - - uint64_t GetElapsedTimeUS() const - { - if (StartTimeUS == (uint64_t)-1) - { - return 0; - } - if (EndTimeUS == (uint64_t)-1) - { - return 0; - } - uint64_t TimeDeltaUS = EndTimeUS - StartTimeUS; - return TimeDeltaUS; - } - - bool IsActive() const { return (StartTimeUS != (uint64_t)-1) && (EndTimeUS == (uint64_t)-1); } - - private: - Stopwatch Timer; - std::atomic<uint64_t> StartTimeUS = (uint64_t)-1; - std::atomic<uint64_t> EndTimeUS = (uint64_t)-1; - std::atomic<uint64_t> LastTimeUS = (uint64_t)-1; - uint64_t LastCount = 0; - uint64_t LastPerSecond = 0; - uint64_t FilteredPerSecond = 0; - }; - - uint64_t GetBytesPerSecond(uint64_t ElapsedWallTimeUS, uint64_t Count) - { - if (ElapsedWallTimeUS == 0) - { - return 0; - } - return Count * 1000000 / ElapsedWallTimeUS; - } - - bool CleanAndRemoveDirectory(WorkerThreadPool& WorkerPool, const std::filesystem::path& Directory) - { - return CleanAndRemoveDirectory(WorkerPool, AbortFlag, PauseFlag, Directory); - } - - void ValidateBuildPart(LoggerRef Log, - ProgressBase& Progress, - TransferThreadWorkers& Workers, - BuildStorageBase& Storage, - const Oid& BuildId, - Oid BuildPartId, - const std::string_view BuildPartName) - { - ZEN_TRACE_CPU("ValidateBuildPart"); - - ProgressBar::SetLogOperationName(ProgressMode, "Validate Part"); - - BuildsOperationValidateBuildPart ValidateOp(Log, - Progress, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - BuildPartId, - BuildPartName, - BuildsOperationValidateBuildPart::Options{.IsQuiet = IsQuiet, .IsVerbose = IsVerbose}); - - ValidateOp.Execute(); - - const uint64_t DownloadedCount = ValidateOp.m_DownloadStats.DownloadedChunkCount + ValidateOp.m_DownloadStats.DownloadedBlockCount; - const uint64_t DownloadedByteCount = - ValidateOp.m_DownloadStats.DownloadedChunkByteCount + ValidateOp.m_DownloadStats.DownloadedBlockByteCount; - ZEN_CONSOLE("Verified: {:>8} ({}), {}B/sec, {}", - DownloadedCount, - NiceBytes(DownloadedByteCount), - NiceNum(GetBytesPerSecond(ValidateOp.m_ValidateStats.ElapsedWallTimeUS, DownloadedByteCount)), - NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000)); - } - - struct UploadFolderOptions - { - std::filesystem::path TempDir; - uint64_t FindBlockMaxCount; - uint8_t BlockReuseMinPercentLimit; - bool AllowMultiparts; - bool CreateBuild; - bool IgnoreExistingBlocks; - bool UploadToZenCache; - const std::vector<std::string>& ExcludeFolders = DefaultExcludeFolders; - const std::vector<std::string>& ExcludeExtensions = DefaultExcludeExtensions; - }; - - std::vector<std::pair<Oid, std::string>> UploadFolder(LoggerRef Log, - ProgressBase& Progress, - TransferThreadWorkers& Workers, - StorageInstance& Storage, - const Oid& BuildId, - const Oid& BuildPartId, - const std::string_view BuildPartName, - const std::filesystem::path& Path, - const std::filesystem::path& ManifestPath, - const CbObject& MetaData, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - const UploadFolderOptions& Options) - { - ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder"); - - Stopwatch UploadTimer; - - BuildsOperationUploadFolder UploadOp( - Log, - Progress, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - Path, - Options.CreateBuild, - std::move(MetaData), - BuildsOperationUploadFolder::Options{.IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .DoExtraContentValidation = DoExtraContentVerify, - .FindBlockMaxCount = Options.FindBlockMaxCount, - .BlockReuseMinPercentLimit = Options.BlockReuseMinPercentLimit, - .AllowMultiparts = Options.AllowMultiparts, - .IgnoreExistingBlocks = Options.IgnoreExistingBlocks, - .TempDir = Options.TempDir, - .ExcludeFolders = Options.ExcludeFolders, - .ExcludeExtensions = Options.ExcludeExtensions, - .ZenExcludeManifestName = ZenExcludeManifestName, - .NonCompressableExtensions = DefaultSplitOnlyExtensions, - .PopulateCache = Options.UploadToZenCache}); - - std::vector<std::pair<Oid, std::string>> UploadedParts = - UploadOp.Execute(BuildPartId, BuildPartName, ManifestPath, ChunkController, ChunkCache); - if (AbortFlag) - { - return {}; - } - - ZEN_CONSOLE_VERBOSE( - "Folder scanning stats:" - "\n FoundFileCount: {}" - "\n FoundFileByteCount: {}" - "\n AcceptedFileCount: {}" - "\n AcceptedFileByteCount: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), - UploadOp.m_LocalFolderScanStats.AcceptedFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.AcceptedFileByteCount.load()), - NiceLatencyNs(UploadOp.m_LocalFolderScanStats.ElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Chunking stats:" - "\n FilesProcessed: {}" - "\n FilesChunked: {}" - "\n BytesHashed: {}" - "\n UniqueChunksFound: {}" - "\n UniqueSequencesFound: {}" - "\n UniqueBytesFound: {}" - "\n FilesFoundInCache: {}" - "\n ChunksFoundInCache: {}" - "\n FilesStoredInCache: {}" - "\n ChunksStoredInCache: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_ChunkingStats.FilesProcessed.load(), - UploadOp.m_ChunkingStats.FilesChunked.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesHashed.load()), - UploadOp.m_ChunkingStats.UniqueChunksFound.load(), - UploadOp.m_ChunkingStats.UniqueSequencesFound.load(), - NiceBytes(UploadOp.m_ChunkingStats.UniqueBytesFound.load()), - UploadOp.m_ChunkingStats.FilesFoundInCache.load(), - UploadOp.m_ChunkingStats.ChunksFoundInCache.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesFoundInCache.load()), - UploadOp.m_ChunkingStats.FilesStoredInCache.load(), - UploadOp.m_ChunkingStats.ChunksStoredInCache.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesStoredInCache.load()), - NiceLatencyNs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Find block stats:" - "\n FindBlockTimeMS: {}" - "\n PotentialChunkCount: {}" - "\n PotentialChunkByteCount: {}" - "\n FoundBlockCount: {}" - "\n FoundBlockChunkCount: {}" - "\n FoundBlockByteCount: {}" - "\n AcceptedBlockCount: {}" - "\n NewBlocksCount: {}" - "\n NewBlocksChunkCount: {}" - "\n NewBlocksChunkByteCount: {}", - NiceTimeSpanMs(UploadOp.m_FindBlocksStats.FindBlockTimeMS), - UploadOp.m_FindBlocksStats.PotentialChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.PotentialChunkByteCount), - UploadOp.m_FindBlocksStats.FoundBlockCount, - UploadOp.m_FindBlocksStats.FoundBlockChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.FoundBlockByteCount), - UploadOp.m_FindBlocksStats.AcceptedBlockCount, - UploadOp.m_FindBlocksStats.NewBlocksCount, - UploadOp.m_FindBlocksStats.NewBlocksChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount)); - - ZEN_CONSOLE_VERBOSE( - "Reuse block stats:" - "\n AcceptedChunkCount: {}" - "\n AcceptedByteCount: {}" - "\n AcceptedRawByteCount: {}" - "\n RejectedBlockCount: {}" - "\n RejectedChunkCount: {}" - "\n RejectedByteCount: {}" - "\n AcceptedReduntantChunkCount: {}" - "\n AcceptedReduntantByteCount: {}", - UploadOp.m_ReuseBlocksStats.AcceptedChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedByteCount), - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedRawByteCount), - UploadOp.m_ReuseBlocksStats.RejectedBlockCount, - UploadOp.m_ReuseBlocksStats.RejectedChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.RejectedByteCount), - UploadOp.m_ReuseBlocksStats.AcceptedReduntantChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedReduntantByteCount)); - - ZEN_CONSOLE_VERBOSE( - "Generate blocks stats:" - "\n GeneratedBlockByteCount: {}" - "\n GeneratedBlockCount: {}" - "\n GenerateBlocksElapsedWallTimeUS: {}", - NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), - UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), - NiceLatencyNs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Generate blocks stats:" - "\n ChunkCount: {}" - "\n ChunkByteCount: {}" - "\n CompressedChunkCount: {}" - "\n CompressChunksElapsedWallTimeUS: {}", - UploadOp.m_LooseChunksStats.ChunkCount, - NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount), - UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), - NiceLatencyNs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Disk stats:" - "\n OpenReadCount: {}" - "\n OpenWriteCount: {}" - "\n ReadCount: {}" - "\n ReadByteCount: {}" - "\n WriteCount: {} ({} cloned)" - "\n WriteByteCount: {} ({} cloned)" - "\n CurrentOpenFileCount: {}", - UploadOp.m_DiskStats.OpenReadCount.load(), - UploadOp.m_DiskStats.OpenWriteCount.load(), - UploadOp.m_DiskStats.ReadCount.load(), - NiceBytes(UploadOp.m_DiskStats.ReadByteCount.load()), - UploadOp.m_DiskStats.WriteCount.load(), - UploadOp.m_DiskStats.CloneCount.load(), - NiceBytes(UploadOp.m_DiskStats.WriteByteCount.load()), - NiceBytes(UploadOp.m_DiskStats.CloneByteCount.load()), - UploadOp.m_DiskStats.CurrentOpenFileCount.load()); - - ZEN_CONSOLE_VERBOSE( - "Upload stats:" - "\n BlockCount: {}" - "\n BlocksBytes: {}" - "\n ChunkCount: {}" - "\n ChunksBytes: {}" - "\n ReadFromDiskBytes: {}" - "\n MultipartAttachmentCount: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_UploadStats.BlockCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), - UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), - NiceBytes(UploadOp.m_UploadStats.ReadFromDiskBytes.load()), - UploadOp.m_UploadStats.MultipartAttachmentCount.load(), - NiceLatencyNs(UploadOp.m_UploadStats.ElapsedWallTimeUS * 1000)); - - const double DeltaByteCountPercent = - UploadOp.m_ChunkingStats.BytesHashed > 0 - ? (100.0 * (UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes)) / - (UploadOp.m_ChunkingStats.BytesHashed) - : 0.0; - - const std::string MultipartAttachmentStats = - Options.AllowMultiparts ? fmt::format(" ({} as multipart)", UploadOp.m_UploadStats.MultipartAttachmentCount.load()) : ""; - - if (!IsQuiet) - { - ZEN_CONSOLE( - "Uploaded part {} ('{}') to build {}, {}\n" - " Scanned files: {:>8} ({}), {}B/sec, {}\n" - " New data: {:>8} ({}) {:.1f}%\n" - " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n" - " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n" - " Uploaded: {:>8} ({}), {}bits/sec, {}\n" - " Blocks: {:>8} ({})\n" - " Chunks: {:>8} ({}){}", - BuildPartId, - BuildPartName, - BuildId, - NiceTimeSpanMs(UploadTimer.GetElapsedTimeMs()), - - UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_ChunkingStats.ElapsedWallTimeUS, UploadOp.m_ChunkingStats.BytesHashed)), - NiceTimeSpanMs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS / 1000), - - UploadOp.m_FindBlocksStats.NewBlocksChunkCount + UploadOp.m_LooseChunksStats.CompressedChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes), - DeltaByteCountPercent, - - UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount), - NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, - UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount)), - NiceTimeSpanMs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000), - - UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkRawBytes), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS, - UploadOp.m_LooseChunksStats.CompressedChunkRawBytes)), - NiceTimeSpanMs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000), - - UploadOp.m_UploadStats.BlockCount.load() + UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes + UploadOp.m_UploadStats.ChunksBytes), - NiceNum(GetBytesPerSecond(UploadOp.m_UploadStats.ElapsedWallTimeUS, - (UploadOp.m_UploadStats.ChunksBytes + UploadOp.m_UploadStats.BlocksBytes) * 8)), - NiceTimeSpanMs(UploadOp.m_UploadStats.ElapsedWallTimeUS / 1000), - - UploadOp.m_UploadStats.BlockCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), - - UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), - MultipartAttachmentStats); - } - return UploadedParts; - } - - struct VerifyFolderStatistics + void WriteResultObject(const std::filesystem::path& Path, const CbObject& Response) { - std::atomic<uint64_t> FilesVerified = 0; - std::atomic<uint64_t> FilesFailed = 0; - std::atomic<uint64_t> ReadBytes = 0; - uint64_t VerifyElapsedWallTimeUs = 0; - }; - - void VerifyFolder(TransferThreadWorkers& Workers, - const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const std::filesystem::path& Path, - const std::vector<std::string>& ExcludeFolders, - bool VerifyFileHash, - VerifyFolderStatistics& VerifyFolderStats) - { - ZEN_TRACE_CPU("VerifyFolder"); - - Stopwatch Timer; - - ProgressBar ProgressBar(ProgressMode, "Verify Files"); - - WorkerThreadPool& VerifyPool = Workers.GetIOWorkerPool(); - - ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - - const uint32_t PathCount = gsl::narrow<uint32_t>(Content.Paths.size()); - - RwLock ErrorLock; - std::vector<std::string> Errors; - - auto IsAcceptedFolder = [ExcludeFolders = ExcludeFolders](const std::string_view& RelativePath) -> bool { - for (const std::string& ExcludeFolder : ExcludeFolders) - { - if (RelativePath.starts_with(ExcludeFolder)) - { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } - } - } - return true; - }; - - for (uint32_t PathIndex = 0; PathIndex < PathCount; PathIndex++) + const MemoryView ResponseView = Response.GetView(); + if (ToLower(Path.extension().string()) == ".cbo") { - if (Work.IsAborted()) - { - break; - } - - Work.ScheduleWork( - VerifyPool, - [&Path, &Content, &Lookup, &ErrorLock, &Errors, &VerifyFolderStats, VerifyFileHash, &IsAcceptedFolder, PathIndex]( - std::atomic<bool>&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("VerifyFile_work"); - - // TODO: Convert ScheduleWork body to function - - const std::filesystem::path TargetPath = (Path / Content.Paths[PathIndex]).make_preferred(); - if (IsAcceptedFolder(TargetPath.parent_path().generic_string())) - { - const uint64_t ExpectedSize = Content.RawSizes[PathIndex]; - if (!IsFile(TargetPath)) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize)); - }); - VerifyFolderStats.FilesFailed++; - } - else - { - std::error_code Ec; - uint64_t SizeOnDisk = gsl::narrow<uint64_t>(FileSizeFromPath(TargetPath, Ec)); - if (Ec) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back( - fmt::format("Failed to get size of file {}: {} ({})", TargetPath, Ec.message(), Ec.value())); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk < ExpectedSize) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Size of file {} is smaller than expected. Expected: {}, Found: {}", - TargetPath, - ExpectedSize, - SizeOnDisk)); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk > ExpectedSize) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Size of file {} is bigger than expected. Expected: {}, Found: {}", - TargetPath, - ExpectedSize, - SizeOnDisk)); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk > 0 && VerifyFileHash) - { - const IoHash& ExpectedRawHash = Content.RawHashes[PathIndex]; - IoBuffer Buffer = IoBufferBuilder::MakeFromFile(TargetPath); - IoHash RawHash = IoHash::HashBuffer(Buffer); - if (RawHash != ExpectedRawHash) - { - uint64_t FileOffset = 0; - const uint32_t SequenceIndex = Lookup.RawHashToSequenceIndex.at(ExpectedRawHash); - const uint32_t OrderOffset = Lookup.SequenceIndexChunkOrderOffset[SequenceIndex]; - for (uint32_t OrderIndex = OrderOffset; - OrderIndex < OrderOffset + Content.ChunkedContent.ChunkCounts[SequenceIndex]; - OrderIndex++) - { - uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex]; - uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; - IoHash ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; - IoBuffer FileChunk = IoBuffer(Buffer, FileOffset, ChunkSize); - if (IoHash::HashBuffer(FileChunk) != ChunkHash) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format( - "WARNING: Hash of file {} does not match expected hash. Expected: {}, Found: {}. " - "Mismatch at chunk {}", - TargetPath, - ExpectedRawHash, - RawHash, - OrderIndex - OrderOffset)); - }); - break; - } - FileOffset += ChunkSize; - } - VerifyFolderStats.FilesFailed++; - } - VerifyFolderStats.ReadBytes += SizeOnDisk; - } - } - } - VerifyFolderStats.FilesVerified++; - } - }, - [&, PathIndex](std::exception_ptr Ex, std::atomic<bool>&) { - std::string Description; - try - { - std::rethrow_exception(Ex); - } - catch (const std::exception& Ex) - { - Description = Ex.what(); - } - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Failed verifying file '{}'. Reason: {}", - (Path / Content.Paths[PathIndex]).make_preferred(), - Description)); - }); - VerifyFolderStats.FilesFailed++; - }); - } - - Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(PendingWork); - std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}", - VerifyFolderStats.FilesVerified.load(), - PathCount, - NiceBytes(VerifyFolderStats.ReadBytes.load()), - VerifyFolderStats.FilesFailed.load()); - ProgressBar.UpdateState({.Task = "Verifying files ", - .Details = Details, - .TotalCount = gsl::narrow<uint64_t>(PathCount), - .RemainingCount = gsl::narrow<uint64_t>(PathCount - VerifyFolderStats.FilesVerified.load()), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }); - VerifyFolderStats.VerifyElapsedWallTimeUs = Timer.GetElapsedTimeUs(); - - ProgressBar.Finish(); - if (AbortFlag) - { - return; - } - - for (const std::string& Error : Errors) - { - ZEN_CONSOLE_ERROR("{}", Error); - } - if (!Errors.empty()) - { - throw std::runtime_error(fmt::format("Verify failed with {} errors", Errors.size())); - } - } - - CbObject GetBuild(BuildStorageBase& Storage, const Oid& BuildId) - { - Stopwatch GetBuildTimer; - CbObject BuildObject = Storage.GetBuild(BuildId); - if (!IsQuiet) - { - ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", - NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), - BuildObject["name"sv].AsString(), - NiceBytes(BuildObject.GetSize())); - - ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv)); - } - return BuildObject; - } - - std::vector<std::filesystem::path> GetNewPaths(const std::span<const std::filesystem::path> KnownPaths, - const std::span<const std::filesystem::path> Paths) - { - tsl::robin_set<std::string> KnownPathsSet; - KnownPathsSet.reserve(KnownPaths.size()); - for (const std::filesystem::path& LocalPath : KnownPaths) - { - KnownPathsSet.insert(LocalPath.generic_string()); - } - - std::vector<std::filesystem::path> NewPaths; - for (const std::filesystem::path& UntrackedPath : Paths) - { - if (!KnownPathsSet.contains(UntrackedPath.generic_string())) - { - NewPaths.push_back(UntrackedPath); - } - } - return NewPaths; - } - - BuildSaveState GetLocalStateFromPaths(TransferThreadWorkers& Workers, - GetFolderContentStatistics& LocalFolderScanStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - std::span<const std::filesystem::path> PathsToCheck) - { - FolderContent FolderState; - ChunkedFolderContent ChunkedContent; - { - ProgressBar ProgressBar(ProgressMode, "Check Files"); - FolderState = GetValidFolderContent( - Workers.GetIOWorkerPool(), - LocalFolderScanStats, - Path, - PathsToCheck, - [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) { - std::string Details = - fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load()); - ProgressBar.UpdateState({.Task = "Checking files ", - .Details = Details, - .TotalCount = PathCount, - .RemainingCount = PathCount - CompletedPathCount, - .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, - false); - }, - GetUpdateDelayMS(ProgressMode), - AbortFlag, - PauseFlag); - ProgressBar.Finish(); - } - - if (FolderState.Paths.size() > 0) - { - uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : FolderState.RawSizes) - { - ByteCountToScan += RawSize; - } - ProgressBar ProgressBar(ProgressMode, "Scan Files"); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkingStatistics LocalChunkingStats; - ChunkedContent = ChunkFolderContent( - LocalChunkingStats, - Workers.GetIOWorkerPool(), - Path, - FolderState, - ChunkController, - ChunkCache, - GetUpdateDelayMS(ProgressMode), - [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { - FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - LocalChunkingStats.FilesProcessed.load(), - FolderState.Paths.size(), - NiceBytes(LocalChunkingStats.BytesHashed.load()), - NiceBytes(ByteCountToScan), - NiceNum(FilteredBytesHashed.GetCurrent()), - LocalChunkingStats.UniqueChunksFound.load(), - NiceBytes(LocalChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }, - AbortFlag, - PauseFlag); - ChunkingStats += LocalChunkingStats; - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - } - - return BuildSaveState{.State = BuildState{.ChunkedContent = std::move(ChunkedContent)}, - .FolderState = FolderState, - .LocalPath = Path}; - } - - BuildSaveState GetLocalContent(TransferThreadWorkers& Workers, - GetFolderContentStatistics& LocalFolderScanStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - const std::filesystem::path& StateFilePath, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache) - { - Stopwatch ReadStateTimer; - bool FileExists = IsFile(StateFilePath); - if (!FileExists) - { - ZEN_CONSOLE("No known local state file in {}, falling back to scanning", Path); - return {}; - } - - BuildSaveState SavedLocalState; - try - { - SavedLocalState = ReadBuildSaveStateFile(StateFilePath); - if (!IsQuiet) - { - ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); - } - } - catch (const std::exception& Ex) - { - ZEN_CONSOLE_WARN("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what()); - return {}; - } - - FolderContent CurrentLocalFolderState; - { - ProgressBar ProgressBar(ProgressMode, "Check Known Files"); - CurrentLocalFolderState = GetValidFolderContent( - Workers.GetIOWorkerPool(), - LocalFolderScanStats, - Path, - SavedLocalState.FolderState.Paths, - [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) { - std::string Details = - fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load()); - ProgressBar.UpdateState({.Task = "Checking files ", - .Details = Details, - .TotalCount = PathCount, - .RemainingCount = PathCount - CompletedPathCount, - .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, - false); - }, - GetUpdateDelayMS(ProgressMode), - AbortFlag, - PauseFlag); - ProgressBar.Finish(); - } - if (AbortFlag) - { - return {}; - } - - if (!SavedLocalState.FolderState.AreKnownFilesEqual(CurrentLocalFolderState)) - { - const size_t LocalStatePathCount = SavedLocalState.FolderState.Paths.size(); - std::vector<std::filesystem::path> DeletedPaths; - FolderContent UpdatedContent = GetUpdatedContent(SavedLocalState.FolderState, CurrentLocalFolderState, DeletedPaths); - if (!DeletedPaths.empty()) - { - SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths); - } - - if (!IsQuiet) - { - ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", - DeletedPaths.size(), - UpdatedContent.Paths.size(), - LocalStatePathCount); - } - if (UpdatedContent.Paths.size() > 0) - { - uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : UpdatedContent.RawSizes) - { - ByteCountToScan += RawSize; - } - ProgressBar ProgressBar(ProgressMode, "Scan Known Files"); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkingStatistics LocalChunkingStats; - ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( - LocalChunkingStats, - Workers.GetIOWorkerPool(), - Path, - UpdatedContent, - ChunkController, - ChunkCache, - GetUpdateDelayMS(ProgressMode), - [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { - FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - LocalChunkingStats.FilesProcessed.load(), - UpdatedContent.Paths.size(), - NiceBytes(LocalChunkingStats.BytesHashed.load()), - NiceBytes(ByteCountToScan), - NiceNum(FilteredBytesHashed.GetCurrent()), - LocalChunkingStats.UniqueChunksFound.load(), - NiceBytes(LocalChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }, - AbortFlag, - PauseFlag); - - ChunkingStats += LocalChunkingStats; - - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - if (AbortFlag) - { - return {}; - } - SavedLocalState.State.ChunkedContent = - MergeChunkedFolderContents(SavedLocalState.State.ChunkedContent, {{UpdatedLocalContent}}); - } + WriteFile(Path, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); } else { - // Remove files from LocalContent no longer in LocalFolderState - tsl::robin_set<std::string> LocalFolderPaths; - LocalFolderPaths.reserve(SavedLocalState.FolderState.Paths.size()); - for (const std::filesystem::path& LocalFolderPath : SavedLocalState.FolderState.Paths) - { - LocalFolderPaths.insert(LocalFolderPath.generic_string()); - } - std::vector<std::filesystem::path> DeletedPaths; - for (const std::filesystem::path& LocalContentPath : SavedLocalState.State.ChunkedContent.Paths) - { - if (!LocalFolderPaths.contains(LocalContentPath.generic_string())) - { - DeletedPaths.push_back(LocalContentPath); - } - } - if (!DeletedPaths.empty()) - { - SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths); - } - } - - SavedLocalState.FolderState = CurrentLocalFolderState; - - return SavedLocalState; - } - - ChunkedFolderContent ScanAndChunkFolder( - TransferThreadWorkers& Workers, - GetFolderContentStatistics& GetFolderContentStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - std::function<bool(const std::string_view& RelativePath)>&& IsAcceptedFolder, - std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache) - { - Stopwatch Timer; - - ZEN_TRACE_CPU("ScanAndChunkFolder"); - - FolderContent Content = GetFolderContent( - GetFolderContentStats, - Path, - std::move(IsAcceptedFolder), - std::move(IsAcceptedFile), - Workers.GetIOWorkerPool(), - GetUpdateDelayMS(ProgressMode), - [](bool, std::ptrdiff_t) {}, - AbortFlag); - if (AbortFlag) - { - return {}; - } - - BuildState LocalContent = GetLocalContent(Workers, - GetFolderContentStats, - ChunkingStats, - Path, - ZenStateFilePath(Path / ZenFolderName), - ChunkController, - ChunkCache) - .State; - - std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalContent.ChunkedContent.Paths, Content.Paths); - - BuildState UntrackedLocalContent = - GetLocalStateFromPaths(Workers, GetFolderContentStats, ChunkingStats, Path, ChunkController, ChunkCache, UntrackedPaths).State; - - ChunkedFolderContent Result = MergeChunkedFolderContents(LocalContent.ChunkedContent, - std::vector<ChunkedFolderContent>{UntrackedLocalContent.ChunkedContent}); - - const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0)); - const uint64_t ChunkedRawSize = - std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0)); - - if (!IsQuiet) - { - ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", - Result.Paths.size(), - NiceBytes(TotalRawSize), - Result.ChunkedContent.ChunkHashes.size(), - NiceBytes(ChunkedRawSize), - Path, - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); - } - return Result; - }; - - struct DownloadOptions - { - std::filesystem::path SystemRootDir; - std::filesystem::path ZenFolderPath; - bool AllowMultiparts = true; - EPartialBlockRequestMode PartialBlockRequestMode = EPartialBlockRequestMode::Mixed; - bool CleanTargetFolder = false; - bool PostDownloadVerify = false; - bool EnableOtherDownloadsScavenging = true; - bool EnableTargetFolderScavenging = true; - bool AllowFileClone = true; - std::vector<std::string> IncludeWildcards; - std::vector<std::string> ExcludeWildcards; - uint64_t MaximumInMemoryPayloadSize = 512u * 1024u; - bool PopulateCache = true; - bool AppendNewContent = false; - std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; - }; - - void DownloadFolder(LoggerRef InLog, - ProgressBase& Progress, - TransferThreadWorkers& Workers, - StorageInstance& Storage, - const BuildStorageCache::Statistics& StorageCacheStats, - const Oid& BuildId, - const std::vector<Oid>& BuildPartIds, - std::span<const std::string> BuildPartNames, - const std::filesystem::path& DownloadSpecPath, - const std::filesystem::path& Path, - const DownloadOptions& Options) - { - ZEN_TRACE_CPU("DownloadFolder"); - ZEN_SCOPED_LOG(InLog); - - ProgressBar::SetLogOperationName(ProgressMode, "Download Folder"); - - enum TaskSteps : uint32_t - { - CheckState, - CompareState, - Download, - Verify, - Cleanup, - StepCount - }; - - auto EndProgress = - MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); - - Stopwatch DownloadTimer; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount); - - const std::filesystem::path ZenTempFolder = ZenTempFolderPath(Options.ZenFolderPath); - CreateDirectories(ZenTempFolder); - - std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); - - std::vector<std::pair<Oid, std::string>> AllBuildParts = - ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); - - BuildManifest Manifest; - if (!DownloadSpecPath.empty()) - { - const std::filesystem::path AbsoluteDownloadSpecPath = - DownloadSpecPath.is_relative() ? MakeSafeAbsolutePath(Path / DownloadSpecPath) : MakeSafeAbsolutePath(DownloadSpecPath); - Manifest = ParseBuildManifest(DownloadSpecPath); - } - - std::vector<ChunkedFolderContent> PartContents; - - std::unique_ptr<ChunkingController> ChunkController; - - std::vector<ChunkBlockDescription> BlockDescriptions; - std::vector<IoHash> LooseChunkHashes; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount); - - ChunkedFolderContent RemoteContent = GetRemoteContent(InLog, - Storage, - BuildId, - AllBuildParts, - Manifest, - Options.IncludeWildcards, - Options.ExcludeWildcards, - ChunkController, - PartContents, - BlockDescriptions, - LooseChunkHashes, - IsQuiet, - IsVerbose, - DoExtraContentVerify); - - const std::uint64_t LargeAttachmentSize = Options.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; - GetFolderContentStatistics LocalFolderScanStats; - ChunkingStatistics ChunkingStats; - - BuildSaveState LocalState; - - if (IsDir(Path)) - { - if (!ChunkController && !IsQuiet) - { - ZEN_CONSOLE_INFO("Unspecified chunking algorithm, using default"); - ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - } - std::unique_ptr<ChunkingCache> ChunkCache(CreateNullChunkingCache()); - - LocalState = GetLocalContent(Workers, - LocalFolderScanStats, - ChunkingStats, - Path, - ZenStateFilePath(Path / ZenFolderName), - *ChunkController, - *ChunkCache); - - std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalState.State.ChunkedContent.Paths, RemoteContent.Paths); - - BuildSaveState UntrackedLocalContent = - GetLocalStateFromPaths(Workers, LocalFolderScanStats, ChunkingStats, Path, *ChunkController, *ChunkCache, UntrackedPaths); - - if (!UntrackedLocalContent.State.ChunkedContent.Paths.empty()) - { - LocalState.State.ChunkedContent = - MergeChunkedFolderContents(LocalState.State.ChunkedContent, - std::vector<ChunkedFolderContent>{UntrackedLocalContent.State.ChunkedContent}); - - // TODO: Helper - LocalState.FolderState.Paths.insert(LocalState.FolderState.Paths.begin(), - UntrackedLocalContent.FolderState.Paths.begin(), - UntrackedLocalContent.FolderState.Paths.end()); - LocalState.FolderState.RawSizes.insert(LocalState.FolderState.RawSizes.begin(), - UntrackedLocalContent.FolderState.RawSizes.begin(), - UntrackedLocalContent.FolderState.RawSizes.end()); - LocalState.FolderState.Attributes.insert(LocalState.FolderState.Attributes.begin(), - UntrackedLocalContent.FolderState.Attributes.begin(), - UntrackedLocalContent.FolderState.Attributes.end()); - LocalState.FolderState.ModificationTicks.insert(LocalState.FolderState.ModificationTicks.begin(), - UntrackedLocalContent.FolderState.ModificationTicks.begin(), - UntrackedLocalContent.FolderState.ModificationTicks.end()); - } - - if (Options.AppendNewContent) - { - RemoteContent = ApplyChunkedContentOverlay(LocalState.State.ChunkedContent, - RemoteContent, - Options.IncludeWildcards, - Options.ExcludeWildcards); - } -#if ZEN_BUILD_DEBUG - ValidateChunkedFolderContent(RemoteContent, - BlockDescriptions, - LooseChunkHashes, - Options.IncludeWildcards, - Options.ExcludeWildcards); -#endif // ZEN_BUILD_DEBUG - } - else - { - CreateDirectories(Path); - } - if (AbortFlag) - { - return; - } - - LocalState.LocalPath = Path; - - { - BuildsSelection::Build RemoteBuildState = {.Id = BuildId, - .IncludeWildcards = Options.IncludeWildcards, - .ExcludeWildcards = Options.ExcludeWildcards}; - RemoteBuildState.Parts.reserve(BuildPartIds.size()); - for (size_t PartIndex = 0; PartIndex < BuildPartIds.size(); PartIndex++) - { - RemoteBuildState.Parts.push_back( - {BuildsSelection::BuildPart{.Id = BuildPartIds[PartIndex], - .Name = PartIndex < BuildPartNames.size() ? BuildPartNames[PartIndex] : ""}}); - } - - if (Options.AppendNewContent) - { - LocalState.State.Selection.Builds.emplace_back(std::move(RemoteBuildState)); - } - else - { - LocalState.State.Selection.Builds = std::vector<BuildsSelection::Build>{std::move(RemoteBuildState)}; - } - } - - if ((Options.EnableTargetFolderScavenging || Options.AppendNewContent) && !Options.CleanTargetFolder && - CompareChunkedContent(RemoteContent, LocalState.State.ChunkedContent)) - { - if (!IsQuiet) - { - ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", - NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); - } - - Stopwatch WriteStateTimer; - - CbObject StateObject = CreateBuildSaveStateObject(LocalState); - CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path()); - TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView()); - if (!IsQuiet) - { - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); - } - - AddDownloadedPath(Options.SystemRootDir, - BuildsDownloadInfo{.Selection = LocalState.State.Selection, - .LocalPath = Path, - .StateFilePath = ZenStateFilePath(Options.ZenFolderPath), - .Iso8601Date = DateTime::Now().ToIso8601()}); - } - else - { - ExtendableStringBuilder<128> BuildPartString; - for (const std::pair<Oid, std::string>& BuildPart : AllBuildParts) - { - BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first)); - } - - uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0)); - - if (!IsQuiet) - { - ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize)); - } - - Stopwatch IndexTimer; - - const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalState.State.ChunkedContent); - const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); - - if (!IsQuiet) - { - ZEN_INFO("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount); - - BuildsOperationUpdateFolder Updater( - InLog, - Progress, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - Path, - LocalState.State.ChunkedContent, - LocalLookup, - RemoteContent, - RemoteLookup, - BlockDescriptions, - LooseChunkHashes, - BuildsOperationUpdateFolder::Options{ - .IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .AllowFileClone = Options.AllowFileClone, - .UseSparseFiles = UseSparseFiles, - .SystemRootDir = Options.SystemRootDir, - .ZenFolderPath = Options.ZenFolderPath, - .LargeAttachmentSize = LargeAttachmentSize, - .PreferredMultipartChunkSize = PreferredMultipartChunkSize, - .PartialBlockRequestMode = Options.PartialBlockRequestMode, - .WipeTargetFolder = Options.CleanTargetFolder, - .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging, - .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging || Options.AppendNewContent, - .ValidateCompletedSequences = Options.PostDownloadVerify, - .ExcludeFolders = Options.ExcludeFolders, - .MaximumInMemoryPayloadSize = Options.MaximumInMemoryPayloadSize, - .PopulateCache = Options.PopulateCache}); - { - ProgressBar::PushLogOperation(ProgressMode, "Download"); - auto _ = MakeGuard([]() { ProgressBar::PopLogOperation(ProgressMode); }); - FolderContent UpdatedLocalFolderState; - Updater.Execute(UpdatedLocalFolderState); - - LocalState.State.ChunkedContent = RemoteContent; - LocalState.FolderState = std::move(UpdatedLocalFolderState); - } - - VerifyFolderStatistics VerifyFolderStats; - if (!AbortFlag) - { - AddDownloadedPath(Options.SystemRootDir, - BuildsDownloadInfo{.Selection = LocalState.State.Selection, - .LocalPath = Path, - .StateFilePath = ZenStateFilePath(Options.ZenFolderPath), - .Iso8601Date = DateTime::Now().ToIso8601()}); - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount); - - VerifyFolder(Workers, - RemoteContent, - RemoteLookup, - Path, - Options.ExcludeFolders, - Options.PostDownloadVerify, - VerifyFolderStats); - - Stopwatch WriteStateTimer; - CbObject StateObject = CreateBuildSaveStateObject(LocalState); - - CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path()); - TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView()); - if (!IsQuiet) - { - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); - } - -#if 0 - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(StateObject, SB); - WriteFile(ZenStateFileJsonPath(Options.ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); -#endif // 0 - const uint64_t DownloadCount = Updater.m_DownloadStats.DownloadedChunkCount.load() + - Updater.m_DownloadStats.DownloadedBlockCount.load() + - Updater.m_DownloadStats.DownloadedPartialBlockCount.load(); - const uint64_t DownloadByteCount = Updater.m_DownloadStats.DownloadedChunkByteCount.load() + - Updater.m_DownloadStats.DownloadedBlockByteCount.load() + - Updater.m_DownloadStats.DownloadedPartialBlockByteCount.load(); - const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs(); - - if (!IsQuiet) - { - std::string CloneInfo; - if (Updater.m_DiskStats.CloneByteCount > 0) - { - CloneInfo = fmt::format(" ({} cloned)", NiceBytes(Updater.m_DiskStats.CloneByteCount.load())); - } - - std::string DownloadDetails; - { - ExtendableStringBuilder<128> SB; - BuildStorageBase::ExtendedStatistics ExtendedDownloadStats; - if (Storage.BuildStorage->GetExtendedStatistics(ExtendedDownloadStats)) - { - if (!ExtendedDownloadStats.ReceivedBytesPerSource.empty()) - { - for (auto& It : ExtendedDownloadStats.ReceivedBytesPerSource) - { - if (SB.Size() > 0) - { - SB.Append(", "sv); - } - SB.Append(It.first); - SB.Append(": "sv); - SB.Append(NiceBytes(It.second)); - } - } - } - if (Storage.CacheStorage) - { - if (SB.Size() > 0) - { - SB.Append(", "sv); - } - SB.Append("Cache: "); - SB.Append(NiceBytes(StorageCacheStats.TotalBytesRead.load())); - } - if (SB.Size() > 0) - { - DownloadDetails = fmt::format(" ({})", SB.ToView()); - } - } - - ZEN_CONSOLE( - "Downloaded build {}, parts:{} in {}\n" - " Scavenge: {} (Target: {}, Cache: {}, Others: {})\n" - " Download: {} ({}) {}bits/s{}\n" - " Write: {} ({}) {}B/s{}\n" - " Clean: {}\n" - " Finalize: {}\n" - " Verify: {}", - BuildId, - BuildPartString.ToView(), - NiceTimeSpanMs(DownloadTimeMs), - - NiceTimeSpanMs((Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs + - Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs + - Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs) / - 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000), - - DownloadCount, - NiceBytes(DownloadByteCount), - NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)), - DownloadDetails, - - Updater.m_DiskStats.WriteCount.load(), - NiceBytes(Updater.m_WrittenChunkByteCount.load()), - NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.WriteTimeUs, Updater.m_DiskStats.WriteByteCount.load())), - CloneInfo, - - NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000), - - NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000), - - NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); - } - } - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); - - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), ZenTempFolder); - } - - void ListBuild(StorageInstance& Storage, - const Oid& BuildId, - const std::vector<Oid>& BuildPartIds, - std::span<const std::string> BuildPartNames, - std::span<const std::string> IncludeWildcards, - std::span<const std::string> ExcludeWildcards, - CbObjectWriter* OptionalStructuredOutput) - { - std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->AddObjectId("buildId"sv, BuildId); - OptionalStructuredOutput->AddObject("build"sv, BuildObject); - } - - std::vector<std::pair<Oid, std::string>> AllBuildParts = - ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); - - if (!AllBuildParts.empty()) - { - Stopwatch GetBuildPartTimer; - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginArray("parts"sv); - } - - for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++) - { - const Oid BuildPartId = AllBuildParts[BuildPartIndex].first; - const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second; - CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginObject(); - OptionalStructuredOutput->AddObjectId("id"sv, BuildPartId); - OptionalStructuredOutput->AddString("partName"sv, BuildPartName); - } - { - if (OptionalStructuredOutput != nullptr) - { - } - else if (!IsQuiet) - { - ZEN_CONSOLE("{}Part: {} ('{}'):\n", - BuildPartIndex > 0 ? "\n" : "", - BuildPartId, - BuildPartName, - NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), - NiceBytes(BuildPartManifest.GetSize())); - } - - std::vector<std::filesystem::path> Paths; - std::vector<IoHash> RawHashes; - std::vector<uint64_t> RawSizes; - std::vector<uint32_t> Attributes; - - SourcePlatform Platform; - std::vector<IoHash> SequenceRawHashes; - std::vector<uint32_t> ChunkCounts; - std::vector<uint32_t> AbsoluteChunkOrders; - std::vector<IoHash> LooseChunkHashes; - std::vector<uint64_t> LooseChunkRawSizes; - std::vector<IoHash> BlockRawHashes; - - ReadBuildContentFromCompactBinary(BuildPartManifest, - Platform, - Paths, - RawHashes, - RawSizes, - Attributes, - SequenceRawHashes, - ChunkCounts, - AbsoluteChunkOrders, - LooseChunkHashes, - LooseChunkRawSizes, - BlockRawHashes); - - std::vector<size_t> Order(Paths.size()); - std::iota(Order.begin(), Order.end(), 0); - - std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) { - const std::filesystem::path& LhsPath = Paths[Lhs]; - const std::filesystem::path& RhsPath = Paths[Rhs]; - return LhsPath < RhsPath; - }); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginArray("files"sv); - } - { - for (size_t Index : Order) - { - const std::filesystem::path& Path = Paths[Index]; - if (IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(Path.generic_string()), /*CaseSensitive*/ true)) - { - const IoHash& RawHash = RawHashes[Index]; - const uint64_t RawSize = RawSizes[Index]; - const uint32_t Attribute = Attributes[Index]; - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginObject(); - { - OptionalStructuredOutput->AddString("path"sv, fmt::format("{}", Path)); - OptionalStructuredOutput->AddInteger("rawSize"sv, RawSize); - OptionalStructuredOutput->AddHash("rawHash"sv, RawHash); - switch (Platform) - { - case SourcePlatform::Windows: - OptionalStructuredOutput->AddInteger("attributes"sv, Attribute); - break; - case SourcePlatform::MacOS: - case SourcePlatform::Linux: - OptionalStructuredOutput->AddString("chmod"sv, fmt::format("{:#04o}", Attribute)); - break; - default: - throw std::runtime_error(fmt::format("Unsupported platform: {}", (int)Platform)); - } - } - OptionalStructuredOutput->EndObject(); - } - else - { - ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash); - } - } - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndArray(); // "files" - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndObject(); - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndArray(); // parts - } - } - } - - void DiffFolders(TransferThreadWorkers& Workers, - const std::filesystem::path& BasePath, - const std::filesystem::path& ComparePath, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - const std::vector<std::string>& ExcludeFolders, - const std::vector<std::string>& ExcludeExtensions) - { - ZEN_TRACE_CPU("DiffFolders"); - - ProgressBar::SetLogOperationName(ProgressMode, "Diff Folders"); - - enum TaskSteps : uint32_t - { - CheckBase, - CheckCompare, - Diff, - Cleanup, - StepCount - }; - - auto EndProgress = - MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); - - ChunkedFolderContent BaseFolderContent; - ChunkedFolderContent CompareFolderContent; - - { - auto IsAcceptedFolder = [ExcludeFolders](const std::string_view& RelativePath) -> bool { - for (const std::string& ExcludeFolder : ExcludeFolders) - { - if (RelativePath.starts_with(ExcludeFolder)) - { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } - } - } - return true; - }; - - auto IsAcceptedFile = [ExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool { - for (const std::string& ExcludeExtension : ExcludeExtensions) - { - if (RelativePath.ends_with(ExcludeExtension)) - { - return false; - } - } - return true; - }; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckBase, TaskSteps::StepCount); - - GetFolderContentStatistics BaseGetFolderContentStats; - ChunkingStatistics BaseChunkingStats; - BaseFolderContent = ScanAndChunkFolder(Workers, - BaseGetFolderContentStats, - BaseChunkingStats, - BasePath, - IsAcceptedFolder, - IsAcceptedFile, - ChunkController, - ChunkCache); - if (AbortFlag) - { - return; - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckCompare, TaskSteps::StepCount); - - GetFolderContentStatistics CompareGetFolderContentStats; - ChunkingStatistics CompareChunkingStats; - CompareFolderContent = ScanAndChunkFolder(Workers, - CompareGetFolderContentStats, - CompareChunkingStats, - ComparePath, - IsAcceptedFolder, - IsAcceptedFile, - ChunkController, - ChunkCache); - - if (AbortFlag) - { - return; - } - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Diff, TaskSteps::StepCount); - - std::vector<IoHash> AddedHashes; - std::vector<IoHash> RemovedHashes; - uint64_t RemovedSize = 0; - uint64_t AddedSize = 0; - - tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> BaseRawHashLookup; - for (size_t PathIndex = 0; PathIndex < BaseFolderContent.RawHashes.size(); PathIndex++) - { - const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex]; - BaseRawHashLookup.insert_or_assign(RawHash, PathIndex); - } - tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CompareRawHashLookup; - for (size_t PathIndex = 0; PathIndex < CompareFolderContent.RawHashes.size(); PathIndex++) - { - const IoHash& RawHash = CompareFolderContent.RawHashes[PathIndex]; - if (!BaseRawHashLookup.contains(RawHash)) - { - AddedHashes.push_back(RawHash); - AddedSize += CompareFolderContent.RawSizes[PathIndex]; - } - CompareRawHashLookup.insert_or_assign(RawHash, PathIndex); - } - for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++) - { - const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex]; - if (!CompareRawHashLookup.contains(RawHash)) - { - RemovedHashes.push_back(RawHash); - RemovedSize += BaseFolderContent.RawSizes[PathIndex]; - } - } - - uint64_t BaseTotalRawSize = 0; - for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++) - { - BaseTotalRawSize += BaseFolderContent.RawSizes[PathIndex]; - } - - double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0; - - ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept", - RemovedHashes.size(), - NiceBytes(RemovedSize), - AddedHashes.size(), - NiceBytes(AddedSize), - BaseFolderContent.Paths.size() - RemovedHashes.size(), - NiceBytes(BaseTotalRawSize - RemovedSize), - KeptPercent); - - uint64_t CompareTotalRawSize = 0; - - uint64_t FoundChunkCount = 0; - uint64_t FoundChunkSize = 0; - uint64_t NewChunkCount = 0; - uint64_t NewChunkSize = 0; - const ChunkedContentLookup BaseFolderLookup = BuildChunkedContentLookup(BaseFolderContent); - for (uint32_t ChunkIndex = 0; ChunkIndex < CompareFolderContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++) - { - const IoHash& ChunkHash = CompareFolderContent.ChunkedContent.ChunkHashes[ChunkIndex]; - if (BaseFolderLookup.ChunkHashToChunkIndex.contains(ChunkHash)) - { - FoundChunkCount++; - FoundChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - } - else - { - NewChunkCount++; - NewChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - } - CompareTotalRawSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(ResponseView, SB); + WriteFile(Path, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); } - - double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0; - double NewPercent = CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0; - - ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.", - FoundChunkCount, - NiceBytes(FoundChunkSize), - FoundPercent, - CompareFolderContent.ChunkedContent.ChunkHashes.size(), - NiceBytes(CompareTotalRawSize), - BaseFolderContent.ChunkedContent.ChunkHashes.size(), - NiceBytes(BaseTotalRawSize), - NewChunkCount, - NiceBytes(NewChunkSize), - NewPercent); - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); } } // namespace builds_impl -////////////////////////////////////////////////////////////////////////////////////////////////////// -// BuildsCommand - Option-adding helpers -// +////////////////////////////////////////////////////////////////////////// void -BuildsCommand::AddSystemOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddSystemOptions(cxxopts::Options& Ops) { - Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>"); + Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(SystemRootDir), "<systemdir>"); Ops.add_option("", "", "use-sparse-files", "Enable use of sparse files when writing large files. Defaults to true.", - cxxopts::value(m_UseSparseFiles), + cxxopts::value(UseSparseFiles), "<usesparsefiles>"); } void -BuildsCommand::AddCloudOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddCloudOptions(cxxopts::Options& Ops) { - m_AuthOptions.AddOptions(Ops); + AuthOptions.AddOptions(Ops); - Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>"); - Ops.add_option("cloud build", - "", - "url", - "Cloud Builds host url (legacy - use --override-host)", - cxxopts::value(m_OverrideHost), - "<url>"); - Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), "<cloud-url>"); - Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>"); + Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(OverrideHost), "<override-host>"); + Ops.add_option("cloud build", "", "url", "Cloud Builds host url (legacy - use --override-host)", cxxopts::value(OverrideHost), "<url>"); + Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(Url), "<cloud-url>"); + Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(Host), "<host>"); Ops.add_option("cloud build", "", "assume-http2", "Assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake", - cxxopts::value(m_AssumeHttp2), + cxxopts::value(AssumeHttp2), "<assumehttp2>"); Ops.add_option("cloud build", "", "verbose-http", "Enable verbose option for http client", - cxxopts::value(m_VerboseHttp), + cxxopts::value(VerboseHttp), "<verbosehttp>"); - Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), "<namespace>"); - Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), "<bucket>"); - Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(m_AllowRedirect), "<allow-redirect>"); + Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(Namespace), "<namespace>"); + Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(Bucket), "<bucket>"); + Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(AllowRedirect), "<allow-redirect>"); } void -BuildsCommand::AddFileOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddFileOptions(cxxopts::Options& Ops) { - Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(m_StoragePath), "<storagepath>"); + Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(StoragePath), "<storagepath>"); Ops.add_option("filestorage", "", "json-metadata", "Write build, part and block metadata as .json files in addition to .cb files", - cxxopts::value(m_WriteMetadataAsJson), + cxxopts::value(WriteMetadataAsJson), "<jsonmetadata>"); } void -BuildsCommand::AddCacheOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddCacheOptions(cxxopts::Options& Ops) { - Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>"); + Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(ZenCacheHost), "<zenhost>"); } void -BuildsCommand::AddOutputOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddOutputOptions(cxxopts::Options& Ops) { - Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), "<plainprogress>"); - Ops.add_option("output", - "", - "log-progress", - "Write @progress style progress to output", - cxxopts::value(m_LogProgress), - "<logprogress>"); - Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>"); - Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>"); + Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(PlainProgress), "<plainprogress>"); + Ops.add_option("output", "", "log-progress", "Write @progress style progress to output", cxxopts::value(LogProgress), "<logprogress>"); + Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(Verbose), "<verbose>"); + Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(Quiet), "<quiet>"); } void -BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddWorkerOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "boost-worker-count", "Increase the number of worker threads - may cause computer to be less responsive", - cxxopts::value(m_BoostWorkerCount), + cxxopts::value(BoostWorkerCount), "<boostworkercount>"); Ops.add_option("", @@ -2077,47 +301,47 @@ BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops) "boost-worker-memory", "Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to " "be less responsive due to high memory usage", - cxxopts::value(m_BoostWorkerMemory), + cxxopts::value(BoostWorkerMemory), "<boostworkermemory>"); Ops.add_option("", "", "boost-workers", "Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive", - cxxopts::value(m_BoostWorkers), + cxxopts::value(BoostWorkers), "<boostworkermemory>"); } void -BuildsCommand::AddZenFolderOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddZenFolderOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "zen-folder-path", - fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", builds_impl::ZenFolderName), - cxxopts::value(m_ZenFolderPath), + fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", ZenFolderName), + cxxopts::value(ZenFolderPath), "<boostworkers>"); } void -BuildsCommand::AddChunkingCacheOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddChunkingCacheOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "chunking-cache-path", "Path to cache for chunking information of scanned files. Default is empty resulting in no caching", - cxxopts::value(m_ChunkingCachePath), + cxxopts::value(ChunkingCachePath), "<chunkingcachepath>"); } void -BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddWildcardOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "wildcard", "Windows style wildcard(s) (using * and ?) to match file paths to include, separated by ;", - cxxopts::value(m_IncludeWildcard), + cxxopts::value(IncludeWildcard), "<wildcard>"); Ops.add_option("", @@ -2125,46 +349,46 @@ BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops) "exclude-wildcard", "Windows style wildcard(s) (using * and ?) to match file paths to exclude, separated by ;. Applied after --wildcard " "include filter", - cxxopts::value(m_ExcludeWildcard), + cxxopts::value(ExcludeWildcard), "<excludewildcard>"); } void -BuildsCommand::AddExcludeFolderOption(cxxopts::Options& Ops) +BuildsConfiguration::AddExcludeFolderOption(cxxopts::Options& Ops) { Ops.add_option("", "", "exclude-folders", "Names of folders to exclude, separated by ;", - cxxopts::value(m_ExcludeFolders), + cxxopts::value(ExcludeFolders), "<excludefolders>"); } void -BuildsCommand::AddExcludeExtensionsOption(cxxopts::Options& Ops) +BuildsConfiguration::AddExcludeExtensionsOption(cxxopts::Options& Ops) { Ops.add_option("", "", "exclude-extensions", "Extensions to exclude, separated by ;" "include filter", - cxxopts::value(m_ExcludeExtensions), + cxxopts::value(ExcludeExtensions), "<excludeextensions>"); } void -BuildsCommand::AddMultipartOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddMultipartOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "allow-multipart", "Allow large attachments to be transfered using multipart protocol. Defaults to true.", - cxxopts::value(m_AllowMultiparts), + cxxopts::value(AllowMultiparts), "<allowmultipart>"); } void -BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddPartialBlockRequestOptions(cxxopts::Options& Ops) { Ops.add_option("", "", @@ -2177,12 +401,12 @@ BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops) "allowed to host\n" " true = multiple partial block ranges requests per block allowed to zen cache and host\n" "Defaults to 'mixed'.", - cxxopts::value(m_AllowPartialBlockRequests), + cxxopts::value(AllowPartialBlockRequests), "<allowpartialblockrequests>"); } void -BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddAppendNewContentOptions(cxxopts::Options& Ops) { Ops.add_option("", "", @@ -2191,26 +415,26 @@ BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops) " false = the local content will be replaced by the remote content\n" " true = the remote data will be overlayed on top of local data\n" "Defaults to false.", - cxxopts::value(m_AppendNewContent), + cxxopts::value(AppendNewContent), "<append>"); } BuildsCommand::BuildsCommand() -: m_ListNamespacesSubCmd(*this) -, m_ListSubCmd(*this) -, m_ListBlocksSubCmd(*this) -, m_UploadSubCmd(*this) -, m_DownloadSubCmd(*this) -, m_LsSubCmd(*this) -, m_DiffSubCmd(*this) -, m_FetchBlobSubCmd(*this) -, m_PrimeCacheSubCmd(*this) -, m_PauseSubCmd(*this) -, m_ResumeSubCmd(*this) -, m_AbortSubCmd(*this) -, m_ValidatePartSubCmd(*this) -, m_TestSubCmd(*this) -, m_MultiTestDownloadSubCmd(*this) +: m_ListNamespacesSubCmd(m_Configuration) +, m_ListSubCmd(m_Configuration) +, m_ListBlocksSubCmd(m_Configuration) +, m_UploadSubCmd(m_Configuration) +, m_DownloadSubCmd(m_Configuration) +, m_LsSubCmd(m_Configuration) +, m_DiffSubCmd(m_Configuration) +, m_FetchBlobSubCmd(m_Configuration) +, m_PrimeCacheSubCmd(m_Configuration) +, m_PauseSubCmd(m_Configuration) +, m_ResumeSubCmd(m_Configuration) +, m_AbortSubCmd(m_Configuration) +, m_ValidatePartSubCmd(m_Configuration) +, m_TestSubCmd(m_Configuration) +, m_MultiTestDownloadSubCmd(m_Configuration) { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("__hidden__", "", "subcommand", "", cxxopts::value<std::string>(m_SubCommand)->default_value(""), ""); @@ -2246,158 +470,222 @@ BuildsCommand::OnParentOptionsParsed(const ZenCliOptions& /*GlobalOptions*/) #endif // ZEN_PLATFORM_WINDOWS // Validate output options - if (m_Verbose && m_Quiet) + if (m_Configuration.Verbose && m_Configuration.Quiet) { throw OptionParseException("'--verbose' conflicts with '--quiet'", {}); } - if (m_LogProgress && m_PlainProgress) + if (m_Configuration.LogProgress && m_Configuration.PlainProgress) { throw OptionParseException("'--plain-progress' conflicts with '--log-progress'", {}); } - if (m_LogProgress && m_Quiet) + if (m_Configuration.LogProgress && m_Configuration.Quiet) { throw OptionParseException("'--quiet' conflicts with '--log-progress'", {}); } - if (m_PlainProgress && m_Quiet) + if (m_Configuration.PlainProgress && m_Configuration.Quiet) { throw OptionParseException("'--quiet' conflicts with '--plain-progress'", {}); } - IsVerbose = m_Verbose; - IsQuiet = m_Quiet; - if (m_LogProgress) + if (m_Configuration.LogProgress) { - ProgressMode = ProgressBar::Mode::Log; + m_Configuration.ProgressMode = ConsoleProgressMode::Log; } - else if (m_PlainProgress) + else if (m_Configuration.PlainProgress) { - ProgressMode = ProgressBar::Mode::Plain; + m_Configuration.ProgressMode = ConsoleProgressMode::Plain; } - else if (IsQuiet) + else if (m_Configuration.Quiet) { - ProgressMode = ProgressBar::Mode::Quiet; + m_Configuration.ProgressMode = ConsoleProgressMode::Quiet; } else { - ProgressMode = ProgressBar::Mode::Pretty; + m_Configuration.ProgressMode = ConsoleProgressMode::Pretty; } - if (m_BoostWorkers) + if (m_Configuration.BoostWorkers) { - m_BoostWorkerCount = true; - m_BoostWorkerMemory = true; + m_Configuration.BoostWorkerCount = true; + m_Configuration.BoostWorkerMemory = true; } // Parse system options - if (m_SystemRootDir.empty()) + if (m_Configuration.SystemRootDir.empty()) { - m_SystemRootDir = PickDefaultSystemRootDirectory(); + m_Configuration.SystemRootDir = PickDefaultSystemRootDirectory(); } - MakeSafeAbsolutePathInPlace(m_SystemRootDir); - - UseSparseFiles = m_UseSparseFiles; + MakeSafeAbsolutePathInPlace(m_Configuration.SystemRootDir); + MakeSafeAbsolutePathInPlace(m_Configuration.ChunkingCachePath); return true; } +std::atomic<bool>& +BuildsSubCmdBase::AbortFlag() const +{ + return builds_impl::AbortFlag; +} + +std::atomic<bool>& +BuildsSubCmdBase::PauseFlag() const +{ + return builds_impl::PauseFlag; +} + +std::unique_ptr<ProgressBase> +BuildsSubCmdBase::CreateProgress() const +{ + return std::unique_ptr<ProgressBase>(CreateConsoleProgress(m_Config.ProgressMode)); +} + +////////////////////////////////////////////////////////////////////////// + +void +BuildsSubCmdBase::LogBanner() +{ + if (!m_Config.Quiet) + { + ZenCmdBase::LogExecutableVersionAndPid(); + } +} + +void +BuildsSubCmdBase::LogWorkersInfo(const TransferThreadWorkers& Workers) +{ + if (!m_Config.Quiet) + { + ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); + } +} + void -BuildsCommand::ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts) +BuildsSubCmdBase::CleanZenFolder() { - if (!m_Url.empty()) + CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), AbortFlag(), PauseFlag(), GetZenFolderPath()); +} + +////////////////////////////////////////////////////////////////////////// + +BuildsSubCmdBase::ResolvedStorage +BuildsSubCmdBase::ParseStorageOptions(std::string& BuildId, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + const std::filesystem::path& SystemRootDirOverride, + const std::filesystem::path& StoragePathOverride) +{ + ResolvedStorage Resolved{.SystemRootDir = SystemRootDirOverride.empty() ? m_Config.SystemRootDir : SystemRootDirOverride, + .Host = m_Config.Host, + .Namespace = m_Config.Namespace, + .Bucket = m_Config.Bucket, + .StoragePath = StoragePathOverride.empty() ? m_Config.StoragePath : StoragePathOverride}; + + if (!m_Config.Url.empty()) { - if (!m_Host.empty()) + if (!Resolved.Host.empty()) { - throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", m_Host, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", Resolved.Host, m_Config.Url), + SubOpts.help()); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", m_Bucket, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", Resolved.Bucket, m_Config.Url), + SubOpts.help()); } if (!BuildId.empty()) { - throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Config.Url), + SubOpts.help()); } - if (!ParseBuildStorageUrl(m_Url, m_Host, m_Namespace, m_Bucket, BuildId)) + if (!ParseBuildStorageUrl(m_Config.Url, Resolved.Host, Resolved.Namespace, Resolved.Bucket, BuildId)) { throw OptionParseException("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", SubOpts.help()); } } - if (!m_OverrideHost.empty() || !m_Host.empty()) + if (!m_Config.OverrideHost.empty() || !Resolved.Host.empty()) { - if (!m_StoragePath.empty()) + if (!Resolved.StoragePath.empty()) { throw OptionParseException( - fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", m_StoragePath), + fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", Resolved.StoragePath), SubOpts.help()); } - if (RequireNamespace && m_Namespace.empty()) + if (Options.RequireNamespace && Resolved.Namespace.empty()) { throw OptionParseException("'--namespace' is required", SubOpts.help()); } - if (RequireBucket && m_Bucket.empty()) + if (Options.RequireBucket && Resolved.Bucket.empty()) { throw OptionParseException("'--bucket' is required", SubOpts.help()); } } - else if (m_StoragePath.empty()) + else if (Resolved.StoragePath.empty()) { throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help()); } - MakeSafeAbsolutePathInPlace(m_StoragePath); + MakeSafeAbsolutePathInPlace(Resolved.StoragePath); + return Resolved; } StorageInstance -BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, - BuildStorageCache::Statistics& StorageCacheStats, - const std::filesystem::path& TempPath, - std::string& BuildId, - bool RequireNamespace, - bool RequireBucket, - bool BoostCacheBackgroundWorkerPool, - std::unique_ptr<AuthMgr>& Auth, - cxxopts::Options& SubOpts) +BuildsSubCmdBase::CreateBuildStorage(const std::filesystem::path& ZenFolderDefault, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + BuildStorageBase::Statistics& StorageStats, + BuildStorageCache::Statistics& StorageCacheStats, + std::unique_ptr<AuthMgr>& Auth, + const ResolvedStorage& Resolved) { using namespace builds_impl; - ParseStorageOptions(BuildId, RequireNamespace, RequireBucket, SubOpts); + ResolveZenFolderPath(ZenFolderDefault); + CreateDirectories(GetZenFolderPath()); + const std::filesystem::path TempPath = ZenTempFolderPath(GetZenFolderPath()); - HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", - .AssumeHttp2 = m_AssumeHttp2, - .AllowResume = true, - .RetryCount = 2, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}; + HttpClientSettings ClientSettings{ + .LogCategory = "httpbuildsclient", + .AssumeHttp2 = m_Config.AssumeHttp2, + .AllowResume = true, + .RetryCount = 2, + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)}; std::string StorageDescription; std::string CacheDescription; StorageInstance Result; - if (!m_Host.empty() || !m_OverrideHost.empty()) - { - m_AuthOptions.ParseOptions(SubOpts, - m_SystemRootDir, - ClientSettings, - m_Host.empty() ? m_OverrideHost : m_Host, - Auth, - IsQuiet, - /*Hidden*/ false, - m_Verbose); - - BuildStorageResolveResult ResolveRes = - ResolveBuildStorage(ConsoleLog(), ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All, m_Verbose); + if (!Resolved.Host.empty() || !m_Config.OverrideHost.empty()) + { + AuthCommandLineOptions AuthOpts = m_Config.AuthOptions; + AuthOpts.ParseOptions(SubOpts, + Resolved.SystemRootDir, + ClientSettings, + Resolved.Host.empty() ? m_Config.OverrideHost : Resolved.Host, + Auth, + m_Config.Quiet, + /*Hidden*/ false, + m_Config.Verbose); + + BuildStorageResolveResult ResolveRes = ResolveBuildStorage(ConsoleLog(), + ClientSettings, + Resolved.Host, + m_Config.OverrideHost, + m_Config.ZenCacheHost, + ZenCacheResolveMode::All, + m_Config.Verbose); if (!ResolveRes.Cloud.Address.empty()) { ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2; Result.BuildStorageHttp = - std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); }); + std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, [this]() { return AbortFlag().load(); }); Result.BuildStorage = CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, - m_Namespace, - m_Bucket, - m_AllowRedirect, + Resolved.Namespace, + Resolved.Bucket, + m_Config.AllowRedirect, TempPath / "storage"); Result.BuildStorageHost = ResolveRes.Cloud; @@ -2408,8 +696,8 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, ResolveRes.Cloud.Name, (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address), Result.BuildStorageHttp->GetSessionId(), - m_Namespace, - m_Bucket, + Resolved.Namespace, + Resolved.Bucket, NiceLatencyNs(HostLatencyNs)); if (!ResolveRes.Cache.Address.empty()) @@ -2423,17 +711,17 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2, .AllowResume = true, .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)}, + [this]() { return AbortFlag().load(); }); Result.CacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, - m_Namespace, - m_Bucket, + Resolved.Namespace, + Resolved.Bucket, TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); + Options.BoostCacheBackgroundWorkers ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); Result.CacheHost = ResolveRes.Cache; uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0; @@ -2445,69 +733,69 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, Result.CacheHttp->GetSessionId(), NiceLatencyNs(CacheLatencyNs)); - if (!m_Namespace.empty()) + if (!Resolved.Namespace.empty()) { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); + CacheDescription += fmt::format(". Namespace '{}'", Resolved.Namespace); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + CacheDescription += fmt::format(" Bucket '{}'", Resolved.Bucket); } } } } - else if (!m_StoragePath.empty()) + else if (!Resolved.StoragePath.empty()) { - StorageDescription = fmt::format("folder {}", m_StoragePath); - Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + StorageDescription = fmt::format("folder {}", Resolved.StoragePath); + Result.BuildStorage = CreateFileBuildStorage(Resolved.StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(), + Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = Resolved.StoragePath.generic_string(), .Name = "Disk", .LatencySec = 1.0 / 100000, // 1 us .Caps = {.MaxRangeCountPerRequest = 2048u}}; - if (!m_ZenCacheHost.empty()) + if (!m_Config.ZenCacheHost.empty()) { - ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp); + ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_Config.ZenCacheHost, m_Config.AssumeHttp2, m_Config.VerboseHttp); if (TestResult.Success) { Result.CacheHttp = std::make_unique<HttpClient>( - m_ZenCacheHost, + m_Config.ZenCacheHost, HttpClientSettings{ .LogCategory = "httpcacheclient", .ConnectTimeout = std::chrono::milliseconds{3000}, .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = m_AssumeHttp2, + .AssumeHttp2 = m_Config.AssumeHttp2, .AllowResume = true, .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)}, + [this]() { return AbortFlag().load(); }); Result.CacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, - m_Namespace, - m_Bucket, + Resolved.Namespace, + Resolved.Bucket, TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); - Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_ZenCacheHost, - .Name = m_ZenCacheHost, - .AssumeHttp2 = m_AssumeHttp2, + Options.BoostCacheBackgroundWorkers ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); + Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_Config.ZenCacheHost, + .Name = m_Config.ZenCacheHost, + .AssumeHttp2 = m_Config.AssumeHttp2, .LatencySec = TestResult.LatencySeconds, .Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}}; CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId()); - if (!m_Namespace.empty()) + if (!Resolved.Namespace.empty()) { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); + CacheDescription += fmt::format(". Namespace '{}'", Resolved.Namespace); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + CacheDescription += fmt::format(" Bucket '{}'", Resolved.Bucket); } } } @@ -2517,7 +805,7 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help()); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Remote: {}", StorageDescription); if (!Result.CacheHost.Name.empty()) @@ -2529,7 +817,7 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, } Oid -BuildsCommand::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts) { if (BuildIdStr.length() != Oid::StringLength) { @@ -2548,7 +836,7 @@ BuildsCommand::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& Sub } Oid -BuildsCommand::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts) { if (BuildPartIdStr.length() != Oid::StringLength) { @@ -2567,7 +855,7 @@ BuildsCommand::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Opti } std::vector<Oid> -BuildsCommand::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts) { std::vector<Oid> BuildPartIds; for (const std::string& BuildPartId : BuildPartIdStrs) @@ -2582,7 +870,7 @@ BuildsCommand::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs } std::vector<std::string> -BuildsCommand::ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts) { std::vector<std::string> BuildPartNames; for (const std::string& BuildPartName : BuildPartNameStrs) @@ -2597,10 +885,10 @@ BuildsCommand::ParseBuildPartNames(const std::vector<std::string>& BuildPartName } CbObject -BuildsCommand::ParseBuildMetadata(bool CreateBuild, - std::filesystem::path& BuildMetadataPath, - const std::string& BuildMetadata, - cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildMetadata(bool CreateBuild, + std::filesystem::path& BuildMetadataPath, + const std::string& BuildMetadata, + cxxopts::Options& SubOpts) { if (CreateBuild) { @@ -2659,7 +947,7 @@ BuildsCommand::ParseBuildMetadata(bool CreateBuild, } void -BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) { if (Path.empty()) { @@ -2669,7 +957,7 @@ BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) } IoHash -BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts) { if (BlobHashStr.empty()) { @@ -2693,7 +981,7 @@ BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& S } void -BuildsCommand::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) +BuildsSubCmdBase::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) { auto SplitAndAppendWildcard = [](const std::string_view Wildcard, std::vector<std::string>& Output) { ForEachStrTok(Wildcard, ';', [&Output](std::string_view Wildcard) { @@ -2719,12 +1007,13 @@ BuildsCommand::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, s }); }; - SplitAndAppendWildcard(m_IncludeWildcard, OutIncludeWildcards); - SplitAndAppendWildcard(m_ExcludeWildcard, OutExcludeWildcards); + SplitAndAppendWildcard(m_Config.IncludeWildcard, OutIncludeWildcards); + SplitAndAppendWildcard(m_Config.ExcludeWildcard, OutExcludeWildcards); } void -BuildsCommand::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions) +BuildsSubCmdBase::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, + std::vector<std::string>& OutExcludeExtensions) { auto SplitAndAppendExclusion = [](const std::string_view Input, std::vector<std::string>& Output) { ForEachStrTok(Input, ";,", [&Output](std::string_view Exclusion) { @@ -2741,34 +1030,31 @@ BuildsCommand::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExclu }); }; - SplitAndAppendExclusion(m_ExcludeFolders, OutExcludeFolders); - SplitAndAppendExclusion(m_ExcludeExtensions, OutExcludeExtensions); + SplitAndAppendExclusion(m_Config.ExcludeFolders, OutExcludeFolders); + SplitAndAppendExclusion(m_Config.ExcludeExtensions, OutExcludeExtensions); } void -BuildsCommand::ResolveZenFolderPath(const std::filesystem::path& DefaultPath) +BuildsSubCmdBase::ResolveZenFolderPath(const std::filesystem::path& DefaultPath) { - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = DefaultPath; - } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + m_ResolvedZenFolderPath = m_Config.ZenFolderPath.empty() ? DefaultPath : m_Config.ZenFolderPath; + MakeSafeAbsolutePathInPlace(m_ResolvedZenFolderPath); } EPartialBlockRequestMode -BuildsCommand::ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts) { - EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests); + EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_Config.AllowPartialBlockRequests); if (Mode == EPartialBlockRequestMode::Invalid) { - throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests), + throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_Config.AllowPartialBlockRequests), SubOpts.help()); } return Mode; } void -BuildsCommand::ParseZenProcessId(int& ZenProcessId) +BuildsSubCmdBase::ParseZenProcessId(int& ZenProcessId) { if (ZenProcessId == -1) { @@ -2789,20 +1075,15 @@ BuildsCommand::ParseZenProcessId(int& ZenProcessId) ////////////////////////////////////////////////////////////////////////// -// --------------------------------------------------------------------------- -// Subcommand implementations -// --------------------------------------------------------------------------- - -BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("list-namespaces", "List all namespaces and optionally their buckets") -, m_Parent(Parent) +BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "list-namespaces", "List all namespaces and optionally their buckets") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "recursive", "Enable fetch of buckets within namespaces also", cxxopts::value(m_Recursive), "<recursive>"); Opts.add_option("", "", @@ -2817,36 +1098,21 @@ BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent) void BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + cxxopts::Options& Opts = SubOptions(); - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ false, - /*RequireBucket*/ false, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + std::string DummyBuildId; + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {.RequireNamespace = false, .RequireBucket = false}, Opts); + StorageInstance Storage = + CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); CbObject Response = Storage.BuildStorage->ListNamespaces(m_Recursive); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); @@ -2858,29 +1124,20 @@ BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - std::filesystem::path ResultPath = MakeSafeAbsolutePath(m_ResultPath); - if (ToLower(ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(MakeSafeAbsolutePath(m_ResultPath), Response); } } -BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list", "List builds matching a query"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsListSubCmd::BuildsListSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "list", "List builds matching a query") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "query-path", @@ -2900,19 +1157,16 @@ BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list" void BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + if (!m_ResultPath.empty()) + { + LogBanner(); + } + + cxxopts::Options& Opts = SubOptions(); MakeSafeAbsolutePathInPlace(m_QueryPath); MakeSafeAbsolutePathInPlace(m_ResultPath); - if (!m_ResultPath.empty()) - { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - } std::string JsonQuery; if (m_QueryPath.empty()) { @@ -2949,25 +1203,14 @@ BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } } + std::string DummyBuildId; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ false, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {.RequireBucket = false}, Opts); + StorageInstance Storage = + CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); CbObject Response = Storage.BuildStorage->ListBuilds(JsonQuery); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); @@ -2979,28 +1222,20 @@ BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } } -BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("list-blocks", "List blocks for a build") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "list-blocks", "List blocks for a build") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); Opts.add_option("", "", @@ -3016,50 +1251,35 @@ BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent) void BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - MakeSafeAbsolutePathInPlace(m_ResultPath); - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } + cxxopts::Options& Opts = SubOptions(); + + MakeSafeAbsolutePathInPlace(m_ResultPath); + if (m_MaxCount == 0) { throw OptionParseException(fmt::format("'--max-count' ('{}') is invalid", m_MaxCount), Opts.help()); } BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + StorageInstance Storage = + CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); CbObject Response = Storage.BuildStorage->FindBlocks(BuildId, m_MaxCount); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); std::vector<ChunkBlockDescription> BlockDescriptions = ParseChunkBlockDescriptionList(Response); - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Response contains {} block", BlockDescriptions.size()); } @@ -3074,35 +1294,25 @@ BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } } -BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("upload", "Upload a folder to build storage") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "upload", "Upload a folder to build storage") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddExcludeFolderOption(Opts); - Parent.AddExcludeExtensionsOption(Opts); - Parent.AddChunkingCacheOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddExcludeFolderOption(Opts); + Config.AddExcludeExtensionsOption(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "", @@ -3150,7 +1360,7 @@ BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) cxxopts::value(m_UploadToZenCache), "<uploadtozencache>"); - Parent.AddMultipartOptions(Opts); + Config.AddMultipartOptions(Opts); Opts.add_option("", "", @@ -3175,50 +1385,30 @@ BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) void BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + cxxopts::Options& Opts = SubOptions(); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + ParsePath(m_Path, Opts); - ZenState InstanceState; - - m_Parent.ParsePath(m_Path, Opts); + builds_impl::ZenState InstanceState; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + StorageInstance Storage = + CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); if (m_BuildPartName.empty() && m_ManifestPath.empty()) { m_BuildPartName = m_Path.filename().string(); } - const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId(m_BuildId, Opts); if (m_BuildId.empty()) { m_BuildId = BuildId.ToString(); @@ -3227,29 +1417,31 @@ BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) Oid BuildPartId; if (!m_BuildPartId.empty()) { - BuildPartId = m_Parent.ParseBuildPartId(m_BuildPartId, Opts); + BuildPartId = ParseBuildPartId(m_BuildPartId, Opts); } - CbObject MetaData = m_Parent.ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts); + CbObject MetaData = ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts); - const std::filesystem::path TempDir = ZenTempFolderPath(m_Parent.GetZenFolderPath()); + const std::filesystem::path TempDir = ZenTempFolderPath(GetZenFolderPath()); std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); - std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); std::vector<std::pair<Oid, std::string>> UploadedParts = UploadFolder(ConsoleLog(), *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId, BuildPartId, m_BuildPartName, @@ -3261,68 +1453,79 @@ BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = TempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = m_CreateBuild, .IgnoreExistingBlocks = m_Clean, .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, .ExcludeFolders = ExcludeFolders, .ExcludeExtensions = ExcludeExtensions}); - if (!AbortFlag) + if (!AbortFlag()) { if (m_PostUploadVerify) { for (const auto& Part : UploadedParts) { - ValidateBuildPart(ConsoleLog(), *Progress, Workers, *Storage.BuildStorage, BuildId, Part.first, Part.second); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + BuildId, + Part.first, + Part.second); } } } - if (true) + if (!m_Config.Quiet) { - if (!IsQuiet) - { - ZEN_CONSOLE( - "{}:\n" - "Read: {}\n" - "Write: {}\n" - "Requests: {}\n" - "Avg Request Time: {}\n" - "Avg I/O Time: {}", - Storage.BuildStorageHost.Name, - NiceBytes(StorageStats.TotalBytesRead.load()), - NiceBytes(StorageStats.TotalBytesWritten.load()), - StorageStats.TotalRequestCount.load(), - StorageStats.TotalExecutionTimeUs.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0, - StorageStats.TotalRequestCount.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0); - } + const BuildStorageResolveResult::Host& Host = Storage.BuildStorageHost; + const BuildStorageBase::Statistics& Stats = StorageStats; + ZEN_CONSOLE( + "{}:\n" + "Read: {}\n" + "Write: {}\n" + "Requests: {}\n" + "Avg Request Time: {}\n" + "Avg I/O Time: {}", + Host.Name, + NiceBytes(Stats.TotalBytesRead.load()), + NiceBytes(Stats.TotalBytesWritten.load()), + Stats.TotalRequestCount.load(), + Stats.TotalExecutionTimeUs.load() > 0 + ? NiceTimeSpanMs(Stats.TotalExecutionTimeUs.load() / 1000 / Stats.TotalRequestCount.load()) + : 0, + Stats.TotalRequestCount.load() > 0 ? NiceTimeSpanMs(Stats.TotalRequestTimeUs.load() / 1000 / Stats.TotalRequestCount.load()) + : 0); } - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Upload aborted"); } } -BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("download", "Download a build to a local folder") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "download", "Download a build to a local folder") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddWildcardOptions(Opts); - Parent.AddAppendNewContentOptions(Opts); - Parent.AddExcludeFolderOption(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddWildcardOptions(Opts); + Config.AddAppendNewContentOptions(Opts); + Config.AddExcludeFolderOption(Opts); Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); @@ -3357,9 +1560,9 @@ BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) "Upload data downloaded from remote host to zen cache", cxxopts::value(m_UploadToZenCache), "<uploadtozencache>"); - Parent.AddMultipartOptions(Opts); + Config.AddMultipartOptions(Opts); - Parent.AddPartialBlockRequestOptions(Opts); + Config.AddPartialBlockRequestOptions(Opts); Opts.add_option( "", @@ -3389,76 +1592,61 @@ BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) void BuildsDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - ZenState InstanceState; + cxxopts::Options& Opts = SubOptions(); - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); std::vector<std::string> IncludeWildcards; std::vector<std::string> ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); + ParseFileFilters(IncludeWildcards, ExcludeWildcards); - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); + builds_impl::ZenState InstanceState; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + StorageInstance Storage = CreateBuildStorage(m_Path / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); - std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(Opts); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); - if (m_Parent.m_AppendNewContent && m_Clean) + if (m_Config.AppendNewContent && m_Clean) { throw OptionParseException("'--append' conflicts with '--clean'", Opts.help()); } std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); - std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); DownloadFolder( ConsoleLog(), *Progress, Workers, Storage, - StorageCacheStats, + AbortFlag(), + PauseFlag(), + CacheStats, BuildId, BuildPartIds, BuildPartNames, m_DownloadSpecPath, m_Path, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, - .ZenFolderPath = m_Parent.GetZenFolderPath(), - .AllowMultiparts = m_Parent.m_AllowMultiparts, + DownloadOptions{.SystemRootDir = m_Config.SystemRootDir, + .ZenFolderPath = GetZenFolderPath(), + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = m_Clean, .PostDownloadVerify = m_PostDownloadVerify, @@ -3467,28 +1655,33 @@ BuildsDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) .AllowFileClone = m_AllowFileClone, .IncludeWildcards = IncludeWildcards, .ExcludeWildcards = ExcludeWildcards, - .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Parent.m_BoostWorkerMemory), + .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory), .PopulateCache = m_UploadToZenCache, - .AppendNewContent = m_Parent.m_AppendNewContent, + .AppendNewContent = m_Config.AppendNewContent, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles, .ExcludeFolders = ExcludeFolders}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Download aborted"); } } -BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "List files in a build"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsLsSubCmd::BuildsLsSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "ls", "List files in a build") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddWildcardOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddWildcardOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); Opts.add_option("", @@ -3526,43 +1719,31 @@ BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "Lis void BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } - ZenState InstanceState; + cxxopts::Options& Opts = SubOptions(); + + builds_impl::ZenState InstanceState; std::vector<std::string> IncludeWildcards; std::vector<std::string> ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); - - m_Parent.ResolveZenFolderPath(m_Parent.m_StoragePath); // ls uses storage path context + ParseFileFilters(IncludeWildcards, ExcludeWildcards); + // ls uses storage path context BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + StorageInstance Storage = CreateBuildStorage(m_Config.StoragePath, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); - std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); std::unique_ptr<CbObjectWriter> StructuredOutput; if (!m_ResultPath.empty()) @@ -3571,38 +1752,30 @@ BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) StructuredOutput = std::make_unique<CbObjectWriter>(); } - ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get()); + ListBuild(m_Config.Quiet, Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get()); if (StructuredOutput) { CbObject Response = StructuredOutput->Save(); - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("List build aborted"); } } -BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff", "Diff two local folders"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "diff", "Diff two local folders") { - auto& Opts = SubOptions(); - Parent.AddOutputOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddExcludeFolderOption(Opts); - Parent.AddExcludeExtensionsOption(Opts); - Parent.AddChunkingCacheOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddOutputOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddExcludeFolderOption(Opts); + Config.AddExcludeExtensionsOption(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>"); Opts.add_option("", @@ -3618,38 +1791,28 @@ BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff" void BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + cxxopts::Options& Opts = SubOptions(); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } - - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); if (m_DiffPath.empty()) { throw OptionParseException("'--compare-path' is required", Opts.help()); } MakeSafeAbsolutePathInPlace(m_DiffPath); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); StandardChunkingControllerSettings ChunkingSettings; std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(ChunkingSettings); - std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); if (m_OnlyChunked) { @@ -3661,24 +1824,37 @@ BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ChunkingSettings.SplitAndCompressExtensions.end()); } - DiffFolders(Workers, m_Path, m_DiffPath, *ChunkController, *ChunkCache, ExcludeFolders, ExcludeExtensions); - if (AbortFlag) + std::unique_ptr<ProgressBase> Progress = CreateProgress(); + + DiffFolders(*Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + Workers, + m_Path, + m_DiffPath, + *ChunkController, + *ChunkCache, + ExcludeFolders, + ExcludeExtensions); + if (AbortFlag()) { throw std::runtime_error("Diff folders aborted"); } } -BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("fetch-blob", "Fetch and validate a specific blob") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "fetch-blob", "Fetch and validate a specific blob") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); Opts.add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), "<blob-hash>"); Opts.parse_positional({"build-id", "blob-hash"}); @@ -3688,68 +1864,47 @@ BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent) void BuildsFetchBlobSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + StorageInstance Storage = + CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - IoHash BlobHash = m_Parent.ParseBlobHash(m_BlobHash, Opts); - - const Oid BuildId = Oid::FromHexString(m_BuildId); + IoHash BlobHash = ParseBlobHash(m_BlobHash, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); uint64_t CompressedSize; uint64_t DecompressedSize; - ValidateBlob(AbortFlag, *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); - if (AbortFlag) + ValidateBlob(AbortFlag(), *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); + if (AbortFlag()) { throw std::runtime_error("Fetch blob aborted"); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", BlobHash, CompressedSize, DecompressedSize); } } -BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("prime-cache", "Prime the zen cache with build data") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "prime-cache", "Prime the zen cache with build data") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); Opts.add_option("", "", @@ -3777,47 +1932,33 @@ BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent) void BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ true, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); - - std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + StorageInstance Storage = CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, + {.BoostCacheBackgroundWorkers = true}, + Opts, + StorageStats, + CacheStats, + Auth, + Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); + + const Oid BuildId = ParseBuildId(m_BuildId, Opts); + + std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); + CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId, m_Config.Quiet); std::vector<std::pair<Oid, std::string>> AllBuildParts = ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); @@ -3829,101 +1970,101 @@ BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) AllBuildPartIds.push_back(BuildPart.first); } - ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache"); - - std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); + Progress->SetLogOperationName("Prime Cache"); BuildsOperationPrimeCache PrimeOp(ConsoleLog(), *Progress, Storage, - AbortFlag, - PauseFlag, + AbortFlag(), + PauseFlag(), Workers.GetNetworkPool(), BuildId, AllBuildPartIds, - BuildsOperationPrimeCache::Options{.IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .ZenFolderPath = m_Parent.GetZenFolderPath(), + BuildsOperationPrimeCache::Options{.IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .ZenFolderPath = GetZenFolderPath(), .LargeAttachmentSize = PreferredMultipartChunkSize * 4u, .PreferredMultipartChunkSize = PreferredMultipartChunkSize, .ForceUpload = m_Force}, - StorageCacheStats); + CacheStats); PrimeOp.Execute(); - if (!IsQuiet) + if (!m_Config.Quiet && Storage.CacheStorage) { - if (Storage.CacheStorage) - { - ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", - StorageCacheStats.PutBlobCount.load(), - NiceBytes(StorageCacheStats.PutBlobByteCount), - Storage.CacheHost.Name); - } + ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", + CacheStats.PutBlobCount.load(), + NiceBytes(CacheStats.PutBlobByteCount), + Storage.CacheHost.Name); } } -BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("pause", "Pause a running zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +namespace { + void AddProcessIdOption(cxxopts::Options& Opts, int& ZenProcessId) + { + Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(ZenProcessId), "<pid>"); + Opts.parse_positional({"process-id"}); + Opts.positional_help("process-id"); + } +} // namespace + +BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "pause", "Pause a running zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsPauseSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Pause.store(true); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(1); } -BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("resume", "Resume a paused zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "resume", "Resume a paused zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsResumeSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Pause.store(false); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(0); } -BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("abort", "Abort a running zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "abort", "Abort a running zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsAbortSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Abort.store(true); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Abort.store(1); } -BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("validate-part", "Validate a build part") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "validate-part", "Validate a build part") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); Opts.add_option("", "", @@ -3944,42 +2085,23 @@ BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent) void BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } - - ZenState InstanceState; + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + StorageInstance Storage = + CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + builds_impl::ZenState InstanceState; - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + Oid BuildId = ParseBuildId(m_BuildId, Opts); if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) { @@ -3988,33 +2110,45 @@ BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) Opts.help()); } - const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : m_Parent.ParseBuildPartId(m_BuildPartId, Opts); + const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(m_BuildPartId, Opts); - std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); - ValidateBuildPart(ConsoleLog(), *Progress, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + BuildId, + BuildPartId, + m_BuildPartName); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Validate build part failed"); } } -BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test", "Run an upload/download test cycle"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsTestSubCmd::BuildsTestSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "test", "Run an upload/download test cycle") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); - Parent.AddMultipartOptions(Opts); - Parent.AddPartialBlockRequestOptions(Opts); - Parent.AddWildcardOptions(Opts); - Parent.AddAppendNewContentOptions(Opts); - Parent.AddChunkingCacheOptions(Opts); + Config.AddMultipartOptions(Opts); + Config.AddPartialBlockRequestOptions(Opts); + Config.AddWildcardOptions(Opts); + Config.AddAppendNewContentOptions(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "", "enable-scavenge", @@ -4034,38 +2168,35 @@ BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test" void BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(m_Parent.m_SystemRootDir); - CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); - auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); }); + m_TestSystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_TestSystemRootDir); + CleanDirectory(m_TestSystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); + auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_TestSystemRootDir); }); - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); - if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty()) + const bool CreatedTempStorage = m_Config.OverrideHost.empty() && m_Config.StoragePath.empty(); + if (CreatedTempStorage) { - m_Parent.m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.m_StoragePath); - CreateDirectories(m_Parent.m_StoragePath); - m_Parent.m_StoragePath = m_Parent.m_StoragePath.generic_string(); + m_TestStoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), m_TestStoragePath); + CreateDirectories(m_TestStoragePath); + m_TestStoragePath = m_TestStoragePath.generic_string(); } - auto StorageGuard = MakeGuard([this]() { - if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty()) + auto StorageGuard = MakeGuard([this, CreatedTempStorage]() { + if (CreatedTempStorage) { - DeleteDirectories(m_Parent.m_StoragePath); + DeleteDirectories(m_TestStoragePath); } }); - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(Opts); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); BuildStorageBase::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -4076,30 +2207,20 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2"); const std::filesystem::path DownloadPath3 = m_Path.parent_path() / (m_BuildPartName + "_test3"); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath2); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath3); - auto DownloadGuard = MakeGuard([&Workers, DownloadPath, DownloadPath2, DownloadPath3]() { - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3); + auto DownloadGuard = MakeGuard([this, &Workers, DownloadPath, DownloadPath2, DownloadPath3]() { + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath2); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath3); }); - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - std::unique_ptr<AuthMgr> Auth; std::string TestBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - TestBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + const ResolvedStorage Resolved = ParseStorageOptions(TestBuildId, {}, Opts, m_TestSystemRootDir, m_TestStoragePath); + StorageInstance Storage = CreateBuildStorage(m_Path / ZenFolderName, {}, Opts, StorageStats, StorageCacheStats, Auth, Resolved); m_BuildId = Oid::NewOid().ToString(); m_BuildPartId = Oid::NewOid().ToString(); @@ -4129,19 +2250,21 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView()); } - const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path); + const std::filesystem::path UploadTempDir = builds_impl::UploadTempDirectory(m_Path); std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); - std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); UploadFolder(ConsoleLog(), *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId, BuildPartId, m_BuildPartName, @@ -4153,12 +2276,14 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload build)"); } @@ -4170,6 +2295,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), Oid::NewOid(), Oid::NewOid(), m_BuildPartName, @@ -4181,42 +2308,56 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload again, chunking is cached)"); } } - ValidateBuildPart(ConsoleLog(), *Progress, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + BuildId, + BuildPartId, + m_BuildPartName); - if (!m_Parent.m_IncludeWildcard.empty() || !m_Parent.m_ExcludeWildcard.empty()) + if (!m_Config.IncludeWildcard.empty() || !m_Config.ExcludeWildcard.empty()) { - auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); }); + auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); }); ZEN_CONSOLE("\nDownload Filtered Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); std::vector<std::string> IncludeWildcards; std::vector<std::string> ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); + ParseFileFilters(IncludeWildcards, ExcludeWildcards); DownloadFolder(ConsoleLog(), *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, @@ -4225,8 +2366,11 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) .AllowFileClone = m_AllowFileClone, .IncludeWildcards = IncludeWildcards, .ExcludeWildcards = ExcludeWildcards, - .AppendNewContent = false}); - if (AbortFlag) + .AppendNewContent = false, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } @@ -4240,15 +2384,17 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, @@ -4257,8 +2403,11 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) .AllowFileClone = m_AllowFileClone, .IncludeWildcards = ExcludeWildcards, .ExcludeWildcards = IncludeWildcards, - .AppendNewContent = true}); - if (AbortFlag) + .AppendNewContent = true, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } @@ -4268,15 +2417,17 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, @@ -4285,8 +2436,11 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) .AllowFileClone = m_AllowFileClone, .IncludeWildcards = {}, .ExcludeWildcards = {}, - .AppendNewContent = false}); - if (AbortFlag) + .AppendNewContent = false, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } @@ -4297,6 +2451,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4304,16 +2460,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = false, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } @@ -4323,6 +2482,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4330,16 +2491,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download identical target)"); } @@ -4371,7 +2535,7 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) return true; }; - ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + ParallelWork Work(AbortFlag(), PauseFlag(), WorkerThreadPool::EMode::EnableBacklog); uint32_t Randomizer = 0; auto FileSizeIt = DownloadContent.FileSizes.begin(); @@ -4389,8 +2553,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { Work.ScheduleWork( Workers.GetIOWorkerPool(), - [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) { - if (!AbortFlag) + [this, SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) { + if (!AbortFlag()) { bool WasReadOnly = SetFileReadOnly(FilePath, false); { @@ -4431,7 +2595,7 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ZEN_UNUSED(IsAborted, IsPaused); ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork); }); - ZEN_ASSERT(!AbortFlag.load()); + ZEN_ASSERT(!AbortFlag().load()); ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }; @@ -4441,6 +2605,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4448,16 +2614,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download scrambled target)"); } @@ -4478,6 +2647,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId2, BuildPartId2, m_BuildPartName, @@ -4489,23 +2660,37 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload scrambled)"); } - ValidateBuildPart(ConsoleLog(), *Progress, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + BuildId, + BuildPartId, + m_BuildPartName); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); DownloadFolder(ConsoleLog(), *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4513,16 +2698,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } @@ -4532,22 +2720,27 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId2, {BuildPartId2}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download scrambled)"); } @@ -4557,22 +2750,27 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId2, {BuildPartId2}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download scrambled)"); } @@ -4582,22 +2780,27 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath2, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath2 / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } @@ -4607,38 +2810,44 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath3, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath3 / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } } -BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("multi-test-download", "Download multiple builds sequentially as a test") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "multi-test-download", "Download multiple builds sequentially as a test") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), "<ids>"); Opts.add_option("", @@ -4660,42 +2869,30 @@ BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Pare void BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } - - m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(m_Parent.m_SystemRootDir); - CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); - auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); }); - - m_Parent.ParsePath(m_Path, Opts); + cxxopts::Options& Opts = SubOptions(); - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); + m_TestSystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_TestSystemRootDir); + CleanDirectory(m_TestSystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); + auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_TestSystemRootDir); }); - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(Opts); + ParsePath(m_Path, Opts); + std::string DummyBuildId; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {}, Opts, m_TestSystemRootDir); + StorageInstance Storage = CreateBuildStorage(m_Path / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - std::unique_ptr<AuthMgr> Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode)); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); + + std::unique_ptr<ProgressBase> Progress = CreateProgress(); Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) @@ -4709,31 +2906,36 @@ BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) *Progress, Workers, Storage, - StorageCacheStats, + AbortFlag(), + PauseFlag(), + CacheStats, BuildId, /*BuildPartIds,*/ {}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, m_Path, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, - .ZenFolderPath = m_Parent.GetZenFolderPath(), - .AllowMultiparts = m_Parent.m_AllowMultiparts, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, + .ZenFolderPath = GetZenFolderPath(), + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = BuildIdString == m_BuildIds.front(), .PostDownloadVerify = true, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = false, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Multitest aborted"); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("\n"); } } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } |