diff options
Diffstat (limited to 'src/zen/cmds/builds_cmd.cpp')
| -rw-r--r-- | src/zen/cmds/builds_cmd.cpp | 3871 |
1 files changed, 1025 insertions, 2846 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index cc8315e0b..84d8424aa 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -4,68 +4,34 @@ #include <zencore/basicfile.h> #include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinaryfile.h> -#include <zencore/compactbinaryfmt.h> -#include <zencore/compactbinaryvalue.h> -#include <zencore/compress.h> -#include <zencore/except.h> +#include <zencore/except_fmt.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> #include <zencore/parallelwork.h> +#include <zencore/process.h> #include <zencore/scopeguard.h> #include <zencore/session.h> -#include <zencore/stream.h> #include <zencore/string.h> -#include <zencore/trace.h> -#include <zencore/uid.h> +#include <zenhttp/auth/authmgr.h> #include <zenhttp/formatters.h> -#include <zenhttp/httpclient.h> -#include <zenhttp/httpclientauth.h> #include <zenhttp/httpcommon.h> -#include <zenremotestore/builds/buildcontent.h> -#include <zenremotestore/builds/buildmanifest.h> -#include <zenremotestore/builds/buildsavedstate.h> -#include <zenremotestore/builds/buildstoragecache.h> -#include <zenremotestore/builds/buildstorageoperations.h> -#include <zenremotestore/builds/buildstorageutil.h> +#include <zenremotestore/builds/buildinspect.h> +#include <zenremotestore/builds/buildprimecache.h> +#include <zenremotestore/builds/buildupdatefolder.h> +#include <zenremotestore/builds/builduploadfolder.h> +#include <zenremotestore/builds/buildvalidatebuildpart.h> #include <zenremotestore/builds/filebuildstorage.h> #include <zenremotestore/builds/jupiterbuildstorage.h> -#include <zenremotestore/chunking/chunkblock.h> -#include <zenremotestore/chunking/chunkedcontent.h> -#include <zenremotestore/chunking/chunkedfile.h> #include <zenremotestore/chunking/chunkingcache.h> #include <zenremotestore/chunking/chunkingcontroller.h> -#include <zenremotestore/filesystemutils.h> -#include <zenremotestore/jupiter/jupiterhost.h> -#include <zenremotestore/operationlogoutput.h> -#include <zenremotestore/transferthreadworkers.h> +#include <zenutil/filesystemutils.h> +#include <zenutil/progress.h> #include <zenutil/wildcard.h> #include <zenutil/workerpools.h> -#include <zenutil/zenserverprocess.h> - -#include "../progressbar.h" #include <signal.h> #include <memory> -#include <numeric> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <tsl/robin_map.h> -#include <tsl/robin_set.h> -#include <json11.hpp> -ZEN_THIRD_PARTY_INCLUDES_END - -#if ZEN_PLATFORM_WINDOWS -# include <zencore/windows.h> -#else -# include <fcntl.h> -# include <sys/file.h> -# include <sys/stat.h> -# include <unistd.h> -#endif - -static const bool DoExtraContentVerify = false; namespace zen { @@ -115,14 +81,6 @@ namespace builds_impl { } }; - struct MemMap - { - void* Handle = nullptr; - void* Data = nullptr; - size_t Size = 0; - std::string Name; - }; - class ZenState { public: @@ -236,1869 +194,103 @@ namespace builds_impl { } } - const std::string ZenFolderName = ".zen"; - std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.cbo"; } - // std::filesystem::path ZenStateFileJsonPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.json"; - // } - - std::filesystem::path UploadTempDirectory(const std::filesystem::path& Path) - { - const std::u8string LocalPathString = Path.generic_u8string(); - IoHash PathHash = IoHash::HashBuffer(LocalPathString.data(), LocalPathString.length()); - return std::filesystem::temp_directory_path() / fmt::format("zen_{}", PathHash); - } - - const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; - - const std::string UnsyncFolderName = ".unsync"; - - const std::string UGSFolderName = ".ugs"; - const std::string LegacyZenTempFolderName = ".zen-tmp"; - - const std::vector<std::string> DefaultExcludeFolders({UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName}); - const std::vector<std::string> DefaultExcludeExtensions({}); - + // Debugging knobs for file build storage - always 0 in shipped builds const double DefaultLatency = 0; // .0010; const double DefaultDelayPerKBSec = 0; // 0.00005; - const bool SingleThreaded = false; - bool UseSparseFiles = false; - - static bool IsVerbose = false; - static bool IsQuiet = false; - static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; - -#undef ZEN_CONSOLE_VERBOSE -#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ - if (IsVerbose) \ - { \ - ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \ - } - - const std::string DefaultAccessTokenEnvVariableName( -#if ZEN_PLATFORM_WINDOWS - "UE-CloudDataCacheAccessToken"sv -#endif -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - "UE_CloudDataCacheAccessToken"sv -#endif - - ); - - static uint64_t GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory) - { - return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u; - } - - class FilteredRate - { - public: - FilteredRate() {} - - void Start() - { - if (StartTimeUS == (uint64_t)-1) - { - uint64_t Expected = (uint64_t)-1; - if (StartTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs())) - { - LastTimeUS = StartTimeUS.load(); - } - } - } - void Stop() - { - if (EndTimeUS == (uint64_t)-1) - { - uint64_t Expected = (uint64_t)-1; - EndTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs()); - } - } - - void Update(uint64_t Count) - { - if (LastTimeUS == (uint64_t)-1) - { - return; - } - uint64_t TimeUS = Timer.GetElapsedTimeUs(); - uint64_t TimeDeltaUS = TimeUS - LastTimeUS; - if (TimeDeltaUS >= 2000000) - { - uint64_t Delta = Count - LastCount; - uint64_t PerSecond = (Delta * 1000000) / TimeDeltaUS; - - LastPerSecond = PerSecond; - - LastCount = Count; - - FilteredPerSecond = (PerSecond + (LastPerSecond * 7)) / 8; - - LastTimeUS = TimeUS; - } - } - - uint64_t GetCurrent() const // If Stopped - return total count / total time - { - if (LastTimeUS == (uint64_t)-1) - { - return 0; - } - return FilteredPerSecond; - } - - uint64_t GetElapsedTimeUS() const - { - if (StartTimeUS == (uint64_t)-1) - { - return 0; - } - if (EndTimeUS == (uint64_t)-1) - { - return 0; - } - uint64_t TimeDeltaUS = EndTimeUS - StartTimeUS; - return TimeDeltaUS; - } - - bool IsActive() const { return (StartTimeUS != (uint64_t)-1) && (EndTimeUS == (uint64_t)-1); } - - private: - Stopwatch Timer; - std::atomic<uint64_t> StartTimeUS = (uint64_t)-1; - std::atomic<uint64_t> EndTimeUS = (uint64_t)-1; - std::atomic<uint64_t> LastTimeUS = (uint64_t)-1; - uint64_t LastCount = 0; - uint64_t LastPerSecond = 0; - uint64_t FilteredPerSecond = 0; - }; - - uint64_t GetBytesPerSecond(uint64_t ElapsedWallTimeUS, uint64_t Count) + void WriteResultObject(const std::filesystem::path& Path, const CbObject& Response) { - if (ElapsedWallTimeUS == 0) - { - return 0; - } - return Count * 1000000 / ElapsedWallTimeUS; - } - - bool CleanAndRemoveDirectory(WorkerThreadPool& WorkerPool, const std::filesystem::path& Directory) - { - return CleanAndRemoveDirectory(WorkerPool, AbortFlag, PauseFlag, Directory); - } - - void ValidateBuildPart(OperationLogOutput& Output, - TransferThreadWorkers& Workers, - BuildStorageBase& Storage, - const Oid& BuildId, - Oid BuildPartId, - const std::string_view BuildPartName) - { - ZEN_TRACE_CPU("ValidateBuildPart"); - - ProgressBar::SetLogOperationName(ProgressMode, "Validate Part"); - - BuildsOperationValidateBuildPart ValidateOp(Output, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - BuildPartId, - BuildPartName, - BuildsOperationValidateBuildPart::Options{.IsQuiet = IsQuiet, .IsVerbose = IsVerbose}); - - ValidateOp.Execute(); - - const uint64_t DownloadedCount = ValidateOp.m_DownloadStats.DownloadedChunkCount + ValidateOp.m_DownloadStats.DownloadedBlockCount; - const uint64_t DownloadedByteCount = - ValidateOp.m_DownloadStats.DownloadedChunkByteCount + ValidateOp.m_DownloadStats.DownloadedBlockByteCount; - ZEN_CONSOLE("Verified: {:>8} ({}), {}B/sec, {}", - DownloadedCount, - NiceBytes(DownloadedByteCount), - NiceNum(GetBytesPerSecond(ValidateOp.m_ValidateStats.ElapsedWallTimeUS, DownloadedByteCount)), - NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000)); - } - - struct UploadFolderOptions - { - std::filesystem::path TempDir; - uint64_t FindBlockMaxCount; - uint8_t BlockReuseMinPercentLimit; - bool AllowMultiparts; - bool CreateBuild; - bool IgnoreExistingBlocks; - bool UploadToZenCache; - const std::vector<std::string>& ExcludeFolders = DefaultExcludeFolders; - const std::vector<std::string>& ExcludeExtensions = DefaultExcludeExtensions; - }; - - std::vector<std::pair<Oid, std::string>> UploadFolder(OperationLogOutput& Output, - TransferThreadWorkers& Workers, - StorageInstance& Storage, - const Oid& BuildId, - const Oid& BuildPartId, - const std::string_view BuildPartName, - const std::filesystem::path& Path, - const std::filesystem::path& ManifestPath, - const CbObject& MetaData, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - const UploadFolderOptions& Options) - { - ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder"); - - Stopwatch UploadTimer; - - BuildsOperationUploadFolder UploadOp( - Output, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - Path, - Options.CreateBuild, - std::move(MetaData), - BuildsOperationUploadFolder::Options{.IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .DoExtraContentValidation = DoExtraContentVerify, - .FindBlockMaxCount = Options.FindBlockMaxCount, - .BlockReuseMinPercentLimit = Options.BlockReuseMinPercentLimit, - .AllowMultiparts = Options.AllowMultiparts, - .IgnoreExistingBlocks = Options.IgnoreExistingBlocks, - .TempDir = Options.TempDir, - .ExcludeFolders = Options.ExcludeFolders, - .ExcludeExtensions = Options.ExcludeExtensions, - .ZenExcludeManifestName = ZenExcludeManifestName, - .NonCompressableExtensions = DefaultSplitOnlyExtensions, - .PopulateCache = Options.UploadToZenCache}); - - std::vector<std::pair<Oid, std::string>> UploadedParts = - UploadOp.Execute(BuildPartId, BuildPartName, ManifestPath, ChunkController, ChunkCache); - if (AbortFlag) - { - return {}; - } - - ZEN_CONSOLE_VERBOSE( - "Folder scanning stats:" - "\n FoundFileCount: {}" - "\n FoundFileByteCount: {}" - "\n AcceptedFileCount: {}" - "\n AcceptedFileByteCount: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), - UploadOp.m_LocalFolderScanStats.AcceptedFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.AcceptedFileByteCount.load()), - NiceLatencyNs(UploadOp.m_LocalFolderScanStats.ElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Chunking stats:" - "\n FilesProcessed: {}" - "\n FilesChunked: {}" - "\n BytesHashed: {}" - "\n UniqueChunksFound: {}" - "\n UniqueSequencesFound: {}" - "\n UniqueBytesFound: {}" - "\n FilesFoundInCache: {}" - "\n ChunksFoundInCache: {}" - "\n FilesStoredInCache: {}" - "\n ChunksStoredInCache: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_ChunkingStats.FilesProcessed.load(), - UploadOp.m_ChunkingStats.FilesChunked.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesHashed.load()), - UploadOp.m_ChunkingStats.UniqueChunksFound.load(), - UploadOp.m_ChunkingStats.UniqueSequencesFound.load(), - NiceBytes(UploadOp.m_ChunkingStats.UniqueBytesFound.load()), - UploadOp.m_ChunkingStats.FilesFoundInCache.load(), - UploadOp.m_ChunkingStats.ChunksFoundInCache.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesFoundInCache.load()), - UploadOp.m_ChunkingStats.FilesStoredInCache.load(), - UploadOp.m_ChunkingStats.ChunksStoredInCache.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesStoredInCache.load()), - NiceLatencyNs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Find block stats:" - "\n FindBlockTimeMS: {}" - "\n PotentialChunkCount: {}" - "\n PotentialChunkByteCount: {}" - "\n FoundBlockCount: {}" - "\n FoundBlockChunkCount: {}" - "\n FoundBlockByteCount: {}" - "\n AcceptedBlockCount: {}" - "\n NewBlocksCount: {}" - "\n NewBlocksChunkCount: {}" - "\n NewBlocksChunkByteCount: {}", - NiceTimeSpanMs(UploadOp.m_FindBlocksStats.FindBlockTimeMS), - UploadOp.m_FindBlocksStats.PotentialChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.PotentialChunkByteCount), - UploadOp.m_FindBlocksStats.FoundBlockCount, - UploadOp.m_FindBlocksStats.FoundBlockChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.FoundBlockByteCount), - UploadOp.m_FindBlocksStats.AcceptedBlockCount, - UploadOp.m_FindBlocksStats.NewBlocksCount, - UploadOp.m_FindBlocksStats.NewBlocksChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount)); - - ZEN_CONSOLE_VERBOSE( - "Reuse block stats:" - "\n AcceptedChunkCount: {}" - "\n AcceptedByteCount: {}" - "\n AcceptedRawByteCount: {}" - "\n RejectedBlockCount: {}" - "\n RejectedChunkCount: {}" - "\n RejectedByteCount: {}" - "\n AcceptedReduntantChunkCount: {}" - "\n AcceptedReduntantByteCount: {}", - UploadOp.m_ReuseBlocksStats.AcceptedChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedByteCount), - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedRawByteCount), - UploadOp.m_ReuseBlocksStats.RejectedBlockCount, - UploadOp.m_ReuseBlocksStats.RejectedChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.RejectedByteCount), - UploadOp.m_ReuseBlocksStats.AcceptedReduntantChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedReduntantByteCount)); - - ZEN_CONSOLE_VERBOSE( - "Generate blocks stats:" - "\n GeneratedBlockByteCount: {}" - "\n GeneratedBlockCount: {}" - "\n GenerateBlocksElapsedWallTimeUS: {}", - NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), - UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), - NiceLatencyNs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Generate blocks stats:" - "\n ChunkCount: {}" - "\n ChunkByteCount: {}" - "\n CompressedChunkCount: {}" - "\n CompressChunksElapsedWallTimeUS: {}", - UploadOp.m_LooseChunksStats.ChunkCount, - NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount), - UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), - NiceLatencyNs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Disk stats:" - "\n OpenReadCount: {}" - "\n OpenWriteCount: {}" - "\n ReadCount: {}" - "\n ReadByteCount: {}" - "\n WriteCount: {} ({} cloned)" - "\n WriteByteCount: {} ({} cloned)" - "\n CurrentOpenFileCount: {}", - UploadOp.m_DiskStats.OpenReadCount.load(), - UploadOp.m_DiskStats.OpenWriteCount.load(), - UploadOp.m_DiskStats.ReadCount.load(), - NiceBytes(UploadOp.m_DiskStats.ReadByteCount.load()), - UploadOp.m_DiskStats.WriteCount.load(), - UploadOp.m_DiskStats.CloneCount.load(), - NiceBytes(UploadOp.m_DiskStats.WriteByteCount.load()), - NiceBytes(UploadOp.m_DiskStats.CloneByteCount.load()), - UploadOp.m_DiskStats.CurrentOpenFileCount.load()); - - ZEN_CONSOLE_VERBOSE( - "Upload stats:" - "\n BlockCount: {}" - "\n BlocksBytes: {}" - "\n ChunkCount: {}" - "\n ChunksBytes: {}" - "\n ReadFromDiskBytes: {}" - "\n MultipartAttachmentCount: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_UploadStats.BlockCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), - UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), - NiceBytes(UploadOp.m_UploadStats.ReadFromDiskBytes.load()), - UploadOp.m_UploadStats.MultipartAttachmentCount.load(), - NiceLatencyNs(UploadOp.m_UploadStats.ElapsedWallTimeUS * 1000)); - - const double DeltaByteCountPercent = - UploadOp.m_ChunkingStats.BytesHashed > 0 - ? (100.0 * (UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes)) / - (UploadOp.m_ChunkingStats.BytesHashed) - : 0.0; - - const std::string MultipartAttachmentStats = - Options.AllowMultiparts ? fmt::format(" ({} as multipart)", UploadOp.m_UploadStats.MultipartAttachmentCount.load()) : ""; - - if (!IsQuiet) - { - ZEN_CONSOLE( - "Uploaded part {} ('{}') to build {}, {}\n" - " Scanned files: {:>8} ({}), {}B/sec, {}\n" - " New data: {:>8} ({}) {:.1f}%\n" - " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n" - " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n" - " Uploaded: {:>8} ({}), {}bits/sec, {}\n" - " Blocks: {:>8} ({})\n" - " Chunks: {:>8} ({}){}", - BuildPartId, - BuildPartName, - BuildId, - NiceTimeSpanMs(UploadTimer.GetElapsedTimeMs()), - - UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_ChunkingStats.ElapsedWallTimeUS, UploadOp.m_ChunkingStats.BytesHashed)), - NiceTimeSpanMs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS / 1000), - - UploadOp.m_FindBlocksStats.NewBlocksChunkCount + UploadOp.m_LooseChunksStats.CompressedChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes), - DeltaByteCountPercent, - - UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount), - NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, - UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount)), - NiceTimeSpanMs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000), - - UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkRawBytes), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS, - UploadOp.m_LooseChunksStats.CompressedChunkRawBytes)), - NiceTimeSpanMs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000), - - UploadOp.m_UploadStats.BlockCount.load() + UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes + UploadOp.m_UploadStats.ChunksBytes), - NiceNum(GetBytesPerSecond(UploadOp.m_UploadStats.ElapsedWallTimeUS, - (UploadOp.m_UploadStats.ChunksBytes + UploadOp.m_UploadStats.BlocksBytes) * 8)), - NiceTimeSpanMs(UploadOp.m_UploadStats.ElapsedWallTimeUS / 1000), - - UploadOp.m_UploadStats.BlockCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), - - UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), - MultipartAttachmentStats); - } - return UploadedParts; - } - - struct VerifyFolderStatistics - { - std::atomic<uint64_t> FilesVerified = 0; - std::atomic<uint64_t> FilesFailed = 0; - std::atomic<uint64_t> ReadBytes = 0; - uint64_t VerifyElapsedWallTimeUs = 0; - }; - - void VerifyFolder(TransferThreadWorkers& Workers, - const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const std::filesystem::path& Path, - const std::vector<std::string>& ExcludeFolders, - bool VerifyFileHash, - VerifyFolderStatistics& VerifyFolderStats) - { - ZEN_TRACE_CPU("VerifyFolder"); - - Stopwatch Timer; - - ProgressBar ProgressBar(ProgressMode, "Verify Files"); - - WorkerThreadPool& VerifyPool = Workers.GetIOWorkerPool(); - - ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - - const uint32_t PathCount = gsl::narrow<uint32_t>(Content.Paths.size()); - - RwLock ErrorLock; - std::vector<std::string> Errors; - - auto IsAcceptedFolder = [ExcludeFolders = ExcludeFolders](const std::string_view& RelativePath) -> bool { - for (const std::string& ExcludeFolder : ExcludeFolders) - { - if (RelativePath.starts_with(ExcludeFolder)) - { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } - } - } - return true; - }; - - for (uint32_t PathIndex = 0; PathIndex < PathCount; PathIndex++) - { - if (Work.IsAborted()) - { - break; - } - - Work.ScheduleWork( - VerifyPool, - [&Path, &Content, &Lookup, &ErrorLock, &Errors, &VerifyFolderStats, VerifyFileHash, &IsAcceptedFolder, PathIndex]( - std::atomic<bool>&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("VerifyFile_work"); - - // TODO: Convert ScheduleWork body to function - - const std::filesystem::path TargetPath = (Path / Content.Paths[PathIndex]).make_preferred(); - if (IsAcceptedFolder(TargetPath.parent_path().generic_string())) - { - const uint64_t ExpectedSize = Content.RawSizes[PathIndex]; - if (!IsFile(TargetPath)) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize)); - }); - VerifyFolderStats.FilesFailed++; - } - else - { - std::error_code Ec; - uint64_t SizeOnDisk = gsl::narrow<uint64_t>(FileSizeFromPath(TargetPath, Ec)); - if (Ec) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back( - fmt::format("Failed to get size of file {}: {} ({})", TargetPath, Ec.message(), Ec.value())); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk < ExpectedSize) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Size of file {} is smaller than expected. Expected: {}, Found: {}", - TargetPath, - ExpectedSize, - SizeOnDisk)); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk > ExpectedSize) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Size of file {} is bigger than expected. Expected: {}, Found: {}", - TargetPath, - ExpectedSize, - SizeOnDisk)); - }); - VerifyFolderStats.FilesFailed++; - } - else if (SizeOnDisk > 0 && VerifyFileHash) - { - const IoHash& ExpectedRawHash = Content.RawHashes[PathIndex]; - IoBuffer Buffer = IoBufferBuilder::MakeFromFile(TargetPath); - IoHash RawHash = IoHash::HashBuffer(Buffer); - if (RawHash != ExpectedRawHash) - { - uint64_t FileOffset = 0; - const uint32_t SequenceIndex = Lookup.RawHashToSequenceIndex.at(ExpectedRawHash); - const uint32_t OrderOffset = Lookup.SequenceIndexChunkOrderOffset[SequenceIndex]; - for (uint32_t OrderIndex = OrderOffset; - OrderIndex < OrderOffset + Content.ChunkedContent.ChunkCounts[SequenceIndex]; - OrderIndex++) - { - uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex]; - uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; - IoHash ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; - IoBuffer FileChunk = IoBuffer(Buffer, FileOffset, ChunkSize); - if (IoHash::HashBuffer(FileChunk) != ChunkHash) - { - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format( - "WARNING: Hash of file {} does not match expected hash. Expected: {}, Found: {}. " - "Mismatch at chunk {}", - TargetPath, - ExpectedRawHash, - RawHash, - OrderIndex - OrderOffset)); - }); - break; - } - FileOffset += ChunkSize; - } - VerifyFolderStats.FilesFailed++; - } - VerifyFolderStats.ReadBytes += SizeOnDisk; - } - } - } - VerifyFolderStats.FilesVerified++; - } - }, - [&, PathIndex](std::exception_ptr Ex, std::atomic<bool>&) { - std::string Description; - try - { - std::rethrow_exception(Ex); - } - catch (const std::exception& Ex) - { - Description = Ex.what(); - } - ErrorLock.WithExclusiveLock([&]() { - Errors.push_back(fmt::format("Failed verifying file '{}'. Reason: {}", - (Path / Content.Paths[PathIndex]).make_preferred(), - Description)); - }); - VerifyFolderStats.FilesFailed++; - }); - } - - Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(PendingWork); - std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}", - VerifyFolderStats.FilesVerified.load(), - PathCount, - NiceBytes(VerifyFolderStats.ReadBytes.load()), - VerifyFolderStats.FilesFailed.load()); - ProgressBar.UpdateState({.Task = "Verifying files ", - .Details = Details, - .TotalCount = gsl::narrow<uint64_t>(PathCount), - .RemainingCount = gsl::narrow<uint64_t>(PathCount - VerifyFolderStats.FilesVerified.load()), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }); - VerifyFolderStats.VerifyElapsedWallTimeUs = Timer.GetElapsedTimeUs(); - - ProgressBar.Finish(); - if (AbortFlag) - { - return; - } - - for (const std::string& Error : Errors) - { - ZEN_CONSOLE_ERROR("{}", Error); - } - if (!Errors.empty()) - { - throw std::runtime_error(fmt::format("Verify failed with {} errors", Errors.size())); - } - } - - CbObject GetBuild(BuildStorageBase& Storage, const Oid& BuildId) - { - Stopwatch GetBuildTimer; - CbObject BuildObject = Storage.GetBuild(BuildId); - if (!IsQuiet) - { - ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", - NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), - BuildObject["name"sv].AsString(), - NiceBytes(BuildObject.GetSize())); - - ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv)); - } - return BuildObject; - } - - std::vector<std::filesystem::path> GetNewPaths(const std::span<const std::filesystem::path> KnownPaths, - const std::span<const std::filesystem::path> Paths) - { - tsl::robin_set<std::string> KnownPathsSet; - KnownPathsSet.reserve(KnownPaths.size()); - for (const std::filesystem::path& LocalPath : KnownPaths) - { - KnownPathsSet.insert(LocalPath.generic_string()); - } - - std::vector<std::filesystem::path> NewPaths; - for (const std::filesystem::path& UntrackedPath : Paths) - { - if (!KnownPathsSet.contains(UntrackedPath.generic_string())) - { - NewPaths.push_back(UntrackedPath); - } - } - return NewPaths; - } - - BuildSaveState GetLocalStateFromPaths(TransferThreadWorkers& Workers, - GetFolderContentStatistics& LocalFolderScanStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - std::span<const std::filesystem::path> PathsToCheck) - { - FolderContent FolderState; - ChunkedFolderContent ChunkedContent; - { - ProgressBar ProgressBar(ProgressMode, "Check Files"); - FolderState = GetValidFolderContent( - Workers.GetIOWorkerPool(), - LocalFolderScanStats, - Path, - PathsToCheck, - [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) { - std::string Details = - fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load()); - ProgressBar.UpdateState({.Task = "Checking files ", - .Details = Details, - .TotalCount = PathCount, - .RemainingCount = PathCount - CompletedPathCount, - .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, - false); - }, - GetUpdateDelayMS(ProgressMode), - AbortFlag, - PauseFlag); - ProgressBar.Finish(); - } - - if (FolderState.Paths.size() > 0) - { - uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : FolderState.RawSizes) - { - ByteCountToScan += RawSize; - } - ProgressBar ProgressBar(ProgressMode, "Scan Files"); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkingStatistics LocalChunkingStats; - ChunkedContent = ChunkFolderContent( - LocalChunkingStats, - Workers.GetIOWorkerPool(), - Path, - FolderState, - ChunkController, - ChunkCache, - GetUpdateDelayMS(ProgressMode), - [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { - FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - LocalChunkingStats.FilesProcessed.load(), - FolderState.Paths.size(), - NiceBytes(LocalChunkingStats.BytesHashed.load()), - NiceBytes(ByteCountToScan), - NiceNum(FilteredBytesHashed.GetCurrent()), - LocalChunkingStats.UniqueChunksFound.load(), - NiceBytes(LocalChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }, - AbortFlag, - PauseFlag); - ChunkingStats += LocalChunkingStats; - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - } - - return BuildSaveState{.State = BuildState{.ChunkedContent = std::move(ChunkedContent)}, - .FolderState = FolderState, - .LocalPath = Path}; - } - - BuildSaveState GetLocalContent(TransferThreadWorkers& Workers, - GetFolderContentStatistics& LocalFolderScanStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - const std::filesystem::path& StateFilePath, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache) - { - Stopwatch ReadStateTimer; - bool FileExists = IsFile(StateFilePath); - if (!FileExists) - { - ZEN_CONSOLE("No known local state file in {}, falling back to scanning", Path); - return {}; - } - - BuildSaveState SavedLocalState; - try - { - SavedLocalState = ReadBuildSaveStateFile(StateFilePath); - if (!IsQuiet) - { - ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); - } - } - catch (const std::exception& Ex) + const MemoryView ResponseView = Response.GetView(); + if (ToLower(Path.extension().string()) == ".cbo") { - ZEN_CONSOLE_WARN("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what()); - return {}; - } - - FolderContent CurrentLocalFolderState; - { - ProgressBar ProgressBar(ProgressMode, "Check Known Files"); - CurrentLocalFolderState = GetValidFolderContent( - Workers.GetIOWorkerPool(), - LocalFolderScanStats, - Path, - SavedLocalState.FolderState.Paths, - [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) { - std::string Details = - fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load()); - ProgressBar.UpdateState({.Task = "Checking files ", - .Details = Details, - .TotalCount = PathCount, - .RemainingCount = PathCount - CompletedPathCount, - .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, - false); - }, - GetUpdateDelayMS(ProgressMode), - AbortFlag, - PauseFlag); - ProgressBar.Finish(); - } - if (AbortFlag) - { - return {}; - } - - if (!SavedLocalState.FolderState.AreKnownFilesEqual(CurrentLocalFolderState)) - { - const size_t LocalStatePathCount = SavedLocalState.FolderState.Paths.size(); - std::vector<std::filesystem::path> DeletedPaths; - FolderContent UpdatedContent = GetUpdatedContent(SavedLocalState.FolderState, CurrentLocalFolderState, DeletedPaths); - if (!DeletedPaths.empty()) - { - SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths); - } - - if (!IsQuiet) - { - ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", - DeletedPaths.size(), - UpdatedContent.Paths.size(), - LocalStatePathCount); - } - if (UpdatedContent.Paths.size() > 0) - { - uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : UpdatedContent.RawSizes) - { - ByteCountToScan += RawSize; - } - ProgressBar ProgressBar(ProgressMode, "Scan Known Files"); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkingStatistics LocalChunkingStats; - ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( - LocalChunkingStats, - Workers.GetIOWorkerPool(), - Path, - UpdatedContent, - ChunkController, - ChunkCache, - GetUpdateDelayMS(ProgressMode), - [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { - FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - LocalChunkingStats.FilesProcessed.load(), - UpdatedContent.Paths.size(), - NiceBytes(LocalChunkingStats.BytesHashed.load()), - NiceBytes(ByteCountToScan), - NiceNum(FilteredBytesHashed.GetCurrent()), - LocalChunkingStats.UniqueChunksFound.load(), - NiceBytes(LocalChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(), - .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }, - AbortFlag, - PauseFlag); - - ChunkingStats += LocalChunkingStats; - - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - if (AbortFlag) - { - return {}; - } - SavedLocalState.State.ChunkedContent = - MergeChunkedFolderContents(SavedLocalState.State.ChunkedContent, {{UpdatedLocalContent}}); - } - } - else - { - // Remove files from LocalContent no longer in LocalFolderState - tsl::robin_set<std::string> LocalFolderPaths; - LocalFolderPaths.reserve(SavedLocalState.FolderState.Paths.size()); - for (const std::filesystem::path& LocalFolderPath : SavedLocalState.FolderState.Paths) - { - LocalFolderPaths.insert(LocalFolderPath.generic_string()); - } - std::vector<std::filesystem::path> DeletedPaths; - for (const std::filesystem::path& LocalContentPath : SavedLocalState.State.ChunkedContent.Paths) - { - if (!LocalFolderPaths.contains(LocalContentPath.generic_string())) - { - DeletedPaths.push_back(LocalContentPath); - } - } - if (!DeletedPaths.empty()) - { - SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths); - } - } - - SavedLocalState.FolderState = CurrentLocalFolderState; - - return SavedLocalState; - } - - ChunkedFolderContent ScanAndChunkFolder( - TransferThreadWorkers& Workers, - GetFolderContentStatistics& GetFolderContentStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - std::function<bool(const std::string_view& RelativePath)>&& IsAcceptedFolder, - std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache) - { - Stopwatch Timer; - - ZEN_TRACE_CPU("ScanAndChunkFolder"); - - FolderContent Content = GetFolderContent( - GetFolderContentStats, - Path, - std::move(IsAcceptedFolder), - std::move(IsAcceptedFile), - Workers.GetIOWorkerPool(), - GetUpdateDelayMS(ProgressMode), - [](bool, std::ptrdiff_t) {}, - AbortFlag); - if (AbortFlag) - { - return {}; - } - - BuildState LocalContent = GetLocalContent(Workers, - GetFolderContentStats, - ChunkingStats, - Path, - ZenStateFilePath(Path / ZenFolderName), - ChunkController, - ChunkCache) - .State; - - std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalContent.ChunkedContent.Paths, Content.Paths); - - BuildState UntrackedLocalContent = - GetLocalStateFromPaths(Workers, GetFolderContentStats, ChunkingStats, Path, ChunkController, ChunkCache, UntrackedPaths).State; - - ChunkedFolderContent Result = MergeChunkedFolderContents(LocalContent.ChunkedContent, - std::vector<ChunkedFolderContent>{UntrackedLocalContent.ChunkedContent}); - - const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0)); - const uint64_t ChunkedRawSize = - std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0)); - - if (!IsQuiet) - { - ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", - Result.Paths.size(), - NiceBytes(TotalRawSize), - Result.ChunkedContent.ChunkHashes.size(), - NiceBytes(ChunkedRawSize), - Path, - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); - } - return Result; - }; - - struct DownloadOptions - { - std::filesystem::path SystemRootDir; - std::filesystem::path ZenFolderPath; - bool AllowMultiparts = true; - EPartialBlockRequestMode PartialBlockRequestMode = EPartialBlockRequestMode::Mixed; - bool CleanTargetFolder = false; - bool PostDownloadVerify = false; - bool PrimeCacheOnly = false; - bool EnableOtherDownloadsScavenging = true; - bool EnableTargetFolderScavenging = true; - bool AllowFileClone = true; - std::vector<std::string> IncludeWildcards; - std::vector<std::string> ExcludeWildcards; - uint64_t MaximumInMemoryPayloadSize = 512u * 1024u; - bool PopulateCache = true; - bool AppendNewContent = false; - std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; - }; - - void DownloadFolder(OperationLogOutput& Output, - TransferThreadWorkers& Workers, - StorageInstance& Storage, - const BuildStorageCache::Statistics& StorageCacheStats, - const Oid& BuildId, - const std::vector<Oid>& BuildPartIds, - std::span<const std::string> BuildPartNames, - const std::filesystem::path& DownloadSpecPath, - const std::filesystem::path& Path, - const DownloadOptions& Options) - { - ZEN_TRACE_CPU("DownloadFolder"); - - ProgressBar::SetLogOperationName(ProgressMode, "Download Folder"); - - enum TaskSteps : uint32_t - { - CheckState, - CompareState, - Download, - Verify, - Cleanup, - StepCount - }; - - auto EndProgress = - MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); - - ZEN_ASSERT((!Options.PrimeCacheOnly) || - (Options.PrimeCacheOnly && (Options.PartialBlockRequestMode == EPartialBlockRequestMode::Off))); - - Stopwatch DownloadTimer; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount); - - const std::filesystem::path ZenTempFolder = ZenTempFolderPath(Options.ZenFolderPath); - CreateDirectories(ZenTempFolder); - - std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); - - std::vector<std::pair<Oid, std::string>> AllBuildParts = - ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); - - BuildManifest Manifest; - if (!DownloadSpecPath.empty()) - { - const std::filesystem::path AbsoluteDownloadSpecPath = - DownloadSpecPath.is_relative() ? MakeSafeAbsolutePath(Path / DownloadSpecPath) : MakeSafeAbsolutePath(DownloadSpecPath); - Manifest = ParseBuildManifest(DownloadSpecPath); - } - - std::vector<ChunkedFolderContent> PartContents; - - std::unique_ptr<ChunkingController> ChunkController; - - std::vector<ChunkBlockDescription> BlockDescriptions; - std::vector<IoHash> LooseChunkHashes; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount); - - ChunkedFolderContent RemoteContent = GetRemoteContent(Output, - Storage, - BuildId, - AllBuildParts, - Manifest, - Options.IncludeWildcards, - Options.ExcludeWildcards, - ChunkController, - PartContents, - BlockDescriptions, - LooseChunkHashes, - IsQuiet, - IsVerbose, - DoExtraContentVerify); - - const std::uint64_t LargeAttachmentSize = Options.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; - GetFolderContentStatistics LocalFolderScanStats; - ChunkingStatistics ChunkingStats; - - BuildSaveState LocalState; - - if (!Options.PrimeCacheOnly) - { - if (IsDir(Path)) - { - if (!ChunkController && !IsQuiet) - { - ZEN_CONSOLE_INFO("Unspecified chunking algorithm, using default"); - ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - } - std::unique_ptr<ChunkingCache> ChunkCache(CreateNullChunkingCache()); - - LocalState = GetLocalContent(Workers, - LocalFolderScanStats, - ChunkingStats, - Path, - ZenStateFilePath(Path / ZenFolderName), - *ChunkController, - *ChunkCache); - - std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalState.State.ChunkedContent.Paths, RemoteContent.Paths); - - BuildSaveState UntrackedLocalContent = GetLocalStateFromPaths(Workers, - LocalFolderScanStats, - ChunkingStats, - Path, - *ChunkController, - *ChunkCache, - UntrackedPaths); - - if (!UntrackedLocalContent.State.ChunkedContent.Paths.empty()) - { - LocalState.State.ChunkedContent = - MergeChunkedFolderContents(LocalState.State.ChunkedContent, - std::vector<ChunkedFolderContent>{UntrackedLocalContent.State.ChunkedContent}); - - // TODO: Helper - LocalState.FolderState.Paths.insert(LocalState.FolderState.Paths.begin(), - UntrackedLocalContent.FolderState.Paths.begin(), - UntrackedLocalContent.FolderState.Paths.end()); - LocalState.FolderState.RawSizes.insert(LocalState.FolderState.RawSizes.begin(), - UntrackedLocalContent.FolderState.RawSizes.begin(), - UntrackedLocalContent.FolderState.RawSizes.end()); - LocalState.FolderState.Attributes.insert(LocalState.FolderState.Attributes.begin(), - UntrackedLocalContent.FolderState.Attributes.begin(), - UntrackedLocalContent.FolderState.Attributes.end()); - LocalState.FolderState.ModificationTicks.insert(LocalState.FolderState.ModificationTicks.begin(), - UntrackedLocalContent.FolderState.ModificationTicks.begin(), - UntrackedLocalContent.FolderState.ModificationTicks.end()); - } - - if (Options.AppendNewContent) - { - RemoteContent = ApplyChunkedContentOverlay(LocalState.State.ChunkedContent, - RemoteContent, - Options.IncludeWildcards, - Options.ExcludeWildcards); - } -#if ZEN_BUILD_DEBUG - ValidateChunkedFolderContent(RemoteContent, - BlockDescriptions, - LooseChunkHashes, - Options.IncludeWildcards, - Options.ExcludeWildcards); -#endif // ZEN_BUILD_DEBUG - } - else - { - CreateDirectories(Path); - } - } - if (AbortFlag) - { - return; - } - - LocalState.LocalPath = Path; - - { - BuildsSelection::Build RemoteBuildState = {.Id = BuildId, - .IncludeWildcards = Options.IncludeWildcards, - .ExcludeWildcards = Options.ExcludeWildcards}; - RemoteBuildState.Parts.reserve(BuildPartIds.size()); - for (size_t PartIndex = 0; PartIndex < BuildPartIds.size(); PartIndex++) - { - RemoteBuildState.Parts.push_back( - {BuildsSelection::BuildPart{.Id = BuildPartIds[PartIndex], - .Name = PartIndex < BuildPartNames.size() ? BuildPartNames[PartIndex] : ""}}); - } - - if (Options.AppendNewContent) - { - LocalState.State.Selection.Builds.emplace_back(std::move(RemoteBuildState)); - } - else - { - LocalState.State.Selection.Builds = std::vector<BuildsSelection::Build>{std::move(RemoteBuildState)}; - } - } - - if ((Options.EnableTargetFolderScavenging || Options.AppendNewContent) && !Options.CleanTargetFolder && - CompareChunkedContent(RemoteContent, LocalState.State.ChunkedContent)) - { - if (!IsQuiet) - { - ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", - NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); - } - - Stopwatch WriteStateTimer; - - CbObject StateObject = CreateBuildSaveStateObject(LocalState); - CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path()); - TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView()); - if (!IsQuiet) - { - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); - } - - AddDownloadedPath(Options.SystemRootDir, - BuildsDownloadInfo{.Selection = LocalState.State.Selection, - .LocalPath = Path, - .StateFilePath = ZenStateFilePath(Options.ZenFolderPath), - .Iso8601Date = DateTime::Now().ToIso8601()}); + WriteFile(Path, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); } else { - ExtendableStringBuilder<128> BuildPartString; - for (const std::pair<Oid, std::string>& BuildPart : AllBuildParts) - { - BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first)); - } - - uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0)); - - if (!IsQuiet) - { - ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize)); - } - - Stopwatch IndexTimer; - - const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalState.State.ChunkedContent); - const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); - - if (!IsQuiet) - { - ZEN_OPERATION_LOG_INFO(Output, "Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount); - - BuildsOperationUpdateFolder Updater( - Output, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, - Path, - LocalState.State.ChunkedContent, - LocalLookup, - RemoteContent, - RemoteLookup, - BlockDescriptions, - LooseChunkHashes, - BuildsOperationUpdateFolder::Options{ - .IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .AllowFileClone = Options.AllowFileClone, - .UseSparseFiles = UseSparseFiles, - .SystemRootDir = Options.SystemRootDir, - .ZenFolderPath = Options.ZenFolderPath, - .LargeAttachmentSize = LargeAttachmentSize, - .PreferredMultipartChunkSize = PreferredMultipartChunkSize, - .PartialBlockRequestMode = Options.PartialBlockRequestMode, - .WipeTargetFolder = Options.CleanTargetFolder, - .PrimeCacheOnly = Options.PrimeCacheOnly, - .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging, - .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging || Options.AppendNewContent, - .ValidateCompletedSequences = Options.PostDownloadVerify, - .ExcludeFolders = Options.ExcludeFolders, - .MaximumInMemoryPayloadSize = Options.MaximumInMemoryPayloadSize, - .PopulateCache = Options.PopulateCache}); - { - ProgressBar::PushLogOperation(ProgressMode, "Download"); - auto _ = MakeGuard([]() { ProgressBar::PopLogOperation(ProgressMode); }); - FolderContent UpdatedLocalFolderState; - Updater.Execute(UpdatedLocalFolderState); - - LocalState.State.ChunkedContent = RemoteContent; - LocalState.FolderState = std::move(UpdatedLocalFolderState); - } - - VerifyFolderStatistics VerifyFolderStats; - if (!AbortFlag) - { - if (!Options.PrimeCacheOnly) - { - AddDownloadedPath(Options.SystemRootDir, - BuildsDownloadInfo{.Selection = LocalState.State.Selection, - .LocalPath = Path, - .StateFilePath = ZenStateFilePath(Options.ZenFolderPath), - .Iso8601Date = DateTime::Now().ToIso8601()}); - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount); - - VerifyFolder(Workers, - RemoteContent, - RemoteLookup, - Path, - Options.ExcludeFolders, - Options.PostDownloadVerify, - VerifyFolderStats); - - Stopwatch WriteStateTimer; - CbObject StateObject = CreateBuildSaveStateObject(LocalState); - - CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path()); - TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView()); - if (!IsQuiet) - { - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); - } - -#if 0 - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(StateObject, SB); - WriteFile(ZenStateFileJsonPath(Options.ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); -#endif // 0 - } - const uint64_t DownloadCount = Updater.m_DownloadStats.DownloadedChunkCount.load() + - Updater.m_DownloadStats.DownloadedBlockCount.load() + - Updater.m_DownloadStats.DownloadedPartialBlockCount.load(); - const uint64_t DownloadByteCount = Updater.m_DownloadStats.DownloadedChunkByteCount.load() + - Updater.m_DownloadStats.DownloadedBlockByteCount.load() + - Updater.m_DownloadStats.DownloadedPartialBlockByteCount.load(); - const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs(); - - if (!IsQuiet) - { - std::string CloneInfo; - if (Updater.m_DiskStats.CloneByteCount > 0) - { - CloneInfo = fmt::format(" ({} cloned)", NiceBytes(Updater.m_DiskStats.CloneByteCount.load())); - } - - std::string DownloadDetails; - { - ExtendableStringBuilder<128> SB; - BuildStorageBase::ExtendedStatistics ExtendedDownloadStats; - if (Storage.BuildStorage->GetExtendedStatistics(ExtendedDownloadStats)) - { - if (!ExtendedDownloadStats.ReceivedBytesPerSource.empty()) - { - for (auto& It : ExtendedDownloadStats.ReceivedBytesPerSource) - { - if (SB.Size() > 0) - { - SB.Append(", "sv); - } - SB.Append(It.first); - SB.Append(": "sv); - SB.Append(NiceBytes(It.second)); - } - } - } - if (Storage.CacheStorage) - { - if (SB.Size() > 0) - { - SB.Append(", "sv); - } - SB.Append("Cache: "); - SB.Append(NiceBytes(StorageCacheStats.TotalBytesRead.load())); - } - if (SB.Size() > 0) - { - DownloadDetails = fmt::format(" ({})", SB.ToView()); - } - } - - ZEN_CONSOLE( - "Downloaded build {}, parts:{} in {}\n" - " Scavenge: {} (Target: {}, Cache: {}, Others: {})\n" - " Download: {} ({}) {}bits/s{}\n" - " Write: {} ({}) {}B/s{}\n" - " Clean: {}\n" - " Finalize: {}\n" - " Verify: {}", - BuildId, - BuildPartString.ToView(), - NiceTimeSpanMs(DownloadTimeMs), - - NiceTimeSpanMs((Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs + - Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs + - Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs) / - 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000), - NiceTimeSpanMs(Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000), - - DownloadCount, - NiceBytes(DownloadByteCount), - NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)), - DownloadDetails, - - Updater.m_DiskStats.WriteCount.load(), - NiceBytes(Updater.m_WrittenChunkByteCount.load()), - NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.WriteTimeUs, Updater.m_DiskStats.WriteByteCount.load())), - CloneInfo, - - NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000), - - NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000), - - NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); - } - } - } - if (Options.PrimeCacheOnly) - { - if (Storage.CacheStorage) - { - Storage.CacheStorage->Flush(5000, [](intptr_t Remaining) { - if (!IsQuiet) - { - if (Remaining == 0) - { - ZEN_CONSOLE("Build cache upload complete"); - } - else - { - ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining); - } - } - return !AbortFlag; - }); - } - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); - - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), ZenTempFolder); - } - - void ListBuild(StorageInstance& Storage, - const Oid& BuildId, - const std::vector<Oid>& BuildPartIds, - std::span<const std::string> BuildPartNames, - std::span<const std::string> IncludeWildcards, - std::span<const std::string> ExcludeWildcards, - CbObjectWriter* OptionalStructuredOutput) - { - std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->AddObjectId("buildId"sv, BuildId); - OptionalStructuredOutput->AddObject("build"sv, BuildObject); - } - - std::vector<std::pair<Oid, std::string>> AllBuildParts = - ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); - - if (!AllBuildParts.empty()) - { - Stopwatch GetBuildPartTimer; - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginArray("parts"sv); - } - - for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++) - { - const Oid BuildPartId = AllBuildParts[BuildPartIndex].first; - const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second; - CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginObject(); - OptionalStructuredOutput->AddObjectId("id"sv, BuildPartId); - OptionalStructuredOutput->AddString("partName"sv, BuildPartName); - } - { - if (OptionalStructuredOutput != nullptr) - { - } - else if (!IsQuiet) - { - ZEN_CONSOLE("{}Part: {} ('{}'):\n", - BuildPartIndex > 0 ? "\n" : "", - BuildPartId, - BuildPartName, - NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), - NiceBytes(BuildPartManifest.GetSize())); - } - - std::vector<std::filesystem::path> Paths; - std::vector<IoHash> RawHashes; - std::vector<uint64_t> RawSizes; - std::vector<uint32_t> Attributes; - - SourcePlatform Platform; - std::vector<IoHash> SequenceRawHashes; - std::vector<uint32_t> ChunkCounts; - std::vector<uint32_t> AbsoluteChunkOrders; - std::vector<IoHash> LooseChunkHashes; - std::vector<uint64_t> LooseChunkRawSizes; - std::vector<IoHash> BlockRawHashes; - - ReadBuildContentFromCompactBinary(BuildPartManifest, - Platform, - Paths, - RawHashes, - RawSizes, - Attributes, - SequenceRawHashes, - ChunkCounts, - AbsoluteChunkOrders, - LooseChunkHashes, - LooseChunkRawSizes, - BlockRawHashes); - - std::vector<size_t> Order(Paths.size()); - std::iota(Order.begin(), Order.end(), 0); - - std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) { - const std::filesystem::path& LhsPath = Paths[Lhs]; - const std::filesystem::path& RhsPath = Paths[Rhs]; - return LhsPath < RhsPath; - }); - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginArray("files"sv); - } - { - for (size_t Index : Order) - { - const std::filesystem::path& Path = Paths[Index]; - if (IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(Path.generic_string()), /*CaseSensitive*/ true)) - { - const IoHash& RawHash = RawHashes[Index]; - const uint64_t RawSize = RawSizes[Index]; - const uint32_t Attribute = Attributes[Index]; - - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->BeginObject(); - { - OptionalStructuredOutput->AddString("path"sv, fmt::format("{}", Path)); - OptionalStructuredOutput->AddInteger("rawSize"sv, RawSize); - OptionalStructuredOutput->AddHash("rawHash"sv, RawHash); - switch (Platform) - { - case SourcePlatform::Windows: - OptionalStructuredOutput->AddInteger("attributes"sv, Attribute); - break; - case SourcePlatform::MacOS: - case SourcePlatform::Linux: - OptionalStructuredOutput->AddString("chmod"sv, fmt::format("{:#04o}", Attribute)); - break; - default: - throw std::runtime_error(fmt::format("Unsupported platform: {}", (int)Platform)); - } - } - OptionalStructuredOutput->EndObject(); - } - else - { - ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash); - } - } - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndArray(); // "files" - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndObject(); - } - } - if (OptionalStructuredOutput != nullptr) - { - OptionalStructuredOutput->EndArray(); // parts - } - } - } - - void DiffFolders(TransferThreadWorkers& Workers, - const std::filesystem::path& BasePath, - const std::filesystem::path& ComparePath, - ChunkingController& ChunkController, - ChunkingCache& ChunkCache, - const std::vector<std::string>& ExcludeFolders, - const std::vector<std::string>& ExcludeExtensions) - { - ZEN_TRACE_CPU("DiffFolders"); - - ProgressBar::SetLogOperationName(ProgressMode, "Diff Folders"); - - enum TaskSteps : uint32_t - { - CheckBase, - CheckCompare, - Diff, - Cleanup, - StepCount - }; - - auto EndProgress = - MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); - - ChunkedFolderContent BaseFolderContent; - ChunkedFolderContent CompareFolderContent; - - { - auto IsAcceptedFolder = [ExcludeFolders](const std::string_view& RelativePath) -> bool { - for (const std::string& ExcludeFolder : ExcludeFolders) - { - if (RelativePath.starts_with(ExcludeFolder)) - { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } - } - } - return true; - }; - - auto IsAcceptedFile = [ExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool { - for (const std::string& ExcludeExtension : ExcludeExtensions) - { - if (RelativePath.ends_with(ExcludeExtension)) - { - return false; - } - } - return true; - }; - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckBase, TaskSteps::StepCount); - - GetFolderContentStatistics BaseGetFolderContentStats; - ChunkingStatistics BaseChunkingStats; - BaseFolderContent = ScanAndChunkFolder(Workers, - BaseGetFolderContentStats, - BaseChunkingStats, - BasePath, - IsAcceptedFolder, - IsAcceptedFile, - ChunkController, - ChunkCache); - if (AbortFlag) - { - return; - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckCompare, TaskSteps::StepCount); - - GetFolderContentStatistics CompareGetFolderContentStats; - ChunkingStatistics CompareChunkingStats; - CompareFolderContent = ScanAndChunkFolder(Workers, - CompareGetFolderContentStats, - CompareChunkingStats, - ComparePath, - IsAcceptedFolder, - IsAcceptedFile, - ChunkController, - ChunkCache); - - if (AbortFlag) - { - return; - } - } - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Diff, TaskSteps::StepCount); - - std::vector<IoHash> AddedHashes; - std::vector<IoHash> RemovedHashes; - uint64_t RemovedSize = 0; - uint64_t AddedSize = 0; - - tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> BaseRawHashLookup; - for (size_t PathIndex = 0; PathIndex < BaseFolderContent.RawHashes.size(); PathIndex++) - { - const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex]; - BaseRawHashLookup.insert_or_assign(RawHash, PathIndex); - } - tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CompareRawHashLookup; - for (size_t PathIndex = 0; PathIndex < CompareFolderContent.RawHashes.size(); PathIndex++) - { - const IoHash& RawHash = CompareFolderContent.RawHashes[PathIndex]; - if (!BaseRawHashLookup.contains(RawHash)) - { - AddedHashes.push_back(RawHash); - AddedSize += CompareFolderContent.RawSizes[PathIndex]; - } - CompareRawHashLookup.insert_or_assign(RawHash, PathIndex); - } - for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++) - { - const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex]; - if (!CompareRawHashLookup.contains(RawHash)) - { - RemovedHashes.push_back(RawHash); - RemovedSize += BaseFolderContent.RawSizes[PathIndex]; - } - } - - uint64_t BaseTotalRawSize = 0; - for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++) - { - BaseTotalRawSize += BaseFolderContent.RawSizes[PathIndex]; - } - - double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0; - - ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept", - RemovedHashes.size(), - NiceBytes(RemovedSize), - AddedHashes.size(), - NiceBytes(AddedSize), - BaseFolderContent.Paths.size() - RemovedHashes.size(), - NiceBytes(BaseTotalRawSize - RemovedSize), - KeptPercent); - - uint64_t CompareTotalRawSize = 0; - - uint64_t FoundChunkCount = 0; - uint64_t FoundChunkSize = 0; - uint64_t NewChunkCount = 0; - uint64_t NewChunkSize = 0; - const ChunkedContentLookup BaseFolderLookup = BuildChunkedContentLookup(BaseFolderContent); - for (uint32_t ChunkIndex = 0; ChunkIndex < CompareFolderContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++) - { - const IoHash& ChunkHash = CompareFolderContent.ChunkedContent.ChunkHashes[ChunkIndex]; - if (BaseFolderLookup.ChunkHashToChunkIndex.contains(ChunkHash)) - { - FoundChunkCount++; - FoundChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - } - else - { - NewChunkCount++; - NewChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - } - CompareTotalRawSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(ResponseView, SB); + WriteFile(Path, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); } - - double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0; - double NewPercent = CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0; - - ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.", - FoundChunkCount, - NiceBytes(FoundChunkSize), - FoundPercent, - CompareFolderContent.ChunkedContent.ChunkHashes.size(), - NiceBytes(CompareTotalRawSize), - BaseFolderContent.ChunkedContent.ChunkHashes.size(), - NiceBytes(BaseTotalRawSize), - NewChunkCount, - NiceBytes(NewChunkSize), - NewPercent); - - ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); } } // namespace builds_impl -////////////////////////////////////////////////////////////////////////////////////////////////////// -// BuildsCommand — Option-adding helpers -// +////////////////////////////////////////////////////////////////////////// void -BuildsCommand::AddSystemOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddSystemOptions(cxxopts::Options& Ops) { - Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>"); + Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(SystemRootDir), "<systemdir>"); Ops.add_option("", "", "use-sparse-files", "Enable use of sparse files when writing large files. Defaults to true.", - cxxopts::value(m_UseSparseFiles), + cxxopts::value(UseSparseFiles), "<usesparsefiles>"); } void -BuildsCommand::AddCloudOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddCloudOptions(cxxopts::Options& Ops) { - m_AuthOptions.AddOptions(Ops); + AuthOptions.AddOptions(Ops); - Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>"); - Ops.add_option("cloud build", - "", - "url", - "Cloud Builds host url (legacy - use --override-host)", - cxxopts::value(m_OverrideHost), - "<url>"); - Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), "<cloud-url>"); - Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>"); + Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(OverrideHost), "<override-host>"); + Ops.add_option("cloud build", "", "url", "Cloud Builds host url (legacy - use --override-host)", cxxopts::value(OverrideHost), "<url>"); + Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(Url), "<cloud-url>"); + Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(Host), "<host>"); Ops.add_option("cloud build", "", "assume-http2", "Assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake", - cxxopts::value(m_AssumeHttp2), + cxxopts::value(AssumeHttp2), "<assumehttp2>"); Ops.add_option("cloud build", "", "verbose-http", "Enable verbose option for http client", - cxxopts::value(m_VerboseHttp), + cxxopts::value(VerboseHttp), "<verbosehttp>"); - Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), "<namespace>"); - Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), "<bucket>"); - Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(m_AllowRedirect), "<allow-redirect>"); + Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(Namespace), "<namespace>"); + Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(Bucket), "<bucket>"); + Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(AllowRedirect), "<allow-redirect>"); } void -BuildsCommand::AddFileOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddFileOptions(cxxopts::Options& Ops) { - Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(m_StoragePath), "<storagepath>"); + Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(StoragePath), "<storagepath>"); Ops.add_option("filestorage", "", "json-metadata", "Write build, part and block metadata as .json files in addition to .cb files", - cxxopts::value(m_WriteMetadataAsJson), + cxxopts::value(WriteMetadataAsJson), "<jsonmetadata>"); } void -BuildsCommand::AddCacheOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddCacheOptions(cxxopts::Options& Ops) { - Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>"); + Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(ZenCacheHost), "<zenhost>"); } void -BuildsCommand::AddOutputOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddOutputOptions(cxxopts::Options& Ops) { - Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), "<plainprogress>"); - Ops.add_option("output", - "", - "log-progress", - "Write @progress style progress to output", - cxxopts::value(m_LogProgress), - "<logprogress>"); - Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>"); - Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>"); + Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(PlainProgress), "<plainprogress>"); + Ops.add_option("output", "", "log-progress", "Write @progress style progress to output", cxxopts::value(LogProgress), "<logprogress>"); + Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(Verbose), "<verbose>"); + Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(Quiet), "<quiet>"); } void -BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddWorkerOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "boost-worker-count", "Increase the number of worker threads - may cause computer to be less responsive", - cxxopts::value(m_BoostWorkerCount), + cxxopts::value(BoostWorkerCount), "<boostworkercount>"); Ops.add_option("", @@ -2106,47 +298,47 @@ BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops) "boost-worker-memory", "Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to " "be less responsive due to high memory usage", - cxxopts::value(m_BoostWorkerMemory), + cxxopts::value(BoostWorkerMemory), "<boostworkermemory>"); Ops.add_option("", "", "boost-workers", "Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive", - cxxopts::value(m_BoostWorkers), + cxxopts::value(BoostWorkers), "<boostworkermemory>"); } void -BuildsCommand::AddZenFolderOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddZenFolderOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "zen-folder-path", - fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", builds_impl::ZenFolderName), - cxxopts::value(m_ZenFolderPath), - "<boostworkers>"); + "Path to the zen state and temp folder used by this command. Overrides the command-specific default.", + cxxopts::value(ZenFolderPath), + "<path>"); } void -BuildsCommand::AddChunkingCacheOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddChunkingCacheOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "chunking-cache-path", "Path to cache for chunking information of scanned files. Default is empty resulting in no caching", - cxxopts::value(m_ChunkingCachePath), + cxxopts::value(ChunkingCachePath), "<chunkingcachepath>"); } void -BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddWildcardOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "wildcard", "Windows style wildcard(s) (using * and ?) to match file paths to include, separated by ;", - cxxopts::value(m_IncludeWildcard), + cxxopts::value(IncludeWildcard), "<wildcard>"); Ops.add_option("", @@ -2154,46 +346,46 @@ BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops) "exclude-wildcard", "Windows style wildcard(s) (using * and ?) to match file paths to exclude, separated by ;. Applied after --wildcard " "include filter", - cxxopts::value(m_ExcludeWildcard), + cxxopts::value(ExcludeWildcard), "<excludewildcard>"); } void -BuildsCommand::AddExcludeFolderOption(cxxopts::Options& Ops) +BuildsConfiguration::AddExcludeFolderOption(cxxopts::Options& Ops) { Ops.add_option("", "", "exclude-folders", "Names of folders to exclude, separated by ;", - cxxopts::value(m_ExcludeFolders), + cxxopts::value(ExcludeFolders), "<excludefolders>"); } void -BuildsCommand::AddExcludeExtensionsOption(cxxopts::Options& Ops) +BuildsConfiguration::AddExcludeExtensionsOption(cxxopts::Options& Ops) { Ops.add_option("", "", "exclude-extensions", "Extensions to exclude, separated by ;" "include filter", - cxxopts::value(m_ExcludeExtensions), + cxxopts::value(ExcludeExtensions), "<excludeextensions>"); } void -BuildsCommand::AddMultipartOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddMultipartOptions(cxxopts::Options& Ops) { Ops.add_option("", "", "allow-multipart", "Allow large attachments to be transfered using multipart protocol. Defaults to true.", - cxxopts::value(m_AllowMultiparts), + cxxopts::value(AllowMultiparts), "<allowmultipart>"); } void -BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddPartialBlockRequestOptions(cxxopts::Options& Ops) { Ops.add_option("", "", @@ -2206,12 +398,12 @@ BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops) "allowed to host\n" " true = multiple partial block ranges requests per block allowed to zen cache and host\n" "Defaults to 'mixed'.", - cxxopts::value(m_AllowPartialBlockRequests), + cxxopts::value(AllowPartialBlockRequests), "<allowpartialblockrequests>"); } void -BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops) +BuildsConfiguration::AddAppendNewContentOptions(cxxopts::Options& Ops) { Ops.add_option("", "", @@ -2220,26 +412,26 @@ BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops) " false = the local content will be replaced by the remote content\n" " true = the remote data will be overlayed on top of local data\n" "Defaults to false.", - cxxopts::value(m_AppendNewContent), + cxxopts::value(AppendNewContent), "<append>"); } BuildsCommand::BuildsCommand() -: m_ListNamespacesSubCmd(*this) -, m_ListSubCmd(*this) -, m_ListBlocksSubCmd(*this) -, m_UploadSubCmd(*this) -, m_DownloadSubCmd(*this) -, m_LsSubCmd(*this) -, m_DiffSubCmd(*this) -, m_FetchBlobSubCmd(*this) -, m_PrimeCacheSubCmd(*this) -, m_PauseSubCmd(*this) -, m_ResumeSubCmd(*this) -, m_AbortSubCmd(*this) -, m_ValidatePartSubCmd(*this) -, m_TestSubCmd(*this) -, m_MultiTestDownloadSubCmd(*this) +: m_ListNamespacesSubCmd(m_Configuration) +, m_ListSubCmd(m_Configuration) +, m_ListBlocksSubCmd(m_Configuration) +, m_UploadSubCmd(m_Configuration) +, m_DownloadSubCmd(m_Configuration) +, m_LsSubCmd(m_Configuration) +, m_DiffSubCmd(m_Configuration) +, m_FetchBlobSubCmd(m_Configuration) +, m_PrimeCacheSubCmd(m_Configuration) +, m_PauseSubCmd(m_Configuration) +, m_ResumeSubCmd(m_Configuration) +, m_AbortSubCmd(m_Configuration) +, m_ValidatePartSubCmd(m_Configuration) +, m_TestSubCmd(m_Configuration) +, m_MultiTestDownloadSubCmd(m_Configuration) { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("__hidden__", "", "subcommand", "", cxxopts::value<std::string>(m_SubCommand)->default_value(""), ""); @@ -2269,174 +461,250 @@ BuildsCommand::OnParentOptionsParsed(const ZenCliOptions& /*GlobalOptions*/) { using namespace builds_impl; - signal(SIGINT, SignalCallbackHandler); + // Held in member variables so the handlers stay installed through the + // subcommand's Run() and are restored when BuildsCommand is destroyed. + m_SigIntGuard.emplace(SIGINT, SignalCallbackHandler); #if ZEN_PLATFORM_WINDOWS - signal(SIGBREAK, SignalCallbackHandler); -#endif // ZEN_PLATFORM_WINDOWS + m_SigBreakGuard.emplace(SIGBREAK, SignalCallbackHandler); +#endif // Validate output options - if (m_Verbose && m_Quiet) + if (m_Configuration.Verbose && m_Configuration.Quiet) { throw OptionParseException("'--verbose' conflicts with '--quiet'", {}); } - if (m_LogProgress && m_PlainProgress) + if (m_Configuration.LogProgress && m_Configuration.PlainProgress) { throw OptionParseException("'--plain-progress' conflicts with '--log-progress'", {}); } - if (m_LogProgress && m_Quiet) + if (m_Configuration.LogProgress && m_Configuration.Quiet) { throw OptionParseException("'--quiet' conflicts with '--log-progress'", {}); } - if (m_PlainProgress && m_Quiet) + if (m_Configuration.PlainProgress && m_Configuration.Quiet) { throw OptionParseException("'--quiet' conflicts with '--plain-progress'", {}); } - IsVerbose = m_Verbose; - IsQuiet = m_Quiet; - if (m_LogProgress) - { - ProgressMode = ProgressBar::Mode::Log; - } - else if (m_PlainProgress) + if (m_Configuration.LogProgress) { - ProgressMode = ProgressBar::Mode::Plain; + m_Configuration.ProgressMode = ConsoleProgressMode::Log; } - else if (m_Verbose) + else if (m_Configuration.PlainProgress) { - ProgressMode = ProgressBar::Mode::Plain; + m_Configuration.ProgressMode = ConsoleProgressMode::Plain; } - else if (IsQuiet) + else if (m_Configuration.Quiet) { - ProgressMode = ProgressBar::Mode::Quiet; + m_Configuration.ProgressMode = ConsoleProgressMode::Quiet; } else { - ProgressMode = ProgressBar::Mode::Pretty; + m_Configuration.ProgressMode = ConsoleProgressMode::Pretty; } - if (m_BoostWorkers) + if (m_Configuration.BoostWorkers) { - m_BoostWorkerCount = true; - m_BoostWorkerMemory = true; + m_Configuration.BoostWorkerCount = true; + m_Configuration.BoostWorkerMemory = true; } // Parse system options - if (m_SystemRootDir.empty()) + if (m_Configuration.SystemRootDir.empty()) { - m_SystemRootDir = PickDefaultSystemRootDirectory(); + m_Configuration.SystemRootDir = PickDefaultSystemRootDirectory(); } - MakeSafeAbsolutePathInPlace(m_SystemRootDir); - - UseSparseFiles = m_UseSparseFiles; + MakeSafeAbsolutePathInPlace(m_Configuration.SystemRootDir); + MakeSafeAbsolutePathInPlace(m_Configuration.ChunkingCachePath); return true; } +std::atomic<bool>& +BuildsSubCmdBase::AbortFlag() const +{ + return builds_impl::AbortFlag; +} + +std::atomic<bool>& +BuildsSubCmdBase::PauseFlag() const +{ + return builds_impl::PauseFlag; +} + +std::unique_ptr<ProgressBase> +BuildsSubCmdBase::CreateProgress() const +{ + return std::unique_ptr<ProgressBase>(CreateConsoleProgress(m_Config.ProgressMode)); +} + +////////////////////////////////////////////////////////////////////////// + +void +BuildsSubCmdBase::LogBanner() +{ + if (!m_Config.Quiet) + { + ZenCmdBase::LogExecutableVersionAndPid(); + } +} + void -BuildsCommand::ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts) +BuildsSubCmdBase::LogWorkersInfo(const TransferThreadWorkers& Workers) { - if (!m_Url.empty()) + if (!m_Config.Quiet) { - if (!m_Host.empty()) + ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); + } +} + +void +BuildsSubCmdBase::EnsureZenFolderExists() +{ + m_CreatedZenFolder = CreateDirectories(GetZenFolderPath()); +} + +void +BuildsSubCmdBase::CleanZenFolder() +{ + using namespace builds_impl; + WorkerThreadPool& Pool = GetSmallWorkerPool(EWorkloadType::Burst); + if (m_CreatedZenFolder) + { + CleanAndRemoveDirectory(Pool, AbortFlag(), PauseFlag(), GetZenFolderPath()); + } + else + { + CleanAndRemoveDirectory(Pool, AbortFlag(), PauseFlag(), ZenTempFolderPath(GetZenFolderPath())); + } +} + +////////////////////////////////////////////////////////////////////////// + +BuildsSubCmdBase::ResolvedStorage +BuildsSubCmdBase::ParseStorageOptions(std::string& BuildId, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + const std::filesystem::path& SystemRootDirOverride, + const std::filesystem::path& StoragePathOverride) +{ + ResolvedStorage Resolved{.SystemRootDir = SystemRootDirOverride.empty() ? m_Config.SystemRootDir : SystemRootDirOverride, + .Host = m_Config.Host, + .Namespace = m_Config.Namespace, + .Bucket = m_Config.Bucket, + .StoragePath = StoragePathOverride.empty() ? m_Config.StoragePath : StoragePathOverride}; + + if (!m_Config.Url.empty()) + { + if (!Resolved.Host.empty()) { - throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", m_Host, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", Resolved.Host, m_Config.Url), + SubOpts.help()); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", m_Bucket, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", Resolved.Bucket, m_Config.Url), + SubOpts.help()); } if (!BuildId.empty()) { - throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Url), SubOpts.help()); + throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Config.Url), + SubOpts.help()); } - if (!ParseBuildStorageUrl(m_Url, m_Host, m_Namespace, m_Bucket, BuildId)) + if (!ParseBuildStorageUrl(m_Config.Url, Resolved.Host, Resolved.Namespace, Resolved.Bucket, BuildId)) { - throw OptionParseException("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", SubOpts.help()); + throw OptionParseException( + fmt::format("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", m_Config.Url), + SubOpts.help()); } } - if (!m_OverrideHost.empty() || !m_Host.empty()) + if (!m_Config.OverrideHost.empty() || !Resolved.Host.empty()) { - if (!m_StoragePath.empty()) + if (!Resolved.StoragePath.empty()) { throw OptionParseException( - fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", m_StoragePath), + fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", Resolved.StoragePath), SubOpts.help()); } - if (RequireNamespace && m_Namespace.empty()) + if (Options.RequireNamespace && Resolved.Namespace.empty()) { throw OptionParseException("'--namespace' is required", SubOpts.help()); } - if (RequireBucket && m_Bucket.empty()) + if (Options.RequireBucket && Resolved.Bucket.empty()) { throw OptionParseException("'--bucket' is required", SubOpts.help()); } } - else if (m_StoragePath.empty()) + else if (Resolved.StoragePath.empty()) { throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help()); } - MakeSafeAbsolutePathInPlace(m_StoragePath); + MakeSafeAbsolutePathInPlace(Resolved.StoragePath); + return Resolved; } StorageInstance -BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, - BuildStorageCache::Statistics& StorageCacheStats, - const std::filesystem::path& TempPath, - std::string& BuildId, - bool RequireNamespace, - bool RequireBucket, - bool BoostCacheBackgroundWorkerPool, - std::unique_ptr<AuthMgr>& Auth, - cxxopts::Options& SubOpts) +BuildsSubCmdBase::CreateBuildStorage(const std::filesystem::path& ZenFolder, + const CreateBuildStorageOptions& Options, + cxxopts::Options& SubOpts, + BuildStorageBase::Statistics& StorageStats, + BuildStorageCache::Statistics& StorageCacheStats, + std::unique_ptr<AuthMgr>& Auth, + const ResolvedStorage& Resolved) { using namespace builds_impl; - ParseStorageOptions(BuildId, RequireNamespace, RequireBucket, SubOpts); + // Empty ZenFolder => operate without a temp directory and keep all downloads in memory. + const bool HasZenFolder = !ZenFolder.empty(); + const std::filesystem::path StorageTempPath = HasZenFolder ? ZenTempFolderPath(ZenFolder) / "storage" : std::filesystem::path{}; + const std::filesystem::path CacheTempPath = HasZenFolder ? ZenTempFolderPath(ZenFolder) / "zencache" : std::filesystem::path{}; - HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", - .AssumeHttp2 = m_AssumeHttp2, - .AllowResume = true, - .RetryCount = 2, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}; + HttpClientSettings ClientSettings{ + .LogCategory = "httpbuildsclient", + .AssumeHttp2 = m_Config.AssumeHttp2, + .AllowResume = true, + .RetryCount = 2, + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = HasZenFolder ? GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory) + : std::numeric_limits<uint64_t>::max()}; std::string StorageDescription; std::string CacheDescription; StorageInstance Result; - if (!m_Host.empty() || !m_OverrideHost.empty()) + if (!Resolved.Host.empty() || !m_Config.OverrideHost.empty()) { - m_AuthOptions.ParseOptions(SubOpts, - m_SystemRootDir, - ClientSettings, - m_Host.empty() ? m_OverrideHost : m_Host, - Auth, - IsQuiet, - /*Hidden*/ false, - m_Verbose); - - BuildStorageResolveResult ResolveRes = ResolveBuildStorage(*CreateConsoleLogOutput(ProgressMode), + AuthCommandLineOptions AuthOpts = m_Config.AuthOptions; + AuthOpts.ParseOptions(SubOpts, + Resolved.SystemRootDir, + ClientSettings, + Resolved.Host.empty() ? m_Config.OverrideHost : Resolved.Host, + Auth, + m_Config.Quiet, + /*Hidden*/ false, + m_Config.Verbose); + + BuildStorageResolveResult ResolveRes = ResolveBuildStorage(ConsoleLog(), ClientSettings, - m_Host, - m_OverrideHost, - m_ZenCacheHost, + Resolved.Host, + m_Config.OverrideHost, + m_Config.ZenCacheHost, ZenCacheResolveMode::All, - m_Verbose); + m_Config.Verbose); if (!ResolveRes.Cloud.Address.empty()) { ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2; Result.BuildStorageHttp = - std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); }); + std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, [this]() { return AbortFlag().load(); }); Result.BuildStorage = CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, - m_Namespace, - m_Bucket, - m_AllowRedirect, - TempPath / "storage"); + Resolved.Namespace, + Resolved.Bucket, + m_Config.AllowRedirect, + StorageTempPath); Result.BuildStorageHost = ResolveRes.Cloud; uint64_t HostLatencyNs = ResolveRes.Cloud.LatencySec >= 0 ? uint64_t(ResolveRes.Cloud.LatencySec * 1000000000.0) : 0; @@ -2446,8 +714,8 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, ResolveRes.Cloud.Name, (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address), Result.BuildStorageHttp->GetSessionId(), - m_Namespace, - m_Bucket, + Resolved.Namespace, + Resolved.Bucket, NiceLatencyNs(HostLatencyNs)); if (!ResolveRes.Cache.Address.empty()) @@ -2461,19 +729,21 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2, .AllowResume = true, .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)}, + [this]() { return AbortFlag().load(); }); Result.CacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, - m_Namespace, - m_Bucket, - TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); + Resolved.Namespace, + Resolved.Bucket, + CacheTempPath, + Options.BoostCacheBackgroundWorkers ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); Result.CacheHost = ResolveRes.Cache; + Result.SetupCacheSession(ResolveRes.Cache.Address, fmt::format("builds {}", m_SubOptions.program()), GetSessionId()); + uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0; CacheDescription = @@ -2483,69 +753,71 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, Result.CacheHttp->GetSessionId(), NiceLatencyNs(CacheLatencyNs)); - if (!m_Namespace.empty()) + if (!Resolved.Namespace.empty()) { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); + CacheDescription += fmt::format(". Namespace '{}'", Resolved.Namespace); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + CacheDescription += fmt::format(" Bucket '{}'", Resolved.Bucket); } } } } - else if (!m_StoragePath.empty()) + else if (!Resolved.StoragePath.empty()) { - StorageDescription = fmt::format("folder {}", m_StoragePath); - Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + StorageDescription = fmt::format("folder {}", Resolved.StoragePath); + Result.BuildStorage = CreateFileBuildStorage(Resolved.StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(), + Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = Resolved.StoragePath.generic_string(), .Name = "Disk", .LatencySec = 1.0 / 100000, // 1 us .Caps = {.MaxRangeCountPerRequest = 2048u}}; - if (!m_ZenCacheHost.empty()) + if (!m_Config.ZenCacheHost.empty()) { - ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp); + ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_Config.ZenCacheHost, m_Config.AssumeHttp2, m_Config.VerboseHttp); if (TestResult.Success) { Result.CacheHttp = std::make_unique<HttpClient>( - m_ZenCacheHost, + m_Config.ZenCacheHost, HttpClientSettings{ .LogCategory = "httpcacheclient", .ConnectTimeout = std::chrono::milliseconds{3000}, .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = m_AssumeHttp2, + .AssumeHttp2 = m_Config.AssumeHttp2, .AllowResume = true, .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); + .Verbose = m_Config.VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)}, + [this]() { return AbortFlag().load(); }); Result.CacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, - m_Namespace, - m_Bucket, - TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); - Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_ZenCacheHost, - .Name = m_ZenCacheHost, - .AssumeHttp2 = m_AssumeHttp2, + Resolved.Namespace, + Resolved.Bucket, + CacheTempPath, + Options.BoostCacheBackgroundWorkers ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); + Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_Config.ZenCacheHost, + .Name = m_Config.ZenCacheHost, + .AssumeHttp2 = m_Config.AssumeHttp2, .LatencySec = TestResult.LatencySeconds, .Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}}; + Result.SetupCacheSession(m_Config.ZenCacheHost, fmt::format("builds {}", m_SubOptions.program()), GetSessionId()); + CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId()); - if (!m_Namespace.empty()) + if (!Resolved.Namespace.empty()) { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); + CacheDescription += fmt::format(". Namespace '{}'", Resolved.Namespace); } - if (!m_Bucket.empty()) + if (!Resolved.Bucket.empty()) { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + CacheDescription += fmt::format(" Bucket '{}'", Resolved.Bucket); } } } @@ -2555,7 +827,7 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help()); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Remote: {}", StorageDescription); if (!Result.CacheHost.Name.empty()) @@ -2567,45 +839,33 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats, } Oid -BuildsCommand::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts) { - if (BuildIdStr.length() != Oid::StringLength) + const Oid BuildId = Oid::TryFromHexString(BuildIdStr); + if (BuildId == Oid::Zero) { throw OptionParseException( - fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", BuildIdStr, Oid::StringLength), + fmt::format("'--build-id' ('{}') is malformed, expected {} hex characters", BuildIdStr, Oid::StringLength), SubOpts.help()); } - else if (Oid BuildId = Oid::FromHexString(BuildIdStr); BuildId == Oid::Zero) - { - throw OptionParseException(fmt::format("'--build-id' ('{}') is invalid", BuildIdStr), SubOpts.help()); - } - else - { - return BuildId; - } + return BuildId; } Oid -BuildsCommand::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts) { - if (BuildPartIdStr.length() != Oid::StringLength) + const Oid BuildPartId = Oid::TryFromHexString(BuildPartIdStr); + if (BuildPartId == Oid::Zero) { throw OptionParseException( - fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", BuildPartIdStr, Oid::StringLength), + fmt::format("'--build-part-id' ('{}') is malformed, expected {} hex characters", BuildPartIdStr, Oid::StringLength), SubOpts.help()); } - else if (Oid BuildPartId = Oid::FromHexString(BuildPartIdStr); BuildPartId == Oid::Zero) - { - throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildPartIdStr), SubOpts.help()); - } - else - { - return BuildPartId; - } + return BuildPartId; } std::vector<Oid> -BuildsCommand::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts) { std::vector<Oid> BuildPartIds; for (const std::string& BuildPartId : BuildPartIdStrs) @@ -2620,7 +880,7 @@ BuildsCommand::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs } std::vector<std::string> -BuildsCommand::ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts) { std::vector<std::string> BuildPartNames; for (const std::string& BuildPartName : BuildPartNameStrs) @@ -2635,10 +895,10 @@ BuildsCommand::ParseBuildPartNames(const std::vector<std::string>& BuildPartName } CbObject -BuildsCommand::ParseBuildMetadata(bool CreateBuild, - std::filesystem::path& BuildMetadataPath, - const std::string& BuildMetadata, - cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBuildMetadata(bool CreateBuild, + std::filesystem::path& BuildMetadataPath, + const std::string& BuildMetadata, + cxxopts::Options& SubOpts) { if (CreateBuild) { @@ -2697,7 +957,7 @@ BuildsCommand::ParseBuildMetadata(bool CreateBuild, } void -BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) { if (Path.empty()) { @@ -2707,7 +967,7 @@ BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts) } IoHash -BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts) { if (BlobHashStr.empty()) { @@ -2731,7 +991,7 @@ BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& S } void -BuildsCommand::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) +BuildsSubCmdBase::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) { auto SplitAndAppendWildcard = [](const std::string_view Wildcard, std::vector<std::string>& Output) { ForEachStrTok(Wildcard, ';', [&Output](std::string_view Wildcard) { @@ -2757,12 +1017,13 @@ BuildsCommand::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, s }); }; - SplitAndAppendWildcard(m_IncludeWildcard, OutIncludeWildcards); - SplitAndAppendWildcard(m_ExcludeWildcard, OutExcludeWildcards); + SplitAndAppendWildcard(m_Config.IncludeWildcard, OutIncludeWildcards); + SplitAndAppendWildcard(m_Config.ExcludeWildcard, OutExcludeWildcards); } void -BuildsCommand::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions) +BuildsSubCmdBase::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, + std::vector<std::string>& OutExcludeExtensions) { auto SplitAndAppendExclusion = [](const std::string_view Input, std::vector<std::string>& Output) { ForEachStrTok(Input, ";,", [&Output](std::string_view Exclusion) { @@ -2779,38 +1040,50 @@ BuildsCommand::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExclu }); }; - SplitAndAppendExclusion(m_ExcludeFolders, OutExcludeFolders); - SplitAndAppendExclusion(m_ExcludeExtensions, OutExcludeExtensions); + SplitAndAppendExclusion(m_Config.ExcludeFolders, OutExcludeFolders); + SplitAndAppendExclusion(m_Config.ExcludeExtensions, OutExcludeExtensions); +} + +void +BuildsSubCmdBase::SetZenFolderPath(const std::filesystem::path& Fallback) +{ + m_ResolvedZenFolderPath = m_Config.ZenFolderPath.empty() ? Fallback : m_Config.ZenFolderPath; + MakeSafeAbsolutePathInPlace(m_ResolvedZenFolderPath); } void -BuildsCommand::ResolveZenFolderPath(const std::filesystem::path& DefaultPath) +BuildsSubCmdBase::ResolveZenFolderPath(const std::filesystem::path& LocalPath) { - if (m_ZenFolderPath.empty()) + using namespace builds_impl; + if (!m_Config.ZenFolderPath.empty()) + { + m_ResolvedZenFolderPath = m_Config.ZenFolderPath; + } + else if (!LocalPath.empty()) + { + m_ResolvedZenFolderPath = LocalPath / ZenFolderName; + } + else { - m_ZenFolderPath = DefaultPath; + m_ResolvedZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ResolvedZenFolderPath); } EPartialBlockRequestMode -BuildsCommand::ParseAllowPartialBlockRequests(bool PrimeCacheOnly, cxxopts::Options& SubOpts) +BuildsSubCmdBase::ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts) { - if (PrimeCacheOnly) - { - return EPartialBlockRequestMode::Off; - } - EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests); + EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_Config.AllowPartialBlockRequests); if (Mode == EPartialBlockRequestMode::Invalid) { - throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests), + throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_Config.AllowPartialBlockRequests), SubOpts.help()); } return Mode; } void -BuildsCommand::ParseZenProcessId(int& ZenProcessId) +BuildsSubCmdBase::ParseZenProcessId(int& ZenProcessId) { if (ZenProcessId == -1) { @@ -2831,20 +1104,15 @@ BuildsCommand::ParseZenProcessId(int& ZenProcessId) ////////////////////////////////////////////////////////////////////////// -// --------------------------------------------------------------------------- -// Subcommand implementations -// --------------------------------------------------------------------------- - -BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("list-namespaces", "List all namespaces and optionally their buckets") -, m_Parent(Parent) +BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "list-namespaces", "List all namespaces and optionally their buckets") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "recursive", "Enable fetch of buckets within namespaces also", cxxopts::value(m_Recursive), "<recursive>"); Opts.add_option("", "", @@ -2859,36 +1127,20 @@ BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent) void BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } - BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); + cxxopts::Options& Opts = SubOptions(); - std::unique_ptr<AuthMgr> Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ false, - /*RequireBucket*/ false, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + std::string DummyBuildId; + BuildStorageBase::Statistics StorageStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {.RequireNamespace = false, .RequireBucket = false}, Opts); + // Read-only listing: no local zen folder needed; downloads stay in memory. + StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved); CbObject Response = Storage.BuildStorage->ListNamespaces(m_Recursive); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); @@ -2900,29 +1152,20 @@ BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - std::filesystem::path ResultPath = MakeSafeAbsolutePath(m_ResultPath); - if (ToLower(ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(MakeSafeAbsolutePath(m_ResultPath), Response); } } -BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list", "List builds matching a query"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsListSubCmd::BuildsListSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "list", "List builds matching a query") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "query-path", @@ -2942,19 +1185,16 @@ BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list" void BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + if (!m_ResultPath.empty()) + { + LogBanner(); + } + + cxxopts::Options& Opts = SubOptions(); MakeSafeAbsolutePathInPlace(m_QueryPath); MakeSafeAbsolutePathInPlace(m_ResultPath); - if (!m_ResultPath.empty()) - { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - } std::string JsonQuery; if (m_QueryPath.empty()) { @@ -2991,25 +1231,13 @@ BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } } + std::string DummyBuildId; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ false, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {.RequireBucket = false}, Opts); + // Read-only listing: no local zen folder needed; downloads stay in memory. + StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved); CbObject Response = Storage.BuildStorage->ListBuilds(JsonQuery); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); @@ -3021,28 +1249,20 @@ BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } } -BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("list-blocks", "List blocks for a build") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "list-blocks", "List blocks for a build") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); Opts.add_option("", "", @@ -3058,50 +1278,34 @@ BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent) void BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - MakeSafeAbsolutePathInPlace(m_ResultPath); - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } + cxxopts::Options& Opts = SubOptions(); + + MakeSafeAbsolutePathInPlace(m_ResultPath); + if (m_MaxCount == 0) { throw OptionParseException(fmt::format("'--max-count' ('{}') is invalid", m_MaxCount), Opts.help()); } BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + // Read-only listing: no local zen folder needed; downloads stay in memory. + StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved); - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); CbObject Response = Storage.BuildStorage->FindBlocks(BuildId, m_MaxCount); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); std::vector<ChunkBlockDescription> BlockDescriptions = ParseChunkBlockDescriptionList(Response); - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Response contains {} block", BlockDescriptions.size()); } @@ -3116,35 +1320,25 @@ BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) } else { - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } } -BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("upload", "Upload a folder to build storage") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "upload", "Upload a folder to build storage") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddExcludeFolderOption(Opts); - Parent.AddExcludeExtensionsOption(Opts); - Parent.AddChunkingCacheOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddExcludeFolderOption(Opts); + Config.AddExcludeExtensionsOption(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "", @@ -3192,7 +1386,7 @@ BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) cxxopts::value(m_UploadToZenCache), "<uploadtozencache>"); - Parent.AddMultipartOptions(Opts); + Config.AddMultipartOptions(Opts); Opts.add_option("", "", @@ -3217,50 +1411,31 @@ BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent) void BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - ZenState InstanceState; + ParsePath(m_Path, Opts); - m_Parent.ParsePath(m_Path, Opts); + builds_impl::ZenState InstanceState; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + ResolveZenFolderPath({}); + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); if (m_BuildPartName.empty() && m_ManifestPath.empty()) { m_BuildPartName = m_Path.filename().string(); } - const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId(m_BuildId, Opts); if (m_BuildId.empty()) { m_BuildId = BuildId.ToString(); @@ -3269,28 +1444,31 @@ BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) Oid BuildPartId; if (!m_BuildPartId.empty()) { - BuildPartId = m_Parent.ParseBuildPartId(m_BuildPartId, Opts); + BuildPartId = ParseBuildPartId(m_BuildPartId, Opts); } - CbObject MetaData = m_Parent.ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts); + CbObject MetaData = ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts); - const std::filesystem::path TempDir = ZenTempFolderPath(m_Parent.GetZenFolderPath()); + const std::filesystem::path TempDir = ZenTempFolderPath(GetZenFolderPath()); std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); std::vector<std::pair<Oid, std::string>> UploadedParts = - UploadFolder(*Output, + UploadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId, BuildPartId, m_BuildPartName, @@ -3302,89 +1480,89 @@ BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = TempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = m_CreateBuild, .IgnoreExistingBlocks = m_Clean, .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, .ExcludeFolders = ExcludeFolders, .ExcludeExtensions = ExcludeExtensions}); - if (!AbortFlag) + if (!AbortFlag()) { if (m_PostUploadVerify) { for (const auto& Part : UploadedParts) { - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, Part.first, Part.second); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + ZenTempFolderPath(GetZenFolderPath()) / "validate", + BuildId, + Part.first, + Part.second); } } } - if (true) + if (!m_Config.Quiet) { - if (!IsQuiet) - { - ZEN_CONSOLE( - "{}:\n" - "Read: {}\n" - "Write: {}\n" - "Requests: {}\n" - "Avg Request Time: {}\n" - "Avg I/O Time: {}", - Storage.BuildStorageHost.Name, - NiceBytes(StorageStats.TotalBytesRead.load()), - NiceBytes(StorageStats.TotalBytesWritten.load()), - StorageStats.TotalRequestCount.load(), - StorageStats.TotalExecutionTimeUs.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0, - StorageStats.TotalRequestCount.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0); - } + const BuildStorageResolveResult::Host& Host = Storage.BuildStorageHost; + const BuildStorageBase::Statistics& Stats = StorageStats; + ZEN_CONSOLE( + "{}:\n" + "Read: {}\n" + "Write: {}\n" + "Requests: {}\n" + "Avg Request Time: {}\n" + "Avg I/O Time: {}", + Host.Name, + NiceBytes(Stats.TotalBytesRead.load()), + NiceBytes(Stats.TotalBytesWritten.load()), + Stats.TotalRequestCount.load(), + Stats.TotalExecutionTimeUs.load() > 0 + ? NiceTimeSpanMs(Stats.TotalExecutionTimeUs.load() / 1000 / Stats.TotalRequestCount.load()) + : 0, + Stats.TotalRequestCount.load() > 0 ? NiceTimeSpanMs(Stats.TotalRequestTimeUs.load() / 1000 / Stats.TotalRequestCount.load()) + : 0); } - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Upload aborted"); } } -BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("download", "Download a build to a local folder") -, m_Parent(Parent) -{ - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddWildcardOptions(Opts); - Parent.AddAppendNewContentOptions(Opts); - Parent.AddExcludeFolderOption(Opts); +////////////////////////////////////////////////////////////////////////// - Opts.add_option("cache", - "", - "cache-prime-only", - "Only download blobs missing in cache and upload to cache", - cxxopts::value(m_PrimeCacheOnly), - "<cacheprimeonly>"); +BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "download", "Download a build to a local folder") +{ + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddWildcardOptions(Opts); + Config.AddAppendNewContentOptions(Opts); + Config.AddExcludeFolderOption(Opts); Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - Opts.add_option("", - "", - "build-part-id", - "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", - cxxopts::value(m_BuildPartIds), - "<id>"); + Opts.add_option("", "", "build-part-id", "Build part Ids list separated by ','.", cxxopts::value(m_BuildPartIds), "<id>"); Opts.add_option("", "", "build-part-name", - "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " - "all parts will be downloaded", + "Build part names list separated by ','. If neither --build-part-id nor --build-part-name is given, " + "the part named 'default' is selected. Use '*' (alone) to select all parts.", cxxopts::value(m_BuildPartNames), "<name>"); Opts.add_option("", @@ -3405,9 +1583,9 @@ BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) "Upload data downloaded from remote host to zen cache", cxxopts::value(m_UploadToZenCache), "<uploadtozencache>"); - Parent.AddMultipartOptions(Opts); + Config.AddMultipartOptions(Opts); - Parent.AddPartialBlockRequestOptions(Opts); + Config.AddPartialBlockRequestOptions(Opts); Opts.add_option( "", @@ -3437,139 +1615,107 @@ BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent) void BuildsDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - ZenState InstanceState; - - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); std::vector<std::string> IncludeWildcards; std::vector<std::string> ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); + ParseFileFilters(IncludeWildcards, ExcludeWildcards); - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); + builds_impl::ZenState InstanceState; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ m_PrimeCacheOnly, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); - - if (m_PostDownloadVerify && m_PrimeCacheOnly) - { - throw OptionParseException("'--cache-prime-only' conflicts with '--verify'", Opts.help()); - } + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + // Preserve zen folder on exit - <zen>/current_state.cbo tracks what has been downloaded into + // this folder and is consulted by later download operations against the same target. + ResolveZenFolderPath(m_Path); + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); - if (m_Clean && m_PrimeCacheOnly) - { - ZEN_CONSOLE_WARN("Ignoring '--clean' option when '--cache-prime-only' is enabled"); - } - - if (m_Force && m_PrimeCacheOnly) - { - ZEN_CONSOLE_WARN("Ignoring '--force' option when '--cache-prime-only' is enabled"); - } - - if (m_Parent.m_AllowPartialBlockRequests != "false" && m_PrimeCacheOnly) - { - ZEN_CONSOLE_WARN("Ignoring '--allow-partial-block-requests' option when '--cache-prime-only' is enabled"); - } + const Oid BuildId = ParseBuildId(m_BuildId, Opts); - std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); + NormalizePartSelection(BuildPartIds, BuildPartNames, Opts.help()); - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(m_PrimeCacheOnly, Opts); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); - if (m_Parent.m_AppendNewContent && m_Clean) + if (m_Config.AppendNewContent && m_Clean) { throw OptionParseException("'--append' conflicts with '--clean'", Opts.help()); } std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); DownloadFolder( - *Output, + ConsoleLog(), + *Progress, Workers, Storage, - StorageCacheStats, + AbortFlag(), + PauseFlag(), + CacheStats, BuildId, BuildPartIds, BuildPartNames, m_DownloadSpecPath, m_Path, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, - .ZenFolderPath = m_Parent.GetZenFolderPath(), - .AllowMultiparts = m_Parent.m_AllowMultiparts, + DownloadOptions{.SystemRootDir = m_Config.SystemRootDir, + .ZenFolderPath = GetZenFolderPath(), + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = m_Clean, .PostDownloadVerify = m_PostDownloadVerify, - .PrimeCacheOnly = m_PrimeCacheOnly, .EnableOtherDownloadsScavenging = m_EnableScavenging && !m_Force, .EnableTargetFolderScavenging = !m_Force, .AllowFileClone = m_AllowFileClone, .IncludeWildcards = IncludeWildcards, .ExcludeWildcards = ExcludeWildcards, - .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Parent.m_BoostWorkerMemory), + .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory), .PopulateCache = m_UploadToZenCache, - .AppendNewContent = m_Parent.m_AppendNewContent, + .AppendNewContent = m_Config.AppendNewContent, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles, .ExcludeFolders = ExcludeFolders}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Download aborted"); } } -BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "List files in a build"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsLsSubCmd::BuildsLsSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "ls", "List files in a build") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddWildcardOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddWildcardOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - Opts.add_option("", - "", - "build-part-id", - "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", - cxxopts::value(m_BuildPartIds), - "<id>"); + Opts.add_option("", "", "build-part-id", "Build part Ids list separated by ','.", cxxopts::value(m_BuildPartIds), "<id>"); Opts.add_option("", "", "build-part-name", - "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " - "all parts will be downloaded", + "Build part names list separated by ','. If neither --build-part-id nor --build-part-name is given, " + "the part named 'default' is selected. Use '*' (alone) to select all parts.", cxxopts::value(m_BuildPartNames), "<name>"); @@ -3594,43 +1740,31 @@ BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "Lis void BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - if (!m_ResultPath.empty()) { - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); } - ZenState InstanceState; + cxxopts::Options& Opts = SubOptions(); + + builds_impl::ZenState InstanceState; std::vector<std::string> IncludeWildcards; std::vector<std::string> ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); - - m_Parent.ResolveZenFolderPath(m_Parent.m_StoragePath); // ls uses storage path context + ParseFileFilters(IncludeWildcards, ExcludeWildcards); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + // Read-only listing: no local zen folder needed; downloads stay in memory. + StorageInstance Storage = CreateBuildStorage({}, {}, Opts, StorageStats, CacheStats, Auth, Resolved); - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); - std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); + NormalizePartSelection(BuildPartIds, BuildPartNames, Opts.help()); std::unique_ptr<CbObjectWriter> StructuredOutput; if (!m_ResultPath.empty()) @@ -3639,38 +1773,30 @@ BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) StructuredOutput = std::make_unique<CbObjectWriter>(); } - ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get()); + ListBuild(m_Config.Quiet, Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get()); if (StructuredOutput) { CbObject Response = StructuredOutput->Save(); - if (ToLower(m_ResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + builds_impl::WriteResultObject(m_ResultPath, Response); } - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("List build aborted"); } } -BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff", "Diff two local folders"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "diff", "Diff two local folders") { - auto& Opts = SubOptions(); - Parent.AddOutputOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddExcludeFolderOption(Opts); - Parent.AddExcludeExtensionsOption(Opts); - Parent.AddChunkingCacheOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddOutputOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddExcludeFolderOption(Opts); + Config.AddExcludeExtensionsOption(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>"); Opts.add_option("", @@ -3686,38 +1812,28 @@ BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff" void BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); if (m_DiffPath.empty()) { throw OptionParseException("'--compare-path' is required", Opts.help()); } MakeSafeAbsolutePathInPlace(m_DiffPath); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); StandardChunkingControllerSettings ChunkingSettings; std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(ChunkingSettings); - std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); if (m_OnlyChunked) { @@ -3729,24 +1845,37 @@ BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ChunkingSettings.SplitAndCompressExtensions.end()); } - DiffFolders(Workers, m_Path, m_DiffPath, *ChunkController, *ChunkCache, ExcludeFolders, ExcludeExtensions); - if (AbortFlag) + std::unique_ptr<ProgressBase> Progress = CreateProgress(); + + DiffFolders(*Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + Workers, + m_Path, + m_DiffPath, + *ChunkController, + *ChunkCache, + ExcludeFolders, + ExcludeExtensions); + if (AbortFlag()) { throw std::runtime_error("Diff folders aborted"); } } -BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("fetch-blob", "Fetch and validate a specific blob") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "fetch-blob", "Fetch and validate a specific blob") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); Opts.add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), "<blob-hash>"); Opts.parse_positional({"build-id", "blob-hash"}); @@ -3756,80 +1885,55 @@ BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent) void BuildsFetchBlobSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); - - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - IoHash BlobHash = m_Parent.ParseBlobHash(m_BlobHash, Opts); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + ResolveZenFolderPath({}); + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - const Oid BuildId = Oid::FromHexString(m_BuildId); + IoHash BlobHash = ParseBlobHash(m_BlobHash, Opts); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); uint64_t CompressedSize; uint64_t DecompressedSize; - ValidateBlob(AbortFlag, *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); - if (AbortFlag) + ValidateBlob(AbortFlag(), *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); + if (AbortFlag()) { throw std::runtime_error("Fetch blob aborted"); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", BlobHash, CompressedSize, DecompressedSize); } } -BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("prime-cache", "Prime the zen cache with build data") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "prime-cache", "Prime the zen cache with build data") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); - Opts.add_option("", - "", - "build-part-id", - "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", - cxxopts::value(m_BuildPartIds), - "<id>"); + Opts.add_option("", "", "build-part-id", "Build part Ids list separated by ','.", cxxopts::value(m_BuildPartIds), "<id>"); Opts.add_option("", "", "build-part-name", - "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " - "all parts will be downloaded", + "Build part names list separated by ','. If neither --build-part-id nor --build-part-name is given, " + "the part named 'default' is selected. Use '*' (alone) to select all parts.", cxxopts::value(m_BuildPartNames), "<name>"); Opts.add_option("", @@ -3845,47 +1949,31 @@ BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent) void BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + ResolveZenFolderPath({}); + StorageInstance Storage = + CreateBuildStorage(GetZenFolderPath(), {.BoostCacheBackgroundWorkers = true}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); + const Oid BuildId = ParseBuildId(m_BuildId, Opts); - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ true, - Auth, - Opts); - - const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); - - std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts); - std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts); + std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts); + NormalizePartSelection(BuildPartIds, BuildPartNames, Opts.help()); std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); + CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId, m_Config.Quiet); std::vector<std::pair<Oid, std::string>> AllBuildParts = ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); @@ -3897,100 +1985,101 @@ BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) AllBuildPartIds.push_back(BuildPart.first); } - ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache"); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); + Progress->SetLogOperationName("Prime Cache"); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); - - BuildsOperationPrimeCache PrimeOp(*Output, + BuildsOperationPrimeCache PrimeOp(ConsoleLog(), + *Progress, Storage, - AbortFlag, - PauseFlag, + AbortFlag(), + PauseFlag(), Workers.GetNetworkPool(), BuildId, AllBuildPartIds, - BuildsOperationPrimeCache::Options{.IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .ZenFolderPath = m_Parent.GetZenFolderPath(), + BuildsOperationPrimeCache::Options{.IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .ZenFolderPath = GetZenFolderPath(), .LargeAttachmentSize = PreferredMultipartChunkSize * 4u, .PreferredMultipartChunkSize = PreferredMultipartChunkSize, .ForceUpload = m_Force}, - StorageCacheStats); + CacheStats); PrimeOp.Execute(); - if (!IsQuiet) + if (!m_Config.Quiet && Storage.CacheStorage) { - if (Storage.CacheStorage) - { - ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", - StorageCacheStats.PutBlobCount.load(), - NiceBytes(StorageCacheStats.PutBlobByteCount), - Storage.CacheHost.Name); - } + ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", + CacheStats.PutBlobCount.load(), + NiceBytes(CacheStats.PutBlobByteCount), + Storage.CacheHost.Name); } } -BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("pause", "Pause a running zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +namespace { + void AddProcessIdOption(cxxopts::Options& Opts, int& ZenProcessId) + { + Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(ZenProcessId), "<pid>"); + Opts.parse_positional({"process-id"}); + Opts.positional_help("process-id"); + } +} // namespace + +BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "pause", "Pause a running zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsPauseSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Pause.store(true); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(1); } -BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("resume", "Resume a paused zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "resume", "Resume a paused zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsResumeSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Pause.store(false); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(0); } -BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("abort", "Abort a running zen process"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "abort", "Abort a running zen process") { - auto& Opts = SubOptions(); - Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>"); - Opts.parse_positional({"process-id"}); - Opts.positional_help("process-id"); + AddProcessIdOption(SubOptions(), m_ZenProcessId); } void BuildsAbortSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - using namespace builds_impl; - m_Parent.ParseZenProcessId(m_ZenProcessId); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Abort.store(true); + ParseZenProcessId(m_ZenProcessId); + builds_impl::ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Abort.store(1); } -BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("validate-part", "Validate a build part") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "validate-part", "Validate a build part") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddWorkerOptions(Opts); - Parent.AddZenFolderOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddWorkerOptions(Opts); + Config.AddZenFolderOptions(Opts); Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); Opts.add_option("", "", @@ -4011,42 +2100,24 @@ BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent) void BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; - - if (!IsQuiet) - { - ZenCmdBase::LogExecutableVersionAndPid(); - } - - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - ZenState InstanceState; + cxxopts::Options& Opts = SubOptions(); BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts); + ResolveZenFolderPath({}); + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto _ = MakeGuard([this]() { CleanZenFolder(); }); - m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName); + builds_impl::ZenState InstanceState; - CreateDirectories(m_Parent.GetZenFolderPath()); - auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); }); - - std::unique_ptr<AuthMgr> Auth; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - m_BuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts); + Oid BuildId = ParseBuildId(m_BuildId, Opts); if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) { @@ -4055,33 +2126,46 @@ BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) Opts.help()); } - const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : m_Parent.ParseBuildPartId(m_BuildPartId, Opts); + const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(m_BuildPartId, Opts); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + ZenTempFolderPath(GetZenFolderPath()) / "validate", + BuildId, + BuildPartId, + m_BuildPartName); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Validate build part failed"); } } -BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test", "Run an upload/download test cycle"), m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsTestSubCmd::BuildsTestSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "test", "Run an upload/download test cycle") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); - Parent.AddMultipartOptions(Opts); - Parent.AddPartialBlockRequestOptions(Opts); - Parent.AddWildcardOptions(Opts); - Parent.AddAppendNewContentOptions(Opts); - Parent.AddChunkingCacheOptions(Opts); + Config.AddMultipartOptions(Opts); + Config.AddPartialBlockRequestOptions(Opts); + Config.AddWildcardOptions(Opts); + Config.AddAppendNewContentOptions(Opts); + Config.AddChunkingCacheOptions(Opts); Opts.add_option("", "", "enable-scavenge", @@ -4101,38 +2185,35 @@ BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test" void BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(m_Parent.m_SystemRootDir); - CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); - auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); }); + m_TestSystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_TestSystemRootDir); + CleanDirectory(m_TestSystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); + auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_TestSystemRootDir); }); - m_Parent.ParsePath(m_Path, Opts); + ParsePath(m_Path, Opts); - if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty()) + const bool CreatedTempStorage = m_Config.OverrideHost.empty() && m_Config.StoragePath.empty(); + if (CreatedTempStorage) { - m_Parent.m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.m_StoragePath); - CreateDirectories(m_Parent.m_StoragePath); - m_Parent.m_StoragePath = m_Parent.m_StoragePath.generic_string(); + m_TestStoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), m_TestStoragePath); + CreateDirectories(m_TestStoragePath); + m_TestStoragePath = m_TestStoragePath.generic_string(); } - auto StorageGuard = MakeGuard([this]() { - if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty()) + auto StorageGuard = MakeGuard([this, CreatedTempStorage]() { + if (CreatedTempStorage) { - DeleteDirectories(m_Parent.m_StoragePath); + DeleteDirectories(m_TestStoragePath); } }); - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(false, Opts); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); BuildStorageBase::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -4143,37 +2224,35 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2"); const std::filesystem::path DownloadPath3 = m_Path.parent_path() / (m_BuildPartName + "_test3"); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath2); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath3); - auto DownloadGuard = MakeGuard([&Workers, DownloadPath, DownloadPath2, DownloadPath3]() { - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2); - CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3); + auto DownloadGuard = MakeGuard([this, &Workers, DownloadPath, DownloadPath2, DownloadPath3]() { + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath2); + CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath3); }); - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); - MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath); - std::unique_ptr<AuthMgr> Auth; std::string TestBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - TestBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - m_BuildId = Oid::NewOid().ToString(); - m_BuildPartId = Oid::NewOid().ToString(); - m_CreateBuild = true; - - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + const ResolvedStorage Resolved = ParseStorageOptions(TestBuildId, {}, Opts, m_TestSystemRootDir, m_TestStoragePath); + // Place scratch next to the folder under test so upload does not self-include, and so the + // scratch area shares the volume of the source data. + { + const std::u8string PathStr = m_Path.generic_u8string(); + const IoHash PathHash = IoHash::HashBuffer(PathStr.data(), PathStr.length()); + SetZenFolderPath(m_Path.parent_path() / fmt::format("zen_{}", PathHash)); + } + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, StorageCacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto ZenGuard = MakeGuard([this]() { CleanZenFolder(); }); + + const Oid BuildId = Oid::NewOid(); + const Oid BuildPartId = Oid::NewOid(); + m_BuildId = BuildId.ToString(); + m_BuildPartId = BuildPartId.ToString(); + m_CreateBuild = true; auto MakeMetaData = [](const Oid& BuildId) -> CbObject { CbObjectWriter BuildMetaDataWriter; @@ -4196,18 +2275,21 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView()); } - const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path); + const std::filesystem::path UploadTempDir = ZenTempFolderPath(GetZenFolderPath()); std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty() + std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty() ? CreateNullChunkingCache() - : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u); + : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + std::unique_ptr<ProgressBase> Progress = CreateProgress(); - UploadFolder(*Output, + UploadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId, BuildPartId, m_BuildPartName, @@ -4219,12 +2301,14 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload build)"); } @@ -4232,9 +2316,12 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}' with chunking cache", m_BuildId, BuildPartId, m_BuildPartName, m_Path); - UploadFolder(*Output, + UploadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), Oid::NewOid(), Oid::NewOid(), m_BuildPartName, @@ -4246,52 +2333,70 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload again, chunking is cached)"); } } - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + ZenTempFolderPath(GetZenFolderPath()) / "validate", + BuildId, + BuildPartId, + m_BuildPartName); - if (!m_Parent.m_IncludeWildcard.empty() || !m_Parent.m_ExcludeWildcard.empty()) + if (!m_Config.IncludeWildcard.empty() || !m_Config.ExcludeWildcard.empty()) { - auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); }); + auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); }); ZEN_CONSOLE("\nDownload Filtered Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); std::vector<std::string> IncludeWildcards; std::vector<std::string> ExcludeWildcards; - m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards); + ParseFileFilters(IncludeWildcards, ExcludeWildcards); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = false, .AllowFileClone = m_AllowFileClone, .IncludeWildcards = IncludeWildcards, .ExcludeWildcards = ExcludeWildcards, - .AppendNewContent = false}); - if (AbortFlag) + .AppendNewContent = false, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } @@ -4301,66 +2406,79 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, .AllowFileClone = m_AllowFileClone, .IncludeWildcards = ExcludeWildcards, .ExcludeWildcards = IncludeWildcards, - .AppendNewContent = true}); - if (AbortFlag) + .AppendNewContent = true, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } ZEN_CONSOLE("\nDownload Full Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, .AllowFileClone = m_AllowFileClone, .IncludeWildcards = {}, .ExcludeWildcards = {}, - .AppendNewContent = false}); - if (AbortFlag) + .AppendNewContent = false, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4368,25 +2486,30 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = true, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = false, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download build)"); } ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4394,17 +2517,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download identical target)"); } @@ -4436,7 +2561,7 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) return true; }; - ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + ParallelWork Work(AbortFlag(), PauseFlag(), WorkerThreadPool::EMode::EnableBacklog); uint32_t Randomizer = 0; auto FileSizeIt = DownloadContent.FileSizes.begin(); @@ -4454,8 +2579,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { Work.ScheduleWork( Workers.GetIOWorkerPool(), - [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) { - if (!AbortFlag) + [this, SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) { + if (!AbortFlag()) { bool WasReadOnly = SetFileReadOnly(FilePath, false); { @@ -4483,7 +2608,14 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) case 1: { (void)SetFileReadOnly(FilePath, false); - (void)RemoveFile(FilePath); + std::error_code Ec; + const bool Removed = RemoveFile(FilePath, Ec); + if (!Removed || Ec) + { + throw zen::runtime_error("ScrambleDir: failed to delete '{}'. Reason: '{}'", + FilePath, + Ec ? Ec.message() : "file no longer present"); + } } break; default: @@ -4496,15 +2628,18 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ZEN_UNUSED(IsAborted, IsPaused); ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork); }); - ZEN_ASSERT(!AbortFlag.load()); + ZEN_ASSERT(!AbortFlag().load()); ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }; ScrambleDir(DownloadPath); ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4512,17 +2647,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download scrambled target)"); } @@ -4539,9 +2676,12 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView()); } - UploadFolder(*Output, + UploadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), BuildId2, BuildPartId2, m_BuildPartName, @@ -4553,22 +2693,38 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) UploadFolderOptions{.TempDir = UploadTempDir, .FindBlockMaxCount = m_FindBlockMaxCount, .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .CreateBuild = true, .IgnoreExistingBlocks = false, - .UploadToZenCache = m_UploadToZenCache}); + .UploadToZenCache = m_UploadToZenCache, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose}); - if (AbortFlag) + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Upload scrambled)"); } - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(ConsoleLog(), + *Progress, + AbortFlag(), + PauseFlag(), + m_Config.Quiet, + m_Config.Verbose, + Workers, + *Storage.BuildStorage, + ZenTempFolderPath(GetZenFolderPath()) / "validate", + BuildId, + BuildPartId, + m_BuildPartName); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, @@ -4576,133 +2732,156 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId2, {BuildPartId2}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download scrambled)"); } ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId2, {BuildPartId2}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Re-download scrambled)"); } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath2, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath2 / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath3); - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, + AbortFlag(), + PauseFlag(), StorageCacheStats, BuildId, {BuildPartId}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, DownloadPath3, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, .ZenFolderPath = DownloadPath3 / ZenFolderName, - .AllowMultiparts = m_Parent.m_AllowMultiparts, + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = false, .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = true, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Test aborted. (Download original)"); } } -BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent) -: ZenSubCmdBase("multi-test-download", "Download multiple builds sequentially as a test") -, m_Parent(Parent) +////////////////////////////////////////////////////////////////////////// + +BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsConfiguration& Config) +: BuildsSubCmdBase(Config, "multi-test-download", "Download multiple builds sequentially as a test") { - auto& Opts = SubOptions(); - Parent.AddSystemOptions(Opts); - Parent.AddCloudOptions(Opts); - Parent.AddFileOptions(Opts); - Parent.AddOutputOptions(Opts); - Parent.AddCacheOptions(Opts); - Parent.AddWorkerOptions(Opts); + cxxopts::Options& Opts = SubOptions(); + Config.AddSystemOptions(Opts); + Config.AddCloudOptions(Opts); + Config.AddFileOptions(Opts); + Config.AddOutputOptions(Opts); + Config.AddCacheOptions(Opts); + Config.AddWorkerOptions(Opts); Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); Opts.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), "<ids>"); Opts.add_option("", @@ -4724,42 +2903,37 @@ BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Pare void BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { - auto& Opts = SubOptions(); - using namespace builds_impl; + LogBanner(); + TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false); + LogWorkersInfo(Workers); - TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded); - if (!IsQuiet) - { - ZEN_CONSOLE("{}", Workers.GetWorkersInfo()); - } + cxxopts::Options& Opts = SubOptions(); - m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(m_Parent.m_SystemRootDir); - CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); - auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); }); + m_TestSystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_TestSystemRootDir); + CleanDirectory(m_TestSystemRootDir, /*ForceRemoveReadOnlyFiles*/ true); + auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_TestSystemRootDir); }); - m_Parent.ParsePath(m_Path, Opts); - - m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName); - - EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(false, Opts); + ParsePath(m_Path, Opts); + std::string DummyBuildId; BuildStorageBase::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + BuildStorageCache::Statistics CacheStats; + std::unique_ptr<AuthMgr> Auth; + const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {}, Opts, m_TestSystemRootDir); + // Place scratch next to the download target so the scratch area shares its volume. + { + const std::u8string PathStr = m_Path.generic_u8string(); + const IoHash PathHash = IoHash::HashBuffer(PathStr.data(), PathStr.length()); + SetZenFolderPath(m_Path.parent_path() / fmt::format("zen_{}", PathHash)); + } + StorageInstance Storage = CreateBuildStorage(GetZenFolderPath(), {}, Opts, StorageStats, CacheStats, Auth, Resolved); + EnsureZenFolderExists(); + auto ZenGuard = MakeGuard([this]() { CleanZenFolder(); }); - std::unique_ptr<AuthMgr> Auth; - std::string DummyBuildId; - StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_Parent.GetZenFolderPath()), - DummyBuildId, - /*RequireNamespace*/ true, - /*RequireBucket*/ true, - /*BoostCacheBackgroundWorkerPool*/ false, - Auth, - Opts); - - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts); + + std::unique_ptr<ProgressBase> Progress = CreateProgress(); Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) @@ -4769,35 +2943,40 @@ BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/) { throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildIdString), Opts.help()); } - DownloadFolder(*Output, + DownloadFolder(ConsoleLog(), + *Progress, Workers, Storage, - StorageCacheStats, + AbortFlag(), + PauseFlag(), + CacheStats, BuildId, /*BuildPartIds,*/ {}, /*BuildPartNames*/ {}, /*ManifestPath*/ {}, m_Path, - DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir, - .ZenFolderPath = m_Parent.GetZenFolderPath(), - .AllowMultiparts = m_Parent.m_AllowMultiparts, + DownloadOptions{.SystemRootDir = m_TestSystemRootDir, + .ZenFolderPath = GetZenFolderPath(), + .AllowMultiparts = m_Config.AllowMultiparts, .PartialBlockRequestMode = PartialBlockRequestMode, .CleanTargetFolder = BuildIdString == m_BuildIds.front(), .PostDownloadVerify = true, - .PrimeCacheOnly = false, .EnableOtherDownloadsScavenging = m_EnableScavenging, .EnableTargetFolderScavenging = false, - .AllowFileClone = m_AllowFileClone}); - if (AbortFlag) + .AllowFileClone = m_AllowFileClone, + .IsQuiet = m_Config.Quiet, + .IsVerbose = m_Config.Verbose, + .UseSparseFiles = m_Config.UseSparseFiles}); + if (AbortFlag()) { throw std::runtime_error("Multitest aborted"); } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("\n"); } } - if (!IsQuiet) + if (!m_Config.Quiet) { ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } |