aboutsummaryrefslogtreecommitdiff
path: root/src/zen/cmds/builds_cmd.cpp
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2026-04-20 07:27:35 +0200
committerGitHub Enterprise <[email protected]>2026-04-20 07:27:35 +0200
commitc7c59cdc5a70bfd6e5f66f3b032ea3f8f6b4d12a (patch)
tree8ce2472f9fbdd29a899be25adc864baf98ff8184 /src/zen/cmds/builds_cmd.cpp
parentMerge pull request #976 from ue-foundation/zs/config-build-storage (diff)
downloadarchived-zen-c7c59cdc5a70bfd6e5f66f3b032ea3f8f6b4d12a.tar.xz
archived-zen-c7c59cdc5a70bfd6e5f66f3b032ea3f8f6b4d12a.zip
builds cmd refactor (#975)
- Bugfix: `builds download` partial-block fetch decisions now account for build storage host latency - Bugfix: Transfer rate displays in `builds` commands now smooth correctly - Split `buildstorageoperations.cpp` (8.5k lines) into per-operation TUs: buildinspect, buildprimecache, buildstorageresolve, buildupdatefolder, builduploadfolder, buildvalidatebuildpart; stats moved to buildstoragestats.h. - FilteredRate extracted to zenutil. - BuildsCommand shared state consolidated into a BuildsConfiguration struct; subcommands inherit from BuildsSubCmdBase holding a `const BuildsConfiguration&` instead of a `BuildsCommand&`. - `ProgressBar` renamed to `ConsoleProgressBar`; mode enum (`ConsoleProgressMode`) lifted to namespace scope; `PushLogOperation`/`PopLogOperation`/`ForceLinebreak` promoted to virtuals on `ProgressBase`. - Free-function wrappers (`UploadFolder`, `DownloadFolder`, `ValidateBuildPart`) added around the existing operation classes so callers stop reimplementing setup + stats logging.
Diffstat (limited to 'src/zen/cmds/builds_cmd.cpp')
-rw-r--r--src/zen/cmds/builds_cmd.cpp3576
1 files changed, 889 insertions, 2687 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 3373506f2..e9b443dcb 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -4,68 +4,30 @@
#include <zencore/basicfile.h>
#include <zencore/compactbinarybuilder.h>
-#include <zencore/compactbinaryfile.h>
-#include <zencore/compactbinaryfmt.h>
-#include <zencore/compactbinaryvalue.h>
-#include <zencore/compress.h>
-#include <zencore/except.h>
#include <zencore/filesystem.h>
-#include <zencore/fmtutils.h>
-#include <zencore/logging.h>
#include <zencore/parallelwork.h>
+#include <zencore/process.h>
#include <zencore/scopeguard.h>
#include <zencore/session.h>
-#include <zencore/stream.h>
#include <zencore/string.h>
-#include <zencore/trace.h>
-#include <zencore/uid.h>
+#include <zenhttp/auth/authmgr.h>
#include <zenhttp/formatters.h>
-#include <zenhttp/httpclient.h>
-#include <zenhttp/httpclientauth.h>
#include <zenhttp/httpcommon.h>
-#include <zenremotestore/builds/buildcontent.h>
-#include <zenremotestore/builds/buildmanifest.h>
-#include <zenremotestore/builds/buildsavedstate.h>
-#include <zenremotestore/builds/buildstoragecache.h>
-#include <zenremotestore/builds/buildstorageoperations.h>
-#include <zenremotestore/builds/buildstorageutil.h>
+#include <zenremotestore/builds/buildinspect.h>
+#include <zenremotestore/builds/buildprimecache.h>
+#include <zenremotestore/builds/buildupdatefolder.h>
+#include <zenremotestore/builds/builduploadfolder.h>
+#include <zenremotestore/builds/buildvalidatebuildpart.h>
#include <zenremotestore/builds/filebuildstorage.h>
#include <zenremotestore/builds/jupiterbuildstorage.h>
-#include <zenremotestore/chunking/chunkblock.h>
-#include <zenremotestore/chunking/chunkedcontent.h>
-#include <zenremotestore/chunking/chunkedfile.h>
#include <zenremotestore/chunking/chunkingcache.h>
#include <zenremotestore/chunking/chunkingcontroller.h>
-#include <zenremotestore/jupiter/jupiterhost.h>
-#include <zenremotestore/transferthreadworkers.h>
#include <zenutil/filesystemutils.h>
#include <zenutil/progress.h>
-#include <zenutil/wildcard.h>
#include <zenutil/workerpools.h>
-#include <zenutil/zenserverprocess.h>
-
-#include "../progressbar.h"
#include <signal.h>
#include <memory>
-#include <numeric>
-
-ZEN_THIRD_PARTY_INCLUDES_START
-#include <tsl/robin_map.h>
-#include <tsl/robin_set.h>
-#include <json11.hpp>
-ZEN_THIRD_PARTY_INCLUDES_END
-
-#if ZEN_PLATFORM_WINDOWS
-# include <zencore/windows.h>
-#else
-# include <fcntl.h>
-# include <sys/file.h>
-# include <sys/stat.h>
-# include <unistd.h>
-#endif
-
-static const bool DoExtraContentVerify = false;
namespace zen {
@@ -115,14 +77,6 @@ namespace builds_impl {
}
};
- struct MemMap
- {
- void* Handle = nullptr;
- void* Data = nullptr;
- size_t Size = 0;
- std::string Name;
- };
-
class ZenState
{
public:
@@ -236,11 +190,6 @@ namespace builds_impl {
}
}
- const std::string ZenFolderName = ".zen";
- std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.cbo"; }
- // std::filesystem::path ZenStateFileJsonPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.json";
- // }
-
std::filesystem::path UploadTempDirectory(const std::filesystem::path& Path)
{
const std::u8string LocalPathString = Path.generic_u8string();
@@ -248,1828 +197,103 @@ namespace builds_impl {
return std::filesystem::temp_directory_path() / fmt::format("zen_{}", PathHash);
}
- const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt";
-
- const std::string UnsyncFolderName = ".unsync";
-
- const std::string UGSFolderName = ".ugs";
- const std::string LegacyZenTempFolderName = ".zen-tmp";
-
- const std::vector<std::string> DefaultExcludeFolders({UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName});
- const std::vector<std::string> DefaultExcludeExtensions({});
-
+ // Debugging knobs for file build storage - always 0 in shipped builds
const double DefaultLatency = 0; // .0010;
const double DefaultDelayPerKBSec = 0; // 0.00005;
- const bool SingleThreaded = false;
- bool UseSparseFiles = false;
-
- static bool IsVerbose = false;
- static bool IsQuiet = false;
- static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty;
-
-#undef ZEN_CONSOLE_VERBOSE
-#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \
- if (IsVerbose) \
- { \
- ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \
- }
-
- const std::string DefaultAccessTokenEnvVariableName(
-#if ZEN_PLATFORM_WINDOWS
- "UE-CloudDataCacheAccessToken"sv
-#endif
-#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
- "UE_CloudDataCacheAccessToken"sv
-#endif
-
- );
-
- static uint64_t GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory)
- {
- return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u;
- }
-
- class FilteredRate
- {
- public:
- FilteredRate() {}
-
- void Start()
- {
- if (StartTimeUS == (uint64_t)-1)
- {
- uint64_t Expected = (uint64_t)-1;
- if (StartTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs()))
- {
- LastTimeUS = StartTimeUS.load();
- }
- }
- }
- void Stop()
- {
- if (EndTimeUS == (uint64_t)-1)
- {
- uint64_t Expected = (uint64_t)-1;
- EndTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs());
- }
- }
-
- void Update(uint64_t Count)
- {
- if (LastTimeUS == (uint64_t)-1)
- {
- return;
- }
- uint64_t TimeUS = Timer.GetElapsedTimeUs();
- uint64_t TimeDeltaUS = TimeUS - LastTimeUS;
- if (TimeDeltaUS >= 2000000)
- {
- uint64_t Delta = Count - LastCount;
- uint64_t PerSecond = (Delta * 1000000) / TimeDeltaUS;
-
- LastPerSecond = PerSecond;
-
- LastCount = Count;
-
- FilteredPerSecond = (PerSecond + (LastPerSecond * 7)) / 8;
-
- LastTimeUS = TimeUS;
- }
- }
-
- uint64_t GetCurrent() const // If Stopped - return total count / total time
- {
- if (LastTimeUS == (uint64_t)-1)
- {
- return 0;
- }
- return FilteredPerSecond;
- }
-
- uint64_t GetElapsedTimeUS() const
- {
- if (StartTimeUS == (uint64_t)-1)
- {
- return 0;
- }
- if (EndTimeUS == (uint64_t)-1)
- {
- return 0;
- }
- uint64_t TimeDeltaUS = EndTimeUS - StartTimeUS;
- return TimeDeltaUS;
- }
-
- bool IsActive() const { return (StartTimeUS != (uint64_t)-1) && (EndTimeUS == (uint64_t)-1); }
-
- private:
- Stopwatch Timer;
- std::atomic<uint64_t> StartTimeUS = (uint64_t)-1;
- std::atomic<uint64_t> EndTimeUS = (uint64_t)-1;
- std::atomic<uint64_t> LastTimeUS = (uint64_t)-1;
- uint64_t LastCount = 0;
- uint64_t LastPerSecond = 0;
- uint64_t FilteredPerSecond = 0;
- };
-
- uint64_t GetBytesPerSecond(uint64_t ElapsedWallTimeUS, uint64_t Count)
- {
- if (ElapsedWallTimeUS == 0)
- {
- return 0;
- }
- return Count * 1000000 / ElapsedWallTimeUS;
- }
-
- bool CleanAndRemoveDirectory(WorkerThreadPool& WorkerPool, const std::filesystem::path& Directory)
- {
- return CleanAndRemoveDirectory(WorkerPool, AbortFlag, PauseFlag, Directory);
- }
-
- void ValidateBuildPart(LoggerRef Log,
- ProgressBase& Progress,
- TransferThreadWorkers& Workers,
- BuildStorageBase& Storage,
- const Oid& BuildId,
- Oid BuildPartId,
- const std::string_view BuildPartName)
- {
- ZEN_TRACE_CPU("ValidateBuildPart");
-
- ProgressBar::SetLogOperationName(ProgressMode, "Validate Part");
-
- BuildsOperationValidateBuildPart ValidateOp(Log,
- Progress,
- Storage,
- AbortFlag,
- PauseFlag,
- Workers.GetIOWorkerPool(),
- Workers.GetNetworkPool(),
- BuildId,
- BuildPartId,
- BuildPartName,
- BuildsOperationValidateBuildPart::Options{.IsQuiet = IsQuiet, .IsVerbose = IsVerbose});
-
- ValidateOp.Execute();
-
- const uint64_t DownloadedCount = ValidateOp.m_DownloadStats.DownloadedChunkCount + ValidateOp.m_DownloadStats.DownloadedBlockCount;
- const uint64_t DownloadedByteCount =
- ValidateOp.m_DownloadStats.DownloadedChunkByteCount + ValidateOp.m_DownloadStats.DownloadedBlockByteCount;
- ZEN_CONSOLE("Verified: {:>8} ({}), {}B/sec, {}",
- DownloadedCount,
- NiceBytes(DownloadedByteCount),
- NiceNum(GetBytesPerSecond(ValidateOp.m_ValidateStats.ElapsedWallTimeUS, DownloadedByteCount)),
- NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000));
- }
-
- struct UploadFolderOptions
- {
- std::filesystem::path TempDir;
- uint64_t FindBlockMaxCount;
- uint8_t BlockReuseMinPercentLimit;
- bool AllowMultiparts;
- bool CreateBuild;
- bool IgnoreExistingBlocks;
- bool UploadToZenCache;
- const std::vector<std::string>& ExcludeFolders = DefaultExcludeFolders;
- const std::vector<std::string>& ExcludeExtensions = DefaultExcludeExtensions;
- };
-
- std::vector<std::pair<Oid, std::string>> UploadFolder(LoggerRef Log,
- ProgressBase& Progress,
- TransferThreadWorkers& Workers,
- StorageInstance& Storage,
- const Oid& BuildId,
- const Oid& BuildPartId,
- const std::string_view BuildPartName,
- const std::filesystem::path& Path,
- const std::filesystem::path& ManifestPath,
- const CbObject& MetaData,
- ChunkingController& ChunkController,
- ChunkingCache& ChunkCache,
- const UploadFolderOptions& Options)
- {
- ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder");
-
- Stopwatch UploadTimer;
-
- BuildsOperationUploadFolder UploadOp(
- Log,
- Progress,
- Storage,
- AbortFlag,
- PauseFlag,
- Workers.GetIOWorkerPool(),
- Workers.GetNetworkPool(),
- BuildId,
- Path,
- Options.CreateBuild,
- std::move(MetaData),
- BuildsOperationUploadFolder::Options{.IsQuiet = IsQuiet,
- .IsVerbose = IsVerbose,
- .DoExtraContentValidation = DoExtraContentVerify,
- .FindBlockMaxCount = Options.FindBlockMaxCount,
- .BlockReuseMinPercentLimit = Options.BlockReuseMinPercentLimit,
- .AllowMultiparts = Options.AllowMultiparts,
- .IgnoreExistingBlocks = Options.IgnoreExistingBlocks,
- .TempDir = Options.TempDir,
- .ExcludeFolders = Options.ExcludeFolders,
- .ExcludeExtensions = Options.ExcludeExtensions,
- .ZenExcludeManifestName = ZenExcludeManifestName,
- .NonCompressableExtensions = DefaultSplitOnlyExtensions,
- .PopulateCache = Options.UploadToZenCache});
-
- std::vector<std::pair<Oid, std::string>> UploadedParts =
- UploadOp.Execute(BuildPartId, BuildPartName, ManifestPath, ChunkController, ChunkCache);
- if (AbortFlag)
- {
- return {};
- }
-
- ZEN_CONSOLE_VERBOSE(
- "Folder scanning stats:"
- "\n FoundFileCount: {}"
- "\n FoundFileByteCount: {}"
- "\n AcceptedFileCount: {}"
- "\n AcceptedFileByteCount: {}"
- "\n ElapsedWallTimeUS: {}",
- UploadOp.m_LocalFolderScanStats.FoundFileCount.load(),
- NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()),
- UploadOp.m_LocalFolderScanStats.AcceptedFileCount.load(),
- NiceBytes(UploadOp.m_LocalFolderScanStats.AcceptedFileByteCount.load()),
- NiceLatencyNs(UploadOp.m_LocalFolderScanStats.ElapsedWallTimeUS * 1000));
-
- ZEN_CONSOLE_VERBOSE(
- "Chunking stats:"
- "\n FilesProcessed: {}"
- "\n FilesChunked: {}"
- "\n BytesHashed: {}"
- "\n UniqueChunksFound: {}"
- "\n UniqueSequencesFound: {}"
- "\n UniqueBytesFound: {}"
- "\n FilesFoundInCache: {}"
- "\n ChunksFoundInCache: {}"
- "\n FilesStoredInCache: {}"
- "\n ChunksStoredInCache: {}"
- "\n ElapsedWallTimeUS: {}",
- UploadOp.m_ChunkingStats.FilesProcessed.load(),
- UploadOp.m_ChunkingStats.FilesChunked.load(),
- NiceBytes(UploadOp.m_ChunkingStats.BytesHashed.load()),
- UploadOp.m_ChunkingStats.UniqueChunksFound.load(),
- UploadOp.m_ChunkingStats.UniqueSequencesFound.load(),
- NiceBytes(UploadOp.m_ChunkingStats.UniqueBytesFound.load()),
- UploadOp.m_ChunkingStats.FilesFoundInCache.load(),
- UploadOp.m_ChunkingStats.ChunksFoundInCache.load(),
- NiceBytes(UploadOp.m_ChunkingStats.BytesFoundInCache.load()),
- UploadOp.m_ChunkingStats.FilesStoredInCache.load(),
- UploadOp.m_ChunkingStats.ChunksStoredInCache.load(),
- NiceBytes(UploadOp.m_ChunkingStats.BytesStoredInCache.load()),
- NiceLatencyNs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS * 1000));
-
- ZEN_CONSOLE_VERBOSE(
- "Find block stats:"
- "\n FindBlockTimeMS: {}"
- "\n PotentialChunkCount: {}"
- "\n PotentialChunkByteCount: {}"
- "\n FoundBlockCount: {}"
- "\n FoundBlockChunkCount: {}"
- "\n FoundBlockByteCount: {}"
- "\n AcceptedBlockCount: {}"
- "\n NewBlocksCount: {}"
- "\n NewBlocksChunkCount: {}"
- "\n NewBlocksChunkByteCount: {}",
- NiceTimeSpanMs(UploadOp.m_FindBlocksStats.FindBlockTimeMS),
- UploadOp.m_FindBlocksStats.PotentialChunkCount,
- NiceBytes(UploadOp.m_FindBlocksStats.PotentialChunkByteCount),
- UploadOp.m_FindBlocksStats.FoundBlockCount,
- UploadOp.m_FindBlocksStats.FoundBlockChunkCount,
- NiceBytes(UploadOp.m_FindBlocksStats.FoundBlockByteCount),
- UploadOp.m_FindBlocksStats.AcceptedBlockCount,
- UploadOp.m_FindBlocksStats.NewBlocksCount,
- UploadOp.m_FindBlocksStats.NewBlocksChunkCount,
- NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount));
-
- ZEN_CONSOLE_VERBOSE(
- "Reuse block stats:"
- "\n AcceptedChunkCount: {}"
- "\n AcceptedByteCount: {}"
- "\n AcceptedRawByteCount: {}"
- "\n RejectedBlockCount: {}"
- "\n RejectedChunkCount: {}"
- "\n RejectedByteCount: {}"
- "\n AcceptedReduntantChunkCount: {}"
- "\n AcceptedReduntantByteCount: {}",
- UploadOp.m_ReuseBlocksStats.AcceptedChunkCount,
- NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedByteCount),
- NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedRawByteCount),
- UploadOp.m_ReuseBlocksStats.RejectedBlockCount,
- UploadOp.m_ReuseBlocksStats.RejectedChunkCount,
- NiceBytes(UploadOp.m_ReuseBlocksStats.RejectedByteCount),
- UploadOp.m_ReuseBlocksStats.AcceptedReduntantChunkCount,
- NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedReduntantByteCount));
-
- ZEN_CONSOLE_VERBOSE(
- "Generate blocks stats:"
- "\n GeneratedBlockByteCount: {}"
- "\n GeneratedBlockCount: {}"
- "\n GenerateBlocksElapsedWallTimeUS: {}",
- NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()),
- UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(),
- NiceLatencyNs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000));
-
- ZEN_CONSOLE_VERBOSE(
- "Generate blocks stats:"
- "\n ChunkCount: {}"
- "\n ChunkByteCount: {}"
- "\n CompressedChunkCount: {}"
- "\n CompressChunksElapsedWallTimeUS: {}",
- UploadOp.m_LooseChunksStats.ChunkCount,
- NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount),
- UploadOp.m_LooseChunksStats.CompressedChunkCount.load(),
- NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()),
- NiceLatencyNs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000));
-
- ZEN_CONSOLE_VERBOSE(
- "Disk stats:"
- "\n OpenReadCount: {}"
- "\n OpenWriteCount: {}"
- "\n ReadCount: {}"
- "\n ReadByteCount: {}"
- "\n WriteCount: {} ({} cloned)"
- "\n WriteByteCount: {} ({} cloned)"
- "\n CurrentOpenFileCount: {}",
- UploadOp.m_DiskStats.OpenReadCount.load(),
- UploadOp.m_DiskStats.OpenWriteCount.load(),
- UploadOp.m_DiskStats.ReadCount.load(),
- NiceBytes(UploadOp.m_DiskStats.ReadByteCount.load()),
- UploadOp.m_DiskStats.WriteCount.load(),
- UploadOp.m_DiskStats.CloneCount.load(),
- NiceBytes(UploadOp.m_DiskStats.WriteByteCount.load()),
- NiceBytes(UploadOp.m_DiskStats.CloneByteCount.load()),
- UploadOp.m_DiskStats.CurrentOpenFileCount.load());
-
- ZEN_CONSOLE_VERBOSE(
- "Upload stats:"
- "\n BlockCount: {}"
- "\n BlocksBytes: {}"
- "\n ChunkCount: {}"
- "\n ChunksBytes: {}"
- "\n ReadFromDiskBytes: {}"
- "\n MultipartAttachmentCount: {}"
- "\n ElapsedWallTimeUS: {}",
- UploadOp.m_UploadStats.BlockCount.load(),
- NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()),
- UploadOp.m_UploadStats.ChunkCount.load(),
- NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()),
- NiceBytes(UploadOp.m_UploadStats.ReadFromDiskBytes.load()),
- UploadOp.m_UploadStats.MultipartAttachmentCount.load(),
- NiceLatencyNs(UploadOp.m_UploadStats.ElapsedWallTimeUS * 1000));
-
- const double DeltaByteCountPercent =
- UploadOp.m_ChunkingStats.BytesHashed > 0
- ? (100.0 * (UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes)) /
- (UploadOp.m_ChunkingStats.BytesHashed)
- : 0.0;
-
- const std::string MultipartAttachmentStats =
- Options.AllowMultiparts ? fmt::format(" ({} as multipart)", UploadOp.m_UploadStats.MultipartAttachmentCount.load()) : "";
-
- if (!IsQuiet)
- {
- ZEN_CONSOLE(
- "Uploaded part {} ('{}') to build {}, {}\n"
- " Scanned files: {:>8} ({}), {}B/sec, {}\n"
- " New data: {:>8} ({}) {:.1f}%\n"
- " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n"
- " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n"
- " Uploaded: {:>8} ({}), {}bits/sec, {}\n"
- " Blocks: {:>8} ({})\n"
- " Chunks: {:>8} ({}){}",
- BuildPartId,
- BuildPartName,
- BuildId,
- NiceTimeSpanMs(UploadTimer.GetElapsedTimeMs()),
-
- UploadOp.m_LocalFolderScanStats.FoundFileCount.load(),
- NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()),
- NiceNum(GetBytesPerSecond(UploadOp.m_ChunkingStats.ElapsedWallTimeUS, UploadOp.m_ChunkingStats.BytesHashed)),
- NiceTimeSpanMs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS / 1000),
-
- UploadOp.m_FindBlocksStats.NewBlocksChunkCount + UploadOp.m_LooseChunksStats.CompressedChunkCount,
- NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes),
- DeltaByteCountPercent,
-
- UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(),
- NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount),
- NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()),
- NiceNum(GetBytesPerSecond(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
- UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount)),
- NiceTimeSpanMs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000),
-
- UploadOp.m_LooseChunksStats.CompressedChunkCount.load(),
- NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkRawBytes),
- NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()),
- NiceNum(GetBytesPerSecond(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS,
- UploadOp.m_LooseChunksStats.CompressedChunkRawBytes)),
- NiceTimeSpanMs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000),
-
- UploadOp.m_UploadStats.BlockCount.load() + UploadOp.m_UploadStats.ChunkCount.load(),
- NiceBytes(UploadOp.m_UploadStats.BlocksBytes + UploadOp.m_UploadStats.ChunksBytes),
- NiceNum(GetBytesPerSecond(UploadOp.m_UploadStats.ElapsedWallTimeUS,
- (UploadOp.m_UploadStats.ChunksBytes + UploadOp.m_UploadStats.BlocksBytes) * 8)),
- NiceTimeSpanMs(UploadOp.m_UploadStats.ElapsedWallTimeUS / 1000),
-
- UploadOp.m_UploadStats.BlockCount.load(),
- NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()),
-
- UploadOp.m_UploadStats.ChunkCount.load(),
- NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()),
- MultipartAttachmentStats);
- }
- return UploadedParts;
- }
-
- struct VerifyFolderStatistics
+ void WriteResultObject(const std::filesystem::path& Path, const CbObject& Response)
{
- std::atomic<uint64_t> FilesVerified = 0;
- std::atomic<uint64_t> FilesFailed = 0;
- std::atomic<uint64_t> ReadBytes = 0;
- uint64_t VerifyElapsedWallTimeUs = 0;
- };
-
- void VerifyFolder(TransferThreadWorkers& Workers,
- const ChunkedFolderContent& Content,
- const ChunkedContentLookup& Lookup,
- const std::filesystem::path& Path,
- const std::vector<std::string>& ExcludeFolders,
- bool VerifyFileHash,
- VerifyFolderStatistics& VerifyFolderStats)
- {
- ZEN_TRACE_CPU("VerifyFolder");
-
- Stopwatch Timer;
-
- ProgressBar ProgressBar(ProgressMode, "Verify Files");
-
- WorkerThreadPool& VerifyPool = Workers.GetIOWorkerPool();
-
- ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
-
- const uint32_t PathCount = gsl::narrow<uint32_t>(Content.Paths.size());
-
- RwLock ErrorLock;
- std::vector<std::string> Errors;
-
- auto IsAcceptedFolder = [ExcludeFolders = ExcludeFolders](const std::string_view& RelativePath) -> bool {
- for (const std::string& ExcludeFolder : ExcludeFolders)
- {
- if (RelativePath.starts_with(ExcludeFolder))
- {
- if (RelativePath.length() == ExcludeFolder.length())
- {
- return false;
- }
- else if (RelativePath[ExcludeFolder.length()] == '/')
- {
- return false;
- }
- }
- }
- return true;
- };
-
- for (uint32_t PathIndex = 0; PathIndex < PathCount; PathIndex++)
+ const MemoryView ResponseView = Response.GetView();
+ if (ToLower(Path.extension().string()) == ".cbo")
{
- if (Work.IsAborted())
- {
- break;
- }
-
- Work.ScheduleWork(
- VerifyPool,
- [&Path, &Content, &Lookup, &ErrorLock, &Errors, &VerifyFolderStats, VerifyFileHash, &IsAcceptedFolder, PathIndex](
- std::atomic<bool>&) {
- if (!AbortFlag)
- {
- ZEN_TRACE_CPU("VerifyFile_work");
-
- // TODO: Convert ScheduleWork body to function
-
- const std::filesystem::path TargetPath = (Path / Content.Paths[PathIndex]).make_preferred();
- if (IsAcceptedFolder(TargetPath.parent_path().generic_string()))
- {
- const uint64_t ExpectedSize = Content.RawSizes[PathIndex];
- if (!IsFile(TargetPath))
- {
- ErrorLock.WithExclusiveLock([&]() {
- Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize));
- });
- VerifyFolderStats.FilesFailed++;
- }
- else
- {
- std::error_code Ec;
- uint64_t SizeOnDisk = gsl::narrow<uint64_t>(FileSizeFromPath(TargetPath, Ec));
- if (Ec)
- {
- ErrorLock.WithExclusiveLock([&]() {
- Errors.push_back(
- fmt::format("Failed to get size of file {}: {} ({})", TargetPath, Ec.message(), Ec.value()));
- });
- VerifyFolderStats.FilesFailed++;
- }
- else if (SizeOnDisk < ExpectedSize)
- {
- ErrorLock.WithExclusiveLock([&]() {
- Errors.push_back(fmt::format("Size of file {} is smaller than expected. Expected: {}, Found: {}",
- TargetPath,
- ExpectedSize,
- SizeOnDisk));
- });
- VerifyFolderStats.FilesFailed++;
- }
- else if (SizeOnDisk > ExpectedSize)
- {
- ErrorLock.WithExclusiveLock([&]() {
- Errors.push_back(fmt::format("Size of file {} is bigger than expected. Expected: {}, Found: {}",
- TargetPath,
- ExpectedSize,
- SizeOnDisk));
- });
- VerifyFolderStats.FilesFailed++;
- }
- else if (SizeOnDisk > 0 && VerifyFileHash)
- {
- const IoHash& ExpectedRawHash = Content.RawHashes[PathIndex];
- IoBuffer Buffer = IoBufferBuilder::MakeFromFile(TargetPath);
- IoHash RawHash = IoHash::HashBuffer(Buffer);
- if (RawHash != ExpectedRawHash)
- {
- uint64_t FileOffset = 0;
- const uint32_t SequenceIndex = Lookup.RawHashToSequenceIndex.at(ExpectedRawHash);
- const uint32_t OrderOffset = Lookup.SequenceIndexChunkOrderOffset[SequenceIndex];
- for (uint32_t OrderIndex = OrderOffset;
- OrderIndex < OrderOffset + Content.ChunkedContent.ChunkCounts[SequenceIndex];
- OrderIndex++)
- {
- uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex];
- uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
- IoHash ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex];
- IoBuffer FileChunk = IoBuffer(Buffer, FileOffset, ChunkSize);
- if (IoHash::HashBuffer(FileChunk) != ChunkHash)
- {
- ErrorLock.WithExclusiveLock([&]() {
- Errors.push_back(fmt::format(
- "WARNING: Hash of file {} does not match expected hash. Expected: {}, Found: {}. "
- "Mismatch at chunk {}",
- TargetPath,
- ExpectedRawHash,
- RawHash,
- OrderIndex - OrderOffset));
- });
- break;
- }
- FileOffset += ChunkSize;
- }
- VerifyFolderStats.FilesFailed++;
- }
- VerifyFolderStats.ReadBytes += SizeOnDisk;
- }
- }
- }
- VerifyFolderStats.FilesVerified++;
- }
- },
- [&, PathIndex](std::exception_ptr Ex, std::atomic<bool>&) {
- std::string Description;
- try
- {
- std::rethrow_exception(Ex);
- }
- catch (const std::exception& Ex)
- {
- Description = Ex.what();
- }
- ErrorLock.WithExclusiveLock([&]() {
- Errors.push_back(fmt::format("Failed verifying file '{}'. Reason: {}",
- (Path / Content.Paths[PathIndex]).make_preferred(),
- Description));
- });
- VerifyFolderStats.FilesFailed++;
- });
- }
-
- Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
- ZEN_UNUSED(PendingWork);
- std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}",
- VerifyFolderStats.FilesVerified.load(),
- PathCount,
- NiceBytes(VerifyFolderStats.ReadBytes.load()),
- VerifyFolderStats.FilesFailed.load());
- ProgressBar.UpdateState({.Task = "Verifying files ",
- .Details = Details,
- .TotalCount = gsl::narrow<uint64_t>(PathCount),
- .RemainingCount = gsl::narrow<uint64_t>(PathCount - VerifyFolderStats.FilesVerified.load()),
- .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- });
- VerifyFolderStats.VerifyElapsedWallTimeUs = Timer.GetElapsedTimeUs();
-
- ProgressBar.Finish();
- if (AbortFlag)
- {
- return;
- }
-
- for (const std::string& Error : Errors)
- {
- ZEN_CONSOLE_ERROR("{}", Error);
- }
- if (!Errors.empty())
- {
- throw std::runtime_error(fmt::format("Verify failed with {} errors", Errors.size()));
- }
- }
-
- CbObject GetBuild(BuildStorageBase& Storage, const Oid& BuildId)
- {
- Stopwatch GetBuildTimer;
- CbObject BuildObject = Storage.GetBuild(BuildId);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}",
- NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()),
- BuildObject["name"sv].AsString(),
- NiceBytes(BuildObject.GetSize()));
-
- ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv));
- }
- return BuildObject;
- }
-
- std::vector<std::filesystem::path> GetNewPaths(const std::span<const std::filesystem::path> KnownPaths,
- const std::span<const std::filesystem::path> Paths)
- {
- tsl::robin_set<std::string> KnownPathsSet;
- KnownPathsSet.reserve(KnownPaths.size());
- for (const std::filesystem::path& LocalPath : KnownPaths)
- {
- KnownPathsSet.insert(LocalPath.generic_string());
- }
-
- std::vector<std::filesystem::path> NewPaths;
- for (const std::filesystem::path& UntrackedPath : Paths)
- {
- if (!KnownPathsSet.contains(UntrackedPath.generic_string()))
- {
- NewPaths.push_back(UntrackedPath);
- }
- }
- return NewPaths;
- }
-
- BuildSaveState GetLocalStateFromPaths(TransferThreadWorkers& Workers,
- GetFolderContentStatistics& LocalFolderScanStats,
- ChunkingStatistics& ChunkingStats,
- const std::filesystem::path& Path,
- ChunkingController& ChunkController,
- ChunkingCache& ChunkCache,
- std::span<const std::filesystem::path> PathsToCheck)
- {
- FolderContent FolderState;
- ChunkedFolderContent ChunkedContent;
- {
- ProgressBar ProgressBar(ProgressMode, "Check Files");
- FolderState = GetValidFolderContent(
- Workers.GetIOWorkerPool(),
- LocalFolderScanStats,
- Path,
- PathsToCheck,
- [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) {
- std::string Details =
- fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load());
- ProgressBar.UpdateState({.Task = "Checking files ",
- .Details = Details,
- .TotalCount = PathCount,
- .RemainingCount = PathCount - CompletedPathCount,
- .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
- false);
- },
- GetUpdateDelayMS(ProgressMode),
- AbortFlag,
- PauseFlag);
- ProgressBar.Finish();
- }
-
- if (FolderState.Paths.size() > 0)
- {
- uint64_t ByteCountToScan = 0;
- for (const uint64_t RawSize : FolderState.RawSizes)
- {
- ByteCountToScan += RawSize;
- }
- ProgressBar ProgressBar(ProgressMode, "Scan Files");
- FilteredRate FilteredBytesHashed;
- FilteredBytesHashed.Start();
- ChunkingStatistics LocalChunkingStats;
- ChunkedContent = ChunkFolderContent(
- LocalChunkingStats,
- Workers.GetIOWorkerPool(),
- Path,
- FolderState,
- ChunkController,
- ChunkCache,
- GetUpdateDelayMS(ProgressMode),
- [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
- FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load());
- std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
- LocalChunkingStats.FilesProcessed.load(),
- FolderState.Paths.size(),
- NiceBytes(LocalChunkingStats.BytesHashed.load()),
- NiceBytes(ByteCountToScan),
- NiceNum(FilteredBytesHashed.GetCurrent()),
- LocalChunkingStats.UniqueChunksFound.load(),
- NiceBytes(LocalChunkingStats.UniqueBytesFound.load()));
- ProgressBar.UpdateState({.Task = "Scanning files ",
- .Details = Details,
- .TotalCount = ByteCountToScan,
- .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(),
- .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- },
- AbortFlag,
- PauseFlag);
- ChunkingStats += LocalChunkingStats;
- FilteredBytesHashed.Stop();
- ProgressBar.Finish();
- }
-
- return BuildSaveState{.State = BuildState{.ChunkedContent = std::move(ChunkedContent)},
- .FolderState = FolderState,
- .LocalPath = Path};
- }
-
- BuildSaveState GetLocalContent(TransferThreadWorkers& Workers,
- GetFolderContentStatistics& LocalFolderScanStats,
- ChunkingStatistics& ChunkingStats,
- const std::filesystem::path& Path,
- const std::filesystem::path& StateFilePath,
- ChunkingController& ChunkController,
- ChunkingCache& ChunkCache)
- {
- Stopwatch ReadStateTimer;
- bool FileExists = IsFile(StateFilePath);
- if (!FileExists)
- {
- ZEN_CONSOLE("No known local state file in {}, falling back to scanning", Path);
- return {};
- }
-
- BuildSaveState SavedLocalState;
- try
- {
- SavedLocalState = ReadBuildSaveStateFile(StateFilePath);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs()));
- }
- }
- catch (const std::exception& Ex)
- {
- ZEN_CONSOLE_WARN("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what());
- return {};
- }
-
- FolderContent CurrentLocalFolderState;
- {
- ProgressBar ProgressBar(ProgressMode, "Check Known Files");
- CurrentLocalFolderState = GetValidFolderContent(
- Workers.GetIOWorkerPool(),
- LocalFolderScanStats,
- Path,
- SavedLocalState.FolderState.Paths,
- [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) {
- std::string Details =
- fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load());
- ProgressBar.UpdateState({.Task = "Checking files ",
- .Details = Details,
- .TotalCount = PathCount,
- .RemainingCount = PathCount - CompletedPathCount,
- .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
- false);
- },
- GetUpdateDelayMS(ProgressMode),
- AbortFlag,
- PauseFlag);
- ProgressBar.Finish();
- }
- if (AbortFlag)
- {
- return {};
- }
-
- if (!SavedLocalState.FolderState.AreKnownFilesEqual(CurrentLocalFolderState))
- {
- const size_t LocalStatePathCount = SavedLocalState.FolderState.Paths.size();
- std::vector<std::filesystem::path> DeletedPaths;
- FolderContent UpdatedContent = GetUpdatedContent(SavedLocalState.FolderState, CurrentLocalFolderState, DeletedPaths);
- if (!DeletedPaths.empty())
- {
- SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths);
- }
-
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}",
- DeletedPaths.size(),
- UpdatedContent.Paths.size(),
- LocalStatePathCount);
- }
- if (UpdatedContent.Paths.size() > 0)
- {
- uint64_t ByteCountToScan = 0;
- for (const uint64_t RawSize : UpdatedContent.RawSizes)
- {
- ByteCountToScan += RawSize;
- }
- ProgressBar ProgressBar(ProgressMode, "Scan Known Files");
- FilteredRate FilteredBytesHashed;
- FilteredBytesHashed.Start();
- ChunkingStatistics LocalChunkingStats;
- ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent(
- LocalChunkingStats,
- Workers.GetIOWorkerPool(),
- Path,
- UpdatedContent,
- ChunkController,
- ChunkCache,
- GetUpdateDelayMS(ProgressMode),
- [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
- FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load());
- std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
- LocalChunkingStats.FilesProcessed.load(),
- UpdatedContent.Paths.size(),
- NiceBytes(LocalChunkingStats.BytesHashed.load()),
- NiceBytes(ByteCountToScan),
- NiceNum(FilteredBytesHashed.GetCurrent()),
- LocalChunkingStats.UniqueChunksFound.load(),
- NiceBytes(LocalChunkingStats.UniqueBytesFound.load()));
- ProgressBar.UpdateState({.Task = "Scanning files ",
- .Details = Details,
- .TotalCount = ByteCountToScan,
- .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(),
- .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- },
- AbortFlag,
- PauseFlag);
-
- ChunkingStats += LocalChunkingStats;
-
- FilteredBytesHashed.Stop();
- ProgressBar.Finish();
- if (AbortFlag)
- {
- return {};
- }
- SavedLocalState.State.ChunkedContent =
- MergeChunkedFolderContents(SavedLocalState.State.ChunkedContent, {{UpdatedLocalContent}});
- }
+ WriteFile(Path, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
}
else
{
- // Remove files from LocalContent no longer in LocalFolderState
- tsl::robin_set<std::string> LocalFolderPaths;
- LocalFolderPaths.reserve(SavedLocalState.FolderState.Paths.size());
- for (const std::filesystem::path& LocalFolderPath : SavedLocalState.FolderState.Paths)
- {
- LocalFolderPaths.insert(LocalFolderPath.generic_string());
- }
- std::vector<std::filesystem::path> DeletedPaths;
- for (const std::filesystem::path& LocalContentPath : SavedLocalState.State.ChunkedContent.Paths)
- {
- if (!LocalFolderPaths.contains(LocalContentPath.generic_string()))
- {
- DeletedPaths.push_back(LocalContentPath);
- }
- }
- if (!DeletedPaths.empty())
- {
- SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths);
- }
- }
-
- SavedLocalState.FolderState = CurrentLocalFolderState;
-
- return SavedLocalState;
- }
-
- ChunkedFolderContent ScanAndChunkFolder(
- TransferThreadWorkers& Workers,
- GetFolderContentStatistics& GetFolderContentStats,
- ChunkingStatistics& ChunkingStats,
- const std::filesystem::path& Path,
- std::function<bool(const std::string_view& RelativePath)>&& IsAcceptedFolder,
- std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile,
- ChunkingController& ChunkController,
- ChunkingCache& ChunkCache)
- {
- Stopwatch Timer;
-
- ZEN_TRACE_CPU("ScanAndChunkFolder");
-
- FolderContent Content = GetFolderContent(
- GetFolderContentStats,
- Path,
- std::move(IsAcceptedFolder),
- std::move(IsAcceptedFile),
- Workers.GetIOWorkerPool(),
- GetUpdateDelayMS(ProgressMode),
- [](bool, std::ptrdiff_t) {},
- AbortFlag);
- if (AbortFlag)
- {
- return {};
- }
-
- BuildState LocalContent = GetLocalContent(Workers,
- GetFolderContentStats,
- ChunkingStats,
- Path,
- ZenStateFilePath(Path / ZenFolderName),
- ChunkController,
- ChunkCache)
- .State;
-
- std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalContent.ChunkedContent.Paths, Content.Paths);
-
- BuildState UntrackedLocalContent =
- GetLocalStateFromPaths(Workers, GetFolderContentStats, ChunkingStats, Path, ChunkController, ChunkCache, UntrackedPaths).State;
-
- ChunkedFolderContent Result = MergeChunkedFolderContents(LocalContent.ChunkedContent,
- std::vector<ChunkedFolderContent>{UntrackedLocalContent.ChunkedContent});
-
- const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0));
- const uint64_t ChunkedRawSize =
- std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0));
-
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
- Result.Paths.size(),
- NiceBytes(TotalRawSize),
- Result.ChunkedContent.ChunkHashes.size(),
- NiceBytes(ChunkedRawSize),
- Path,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
- NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
- }
- return Result;
- };
-
- struct DownloadOptions
- {
- std::filesystem::path SystemRootDir;
- std::filesystem::path ZenFolderPath;
- bool AllowMultiparts = true;
- EPartialBlockRequestMode PartialBlockRequestMode = EPartialBlockRequestMode::Mixed;
- bool CleanTargetFolder = false;
- bool PostDownloadVerify = false;
- bool EnableOtherDownloadsScavenging = true;
- bool EnableTargetFolderScavenging = true;
- bool AllowFileClone = true;
- std::vector<std::string> IncludeWildcards;
- std::vector<std::string> ExcludeWildcards;
- uint64_t MaximumInMemoryPayloadSize = 512u * 1024u;
- bool PopulateCache = true;
- bool AppendNewContent = false;
- std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
- };
-
- void DownloadFolder(LoggerRef InLog,
- ProgressBase& Progress,
- TransferThreadWorkers& Workers,
- StorageInstance& Storage,
- const BuildStorageCache::Statistics& StorageCacheStats,
- const Oid& BuildId,
- const std::vector<Oid>& BuildPartIds,
- std::span<const std::string> BuildPartNames,
- const std::filesystem::path& DownloadSpecPath,
- const std::filesystem::path& Path,
- const DownloadOptions& Options)
- {
- ZEN_TRACE_CPU("DownloadFolder");
- ZEN_SCOPED_LOG(InLog);
-
- ProgressBar::SetLogOperationName(ProgressMode, "Download Folder");
-
- enum TaskSteps : uint32_t
- {
- CheckState,
- CompareState,
- Download,
- Verify,
- Cleanup,
- StepCount
- };
-
- auto EndProgress =
- MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); });
-
- Stopwatch DownloadTimer;
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount);
-
- const std::filesystem::path ZenTempFolder = ZenTempFolderPath(Options.ZenFolderPath);
- CreateDirectories(ZenTempFolder);
-
- std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
-
- CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId);
-
- std::vector<std::pair<Oid, std::string>> AllBuildParts =
- ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
-
- BuildManifest Manifest;
- if (!DownloadSpecPath.empty())
- {
- const std::filesystem::path AbsoluteDownloadSpecPath =
- DownloadSpecPath.is_relative() ? MakeSafeAbsolutePath(Path / DownloadSpecPath) : MakeSafeAbsolutePath(DownloadSpecPath);
- Manifest = ParseBuildManifest(DownloadSpecPath);
- }
-
- std::vector<ChunkedFolderContent> PartContents;
-
- std::unique_ptr<ChunkingController> ChunkController;
-
- std::vector<ChunkBlockDescription> BlockDescriptions;
- std::vector<IoHash> LooseChunkHashes;
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount);
-
- ChunkedFolderContent RemoteContent = GetRemoteContent(InLog,
- Storage,
- BuildId,
- AllBuildParts,
- Manifest,
- Options.IncludeWildcards,
- Options.ExcludeWildcards,
- ChunkController,
- PartContents,
- BlockDescriptions,
- LooseChunkHashes,
- IsQuiet,
- IsVerbose,
- DoExtraContentVerify);
-
- const std::uint64_t LargeAttachmentSize = Options.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
- GetFolderContentStatistics LocalFolderScanStats;
- ChunkingStatistics ChunkingStats;
-
- BuildSaveState LocalState;
-
- if (IsDir(Path))
- {
- if (!ChunkController && !IsQuiet)
- {
- ZEN_CONSOLE_INFO("Unspecified chunking algorithm, using default");
- ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
- }
- std::unique_ptr<ChunkingCache> ChunkCache(CreateNullChunkingCache());
-
- LocalState = GetLocalContent(Workers,
- LocalFolderScanStats,
- ChunkingStats,
- Path,
- ZenStateFilePath(Path / ZenFolderName),
- *ChunkController,
- *ChunkCache);
-
- std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalState.State.ChunkedContent.Paths, RemoteContent.Paths);
-
- BuildSaveState UntrackedLocalContent =
- GetLocalStateFromPaths(Workers, LocalFolderScanStats, ChunkingStats, Path, *ChunkController, *ChunkCache, UntrackedPaths);
-
- if (!UntrackedLocalContent.State.ChunkedContent.Paths.empty())
- {
- LocalState.State.ChunkedContent =
- MergeChunkedFolderContents(LocalState.State.ChunkedContent,
- std::vector<ChunkedFolderContent>{UntrackedLocalContent.State.ChunkedContent});
-
- // TODO: Helper
- LocalState.FolderState.Paths.insert(LocalState.FolderState.Paths.begin(),
- UntrackedLocalContent.FolderState.Paths.begin(),
- UntrackedLocalContent.FolderState.Paths.end());
- LocalState.FolderState.RawSizes.insert(LocalState.FolderState.RawSizes.begin(),
- UntrackedLocalContent.FolderState.RawSizes.begin(),
- UntrackedLocalContent.FolderState.RawSizes.end());
- LocalState.FolderState.Attributes.insert(LocalState.FolderState.Attributes.begin(),
- UntrackedLocalContent.FolderState.Attributes.begin(),
- UntrackedLocalContent.FolderState.Attributes.end());
- LocalState.FolderState.ModificationTicks.insert(LocalState.FolderState.ModificationTicks.begin(),
- UntrackedLocalContent.FolderState.ModificationTicks.begin(),
- UntrackedLocalContent.FolderState.ModificationTicks.end());
- }
-
- if (Options.AppendNewContent)
- {
- RemoteContent = ApplyChunkedContentOverlay(LocalState.State.ChunkedContent,
- RemoteContent,
- Options.IncludeWildcards,
- Options.ExcludeWildcards);
- }
-#if ZEN_BUILD_DEBUG
- ValidateChunkedFolderContent(RemoteContent,
- BlockDescriptions,
- LooseChunkHashes,
- Options.IncludeWildcards,
- Options.ExcludeWildcards);
-#endif // ZEN_BUILD_DEBUG
- }
- else
- {
- CreateDirectories(Path);
- }
- if (AbortFlag)
- {
- return;
- }
-
- LocalState.LocalPath = Path;
-
- {
- BuildsSelection::Build RemoteBuildState = {.Id = BuildId,
- .IncludeWildcards = Options.IncludeWildcards,
- .ExcludeWildcards = Options.ExcludeWildcards};
- RemoteBuildState.Parts.reserve(BuildPartIds.size());
- for (size_t PartIndex = 0; PartIndex < BuildPartIds.size(); PartIndex++)
- {
- RemoteBuildState.Parts.push_back(
- {BuildsSelection::BuildPart{.Id = BuildPartIds[PartIndex],
- .Name = PartIndex < BuildPartNames.size() ? BuildPartNames[PartIndex] : ""}});
- }
-
- if (Options.AppendNewContent)
- {
- LocalState.State.Selection.Builds.emplace_back(std::move(RemoteBuildState));
- }
- else
- {
- LocalState.State.Selection.Builds = std::vector<BuildsSelection::Build>{std::move(RemoteBuildState)};
- }
- }
-
- if ((Options.EnableTargetFolderScavenging || Options.AppendNewContent) && !Options.CleanTargetFolder &&
- CompareChunkedContent(RemoteContent, LocalState.State.ChunkedContent))
- {
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.",
- NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs()));
- }
-
- Stopwatch WriteStateTimer;
-
- CbObject StateObject = CreateBuildSaveStateObject(LocalState);
- CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path());
- TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView());
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
- }
-
- AddDownloadedPath(Options.SystemRootDir,
- BuildsDownloadInfo{.Selection = LocalState.State.Selection,
- .LocalPath = Path,
- .StateFilePath = ZenStateFilePath(Options.ZenFolderPath),
- .Iso8601Date = DateTime::Now().ToIso8601()});
- }
- else
- {
- ExtendableStringBuilder<128> BuildPartString;
- for (const std::pair<Oid, std::string>& BuildPart : AllBuildParts)
- {
- BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first));
- }
-
- uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0));
-
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize));
- }
-
- Stopwatch IndexTimer;
-
- const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalState.State.ChunkedContent);
- const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent);
-
- if (!IsQuiet)
- {
- ZEN_INFO("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
- }
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount);
-
- BuildsOperationUpdateFolder Updater(
- InLog,
- Progress,
- Storage,
- AbortFlag,
- PauseFlag,
- Workers.GetIOWorkerPool(),
- Workers.GetNetworkPool(),
- BuildId,
- Path,
- LocalState.State.ChunkedContent,
- LocalLookup,
- RemoteContent,
- RemoteLookup,
- BlockDescriptions,
- LooseChunkHashes,
- BuildsOperationUpdateFolder::Options{
- .IsQuiet = IsQuiet,
- .IsVerbose = IsVerbose,
- .AllowFileClone = Options.AllowFileClone,
- .UseSparseFiles = UseSparseFiles,
- .SystemRootDir = Options.SystemRootDir,
- .ZenFolderPath = Options.ZenFolderPath,
- .LargeAttachmentSize = LargeAttachmentSize,
- .PreferredMultipartChunkSize = PreferredMultipartChunkSize,
- .PartialBlockRequestMode = Options.PartialBlockRequestMode,
- .WipeTargetFolder = Options.CleanTargetFolder,
- .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging,
- .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging || Options.AppendNewContent,
- .ValidateCompletedSequences = Options.PostDownloadVerify,
- .ExcludeFolders = Options.ExcludeFolders,
- .MaximumInMemoryPayloadSize = Options.MaximumInMemoryPayloadSize,
- .PopulateCache = Options.PopulateCache});
- {
- ProgressBar::PushLogOperation(ProgressMode, "Download");
- auto _ = MakeGuard([]() { ProgressBar::PopLogOperation(ProgressMode); });
- FolderContent UpdatedLocalFolderState;
- Updater.Execute(UpdatedLocalFolderState);
-
- LocalState.State.ChunkedContent = RemoteContent;
- LocalState.FolderState = std::move(UpdatedLocalFolderState);
- }
-
- VerifyFolderStatistics VerifyFolderStats;
- if (!AbortFlag)
- {
- AddDownloadedPath(Options.SystemRootDir,
- BuildsDownloadInfo{.Selection = LocalState.State.Selection,
- .LocalPath = Path,
- .StateFilePath = ZenStateFilePath(Options.ZenFolderPath),
- .Iso8601Date = DateTime::Now().ToIso8601()});
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount);
-
- VerifyFolder(Workers,
- RemoteContent,
- RemoteLookup,
- Path,
- Options.ExcludeFolders,
- Options.PostDownloadVerify,
- VerifyFolderStats);
-
- Stopwatch WriteStateTimer;
- CbObject StateObject = CreateBuildSaveStateObject(LocalState);
-
- CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path());
- TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView());
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
- }
-
-#if 0
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(StateObject, SB);
- WriteFile(ZenStateFileJsonPath(Options.ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
-#endif // 0
- const uint64_t DownloadCount = Updater.m_DownloadStats.DownloadedChunkCount.load() +
- Updater.m_DownloadStats.DownloadedBlockCount.load() +
- Updater.m_DownloadStats.DownloadedPartialBlockCount.load();
- const uint64_t DownloadByteCount = Updater.m_DownloadStats.DownloadedChunkByteCount.load() +
- Updater.m_DownloadStats.DownloadedBlockByteCount.load() +
- Updater.m_DownloadStats.DownloadedPartialBlockByteCount.load();
- const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs();
-
- if (!IsQuiet)
- {
- std::string CloneInfo;
- if (Updater.m_DiskStats.CloneByteCount > 0)
- {
- CloneInfo = fmt::format(" ({} cloned)", NiceBytes(Updater.m_DiskStats.CloneByteCount.load()));
- }
-
- std::string DownloadDetails;
- {
- ExtendableStringBuilder<128> SB;
- BuildStorageBase::ExtendedStatistics ExtendedDownloadStats;
- if (Storage.BuildStorage->GetExtendedStatistics(ExtendedDownloadStats))
- {
- if (!ExtendedDownloadStats.ReceivedBytesPerSource.empty())
- {
- for (auto& It : ExtendedDownloadStats.ReceivedBytesPerSource)
- {
- if (SB.Size() > 0)
- {
- SB.Append(", "sv);
- }
- SB.Append(It.first);
- SB.Append(": "sv);
- SB.Append(NiceBytes(It.second));
- }
- }
- }
- if (Storage.CacheStorage)
- {
- if (SB.Size() > 0)
- {
- SB.Append(", "sv);
- }
- SB.Append("Cache: ");
- SB.Append(NiceBytes(StorageCacheStats.TotalBytesRead.load()));
- }
- if (SB.Size() > 0)
- {
- DownloadDetails = fmt::format(" ({})", SB.ToView());
- }
- }
-
- ZEN_CONSOLE(
- "Downloaded build {}, parts:{} in {}\n"
- " Scavenge: {} (Target: {}, Cache: {}, Others: {})\n"
- " Download: {} ({}) {}bits/s{}\n"
- " Write: {} ({}) {}B/s{}\n"
- " Clean: {}\n"
- " Finalize: {}\n"
- " Verify: {}",
- BuildId,
- BuildPartString.ToView(),
- NiceTimeSpanMs(DownloadTimeMs),
-
- NiceTimeSpanMs((Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs +
- Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs +
- Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs) /
- 1000),
- NiceTimeSpanMs(Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000),
- NiceTimeSpanMs(Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000),
- NiceTimeSpanMs(Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000),
-
- DownloadCount,
- NiceBytes(DownloadByteCount),
- NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)),
- DownloadDetails,
-
- Updater.m_DiskStats.WriteCount.load(),
- NiceBytes(Updater.m_WrittenChunkByteCount.load()),
- NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.WriteTimeUs, Updater.m_DiskStats.WriteByteCount.load())),
- CloneInfo,
-
- NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000),
-
- NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000),
-
- NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000));
- }
- }
- }
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount);
-
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), ZenTempFolder);
- }
-
- void ListBuild(StorageInstance& Storage,
- const Oid& BuildId,
- const std::vector<Oid>& BuildPartIds,
- std::span<const std::string> BuildPartNames,
- std::span<const std::string> IncludeWildcards,
- std::span<const std::string> ExcludeWildcards,
- CbObjectWriter* OptionalStructuredOutput)
- {
- std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
-
- CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId);
-
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->AddObjectId("buildId"sv, BuildId);
- OptionalStructuredOutput->AddObject("build"sv, BuildObject);
- }
-
- std::vector<std::pair<Oid, std::string>> AllBuildParts =
- ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
-
- if (!AllBuildParts.empty())
- {
- Stopwatch GetBuildPartTimer;
-
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->BeginArray("parts"sv);
- }
-
- for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++)
- {
- const Oid BuildPartId = AllBuildParts[BuildPartIndex].first;
- const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second;
- CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
-
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->BeginObject();
- OptionalStructuredOutput->AddObjectId("id"sv, BuildPartId);
- OptionalStructuredOutput->AddString("partName"sv, BuildPartName);
- }
- {
- if (OptionalStructuredOutput != nullptr)
- {
- }
- else if (!IsQuiet)
- {
- ZEN_CONSOLE("{}Part: {} ('{}'):\n",
- BuildPartIndex > 0 ? "\n" : "",
- BuildPartId,
- BuildPartName,
- NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
- NiceBytes(BuildPartManifest.GetSize()));
- }
-
- std::vector<std::filesystem::path> Paths;
- std::vector<IoHash> RawHashes;
- std::vector<uint64_t> RawSizes;
- std::vector<uint32_t> Attributes;
-
- SourcePlatform Platform;
- std::vector<IoHash> SequenceRawHashes;
- std::vector<uint32_t> ChunkCounts;
- std::vector<uint32_t> AbsoluteChunkOrders;
- std::vector<IoHash> LooseChunkHashes;
- std::vector<uint64_t> LooseChunkRawSizes;
- std::vector<IoHash> BlockRawHashes;
-
- ReadBuildContentFromCompactBinary(BuildPartManifest,
- Platform,
- Paths,
- RawHashes,
- RawSizes,
- Attributes,
- SequenceRawHashes,
- ChunkCounts,
- AbsoluteChunkOrders,
- LooseChunkHashes,
- LooseChunkRawSizes,
- BlockRawHashes);
-
- std::vector<size_t> Order(Paths.size());
- std::iota(Order.begin(), Order.end(), 0);
-
- std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) {
- const std::filesystem::path& LhsPath = Paths[Lhs];
- const std::filesystem::path& RhsPath = Paths[Rhs];
- return LhsPath < RhsPath;
- });
-
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->BeginArray("files"sv);
- }
- {
- for (size_t Index : Order)
- {
- const std::filesystem::path& Path = Paths[Index];
- if (IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(Path.generic_string()), /*CaseSensitive*/ true))
- {
- const IoHash& RawHash = RawHashes[Index];
- const uint64_t RawSize = RawSizes[Index];
- const uint32_t Attribute = Attributes[Index];
-
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->BeginObject();
- {
- OptionalStructuredOutput->AddString("path"sv, fmt::format("{}", Path));
- OptionalStructuredOutput->AddInteger("rawSize"sv, RawSize);
- OptionalStructuredOutput->AddHash("rawHash"sv, RawHash);
- switch (Platform)
- {
- case SourcePlatform::Windows:
- OptionalStructuredOutput->AddInteger("attributes"sv, Attribute);
- break;
- case SourcePlatform::MacOS:
- case SourcePlatform::Linux:
- OptionalStructuredOutput->AddString("chmod"sv, fmt::format("{:#04o}", Attribute));
- break;
- default:
- throw std::runtime_error(fmt::format("Unsupported platform: {}", (int)Platform));
- }
- }
- OptionalStructuredOutput->EndObject();
- }
- else
- {
- ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash);
- }
- }
- }
- }
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->EndArray(); // "files"
- }
- }
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->EndObject();
- }
- }
- if (OptionalStructuredOutput != nullptr)
- {
- OptionalStructuredOutput->EndArray(); // parts
- }
- }
- }
-
- void DiffFolders(TransferThreadWorkers& Workers,
- const std::filesystem::path& BasePath,
- const std::filesystem::path& ComparePath,
- ChunkingController& ChunkController,
- ChunkingCache& ChunkCache,
- const std::vector<std::string>& ExcludeFolders,
- const std::vector<std::string>& ExcludeExtensions)
- {
- ZEN_TRACE_CPU("DiffFolders");
-
- ProgressBar::SetLogOperationName(ProgressMode, "Diff Folders");
-
- enum TaskSteps : uint32_t
- {
- CheckBase,
- CheckCompare,
- Diff,
- Cleanup,
- StepCount
- };
-
- auto EndProgress =
- MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); });
-
- ChunkedFolderContent BaseFolderContent;
- ChunkedFolderContent CompareFolderContent;
-
- {
- auto IsAcceptedFolder = [ExcludeFolders](const std::string_view& RelativePath) -> bool {
- for (const std::string& ExcludeFolder : ExcludeFolders)
- {
- if (RelativePath.starts_with(ExcludeFolder))
- {
- if (RelativePath.length() == ExcludeFolder.length())
- {
- return false;
- }
- else if (RelativePath[ExcludeFolder.length()] == '/')
- {
- return false;
- }
- }
- }
- return true;
- };
-
- auto IsAcceptedFile = [ExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool {
- for (const std::string& ExcludeExtension : ExcludeExtensions)
- {
- if (RelativePath.ends_with(ExcludeExtension))
- {
- return false;
- }
- }
- return true;
- };
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckBase, TaskSteps::StepCount);
-
- GetFolderContentStatistics BaseGetFolderContentStats;
- ChunkingStatistics BaseChunkingStats;
- BaseFolderContent = ScanAndChunkFolder(Workers,
- BaseGetFolderContentStats,
- BaseChunkingStats,
- BasePath,
- IsAcceptedFolder,
- IsAcceptedFile,
- ChunkController,
- ChunkCache);
- if (AbortFlag)
- {
- return;
- }
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckCompare, TaskSteps::StepCount);
-
- GetFolderContentStatistics CompareGetFolderContentStats;
- ChunkingStatistics CompareChunkingStats;
- CompareFolderContent = ScanAndChunkFolder(Workers,
- CompareGetFolderContentStats,
- CompareChunkingStats,
- ComparePath,
- IsAcceptedFolder,
- IsAcceptedFile,
- ChunkController,
- ChunkCache);
-
- if (AbortFlag)
- {
- return;
- }
- }
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Diff, TaskSteps::StepCount);
-
- std::vector<IoHash> AddedHashes;
- std::vector<IoHash> RemovedHashes;
- uint64_t RemovedSize = 0;
- uint64_t AddedSize = 0;
-
- tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> BaseRawHashLookup;
- for (size_t PathIndex = 0; PathIndex < BaseFolderContent.RawHashes.size(); PathIndex++)
- {
- const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex];
- BaseRawHashLookup.insert_or_assign(RawHash, PathIndex);
- }
- tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CompareRawHashLookup;
- for (size_t PathIndex = 0; PathIndex < CompareFolderContent.RawHashes.size(); PathIndex++)
- {
- const IoHash& RawHash = CompareFolderContent.RawHashes[PathIndex];
- if (!BaseRawHashLookup.contains(RawHash))
- {
- AddedHashes.push_back(RawHash);
- AddedSize += CompareFolderContent.RawSizes[PathIndex];
- }
- CompareRawHashLookup.insert_or_assign(RawHash, PathIndex);
- }
- for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++)
- {
- const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex];
- if (!CompareRawHashLookup.contains(RawHash))
- {
- RemovedHashes.push_back(RawHash);
- RemovedSize += BaseFolderContent.RawSizes[PathIndex];
- }
- }
-
- uint64_t BaseTotalRawSize = 0;
- for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++)
- {
- BaseTotalRawSize += BaseFolderContent.RawSizes[PathIndex];
- }
-
- double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0;
-
- ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept",
- RemovedHashes.size(),
- NiceBytes(RemovedSize),
- AddedHashes.size(),
- NiceBytes(AddedSize),
- BaseFolderContent.Paths.size() - RemovedHashes.size(),
- NiceBytes(BaseTotalRawSize - RemovedSize),
- KeptPercent);
-
- uint64_t CompareTotalRawSize = 0;
-
- uint64_t FoundChunkCount = 0;
- uint64_t FoundChunkSize = 0;
- uint64_t NewChunkCount = 0;
- uint64_t NewChunkSize = 0;
- const ChunkedContentLookup BaseFolderLookup = BuildChunkedContentLookup(BaseFolderContent);
- for (uint32_t ChunkIndex = 0; ChunkIndex < CompareFolderContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++)
- {
- const IoHash& ChunkHash = CompareFolderContent.ChunkedContent.ChunkHashes[ChunkIndex];
- if (BaseFolderLookup.ChunkHashToChunkIndex.contains(ChunkHash))
- {
- FoundChunkCount++;
- FoundChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
- }
- else
- {
- NewChunkCount++;
- NewChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
- }
- CompareTotalRawSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ ExtendableStringBuilder<1024> SB;
+ CompactBinaryToJson(ResponseView, SB);
+ WriteFile(Path, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
}
-
- double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0;
- double NewPercent = CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0;
-
- ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.",
- FoundChunkCount,
- NiceBytes(FoundChunkSize),
- FoundPercent,
- CompareFolderContent.ChunkedContent.ChunkHashes.size(),
- NiceBytes(CompareTotalRawSize),
- BaseFolderContent.ChunkedContent.ChunkHashes.size(),
- NiceBytes(BaseTotalRawSize),
- NewChunkCount,
- NiceBytes(NewChunkSize),
- NewPercent);
-
- ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount);
}
} // namespace builds_impl
-//////////////////////////////////////////////////////////////////////////////////////////////////////
-// BuildsCommand - Option-adding helpers
-//
+//////////////////////////////////////////////////////////////////////////
void
-BuildsCommand::AddSystemOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddSystemOptions(cxxopts::Options& Ops)
{
- Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>");
+ Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(SystemRootDir), "<systemdir>");
Ops.add_option("",
"",
"use-sparse-files",
"Enable use of sparse files when writing large files. Defaults to true.",
- cxxopts::value(m_UseSparseFiles),
+ cxxopts::value(UseSparseFiles),
"<usesparsefiles>");
}
void
-BuildsCommand::AddCloudOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddCloudOptions(cxxopts::Options& Ops)
{
- m_AuthOptions.AddOptions(Ops);
+ AuthOptions.AddOptions(Ops);
- Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>");
- Ops.add_option("cloud build",
- "",
- "url",
- "Cloud Builds host url (legacy - use --override-host)",
- cxxopts::value(m_OverrideHost),
- "<url>");
- Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), "<cloud-url>");
- Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>");
+ Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(OverrideHost), "<override-host>");
+ Ops.add_option("cloud build", "", "url", "Cloud Builds host url (legacy - use --override-host)", cxxopts::value(OverrideHost), "<url>");
+ Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(Url), "<cloud-url>");
+ Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(Host), "<host>");
Ops.add_option("cloud build",
"",
"assume-http2",
"Assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake",
- cxxopts::value(m_AssumeHttp2),
+ cxxopts::value(AssumeHttp2),
"<assumehttp2>");
Ops.add_option("cloud build",
"",
"verbose-http",
"Enable verbose option for http client",
- cxxopts::value(m_VerboseHttp),
+ cxxopts::value(VerboseHttp),
"<verbosehttp>");
- Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), "<namespace>");
- Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), "<bucket>");
- Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(m_AllowRedirect), "<allow-redirect>");
+ Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(Namespace), "<namespace>");
+ Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(Bucket), "<bucket>");
+ Ops.add_option("cloud build", "", "allow-redirect", "Allow redirect of requests", cxxopts::value(AllowRedirect), "<allow-redirect>");
}
void
-BuildsCommand::AddFileOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddFileOptions(cxxopts::Options& Ops)
{
- Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(m_StoragePath), "<storagepath>");
+ Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(StoragePath), "<storagepath>");
Ops.add_option("filestorage",
"",
"json-metadata",
"Write build, part and block metadata as .json files in addition to .cb files",
- cxxopts::value(m_WriteMetadataAsJson),
+ cxxopts::value(WriteMetadataAsJson),
"<jsonmetadata>");
}
void
-BuildsCommand::AddCacheOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddCacheOptions(cxxopts::Options& Ops)
{
- Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>");
+ Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(ZenCacheHost), "<zenhost>");
}
void
-BuildsCommand::AddOutputOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddOutputOptions(cxxopts::Options& Ops)
{
- Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), "<plainprogress>");
- Ops.add_option("output",
- "",
- "log-progress",
- "Write @progress style progress to output",
- cxxopts::value(m_LogProgress),
- "<logprogress>");
- Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>");
- Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(m_Quiet), "<quiet>");
+ Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(PlainProgress), "<plainprogress>");
+ Ops.add_option("output", "", "log-progress", "Write @progress style progress to output", cxxopts::value(LogProgress), "<logprogress>");
+ Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(Verbose), "<verbose>");
+ Ops.add_option("output", "", "quiet", "Suppress non-essential output", cxxopts::value(Quiet), "<quiet>");
}
void
-BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddWorkerOptions(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
"boost-worker-count",
"Increase the number of worker threads - may cause computer to be less responsive",
- cxxopts::value(m_BoostWorkerCount),
+ cxxopts::value(BoostWorkerCount),
"<boostworkercount>");
Ops.add_option("",
@@ -2077,47 +301,47 @@ BuildsCommand::AddWorkerOptions(cxxopts::Options& Ops)
"boost-worker-memory",
"Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to "
"be less responsive due to high memory usage",
- cxxopts::value(m_BoostWorkerMemory),
+ cxxopts::value(BoostWorkerMemory),
"<boostworkermemory>");
Ops.add_option("",
"",
"boost-workers",
"Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive",
- cxxopts::value(m_BoostWorkers),
+ cxxopts::value(BoostWorkers),
"<boostworkermemory>");
}
void
-BuildsCommand::AddZenFolderOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddZenFolderOptions(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
"zen-folder-path",
- fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", builds_impl::ZenFolderName),
- cxxopts::value(m_ZenFolderPath),
+ fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", ZenFolderName),
+ cxxopts::value(ZenFolderPath),
"<boostworkers>");
}
void
-BuildsCommand::AddChunkingCacheOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddChunkingCacheOptions(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
"chunking-cache-path",
"Path to cache for chunking information of scanned files. Default is empty resulting in no caching",
- cxxopts::value(m_ChunkingCachePath),
+ cxxopts::value(ChunkingCachePath),
"<chunkingcachepath>");
}
void
-BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddWildcardOptions(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
"wildcard",
"Windows style wildcard(s) (using * and ?) to match file paths to include, separated by ;",
- cxxopts::value(m_IncludeWildcard),
+ cxxopts::value(IncludeWildcard),
"<wildcard>");
Ops.add_option("",
@@ -2125,46 +349,46 @@ BuildsCommand::AddWildcardOptions(cxxopts::Options& Ops)
"exclude-wildcard",
"Windows style wildcard(s) (using * and ?) to match file paths to exclude, separated by ;. Applied after --wildcard "
"include filter",
- cxxopts::value(m_ExcludeWildcard),
+ cxxopts::value(ExcludeWildcard),
"<excludewildcard>");
}
void
-BuildsCommand::AddExcludeFolderOption(cxxopts::Options& Ops)
+BuildsConfiguration::AddExcludeFolderOption(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
"exclude-folders",
"Names of folders to exclude, separated by ;",
- cxxopts::value(m_ExcludeFolders),
+ cxxopts::value(ExcludeFolders),
"<excludefolders>");
}
void
-BuildsCommand::AddExcludeExtensionsOption(cxxopts::Options& Ops)
+BuildsConfiguration::AddExcludeExtensionsOption(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
"exclude-extensions",
"Extensions to exclude, separated by ;"
"include filter",
- cxxopts::value(m_ExcludeExtensions),
+ cxxopts::value(ExcludeExtensions),
"<excludeextensions>");
}
void
-BuildsCommand::AddMultipartOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddMultipartOptions(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
"allow-multipart",
"Allow large attachments to be transfered using multipart protocol. Defaults to true.",
- cxxopts::value(m_AllowMultiparts),
+ cxxopts::value(AllowMultiparts),
"<allowmultipart>");
}
void
-BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddPartialBlockRequestOptions(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
@@ -2177,12 +401,12 @@ BuildsCommand::AddPartialBlockRequestOptions(cxxopts::Options& Ops)
"allowed to host\n"
" true = multiple partial block ranges requests per block allowed to zen cache and host\n"
"Defaults to 'mixed'.",
- cxxopts::value(m_AllowPartialBlockRequests),
+ cxxopts::value(AllowPartialBlockRequests),
"<allowpartialblockrequests>");
}
void
-BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops)
+BuildsConfiguration::AddAppendNewContentOptions(cxxopts::Options& Ops)
{
Ops.add_option("",
"",
@@ -2191,26 +415,26 @@ BuildsCommand::AddAppendNewContentOptions(cxxopts::Options& Ops)
" false = the local content will be replaced by the remote content\n"
" true = the remote data will be overlayed on top of local data\n"
"Defaults to false.",
- cxxopts::value(m_AppendNewContent),
+ cxxopts::value(AppendNewContent),
"<append>");
}
BuildsCommand::BuildsCommand()
-: m_ListNamespacesSubCmd(*this)
-, m_ListSubCmd(*this)
-, m_ListBlocksSubCmd(*this)
-, m_UploadSubCmd(*this)
-, m_DownloadSubCmd(*this)
-, m_LsSubCmd(*this)
-, m_DiffSubCmd(*this)
-, m_FetchBlobSubCmd(*this)
-, m_PrimeCacheSubCmd(*this)
-, m_PauseSubCmd(*this)
-, m_ResumeSubCmd(*this)
-, m_AbortSubCmd(*this)
-, m_ValidatePartSubCmd(*this)
-, m_TestSubCmd(*this)
-, m_MultiTestDownloadSubCmd(*this)
+: m_ListNamespacesSubCmd(m_Configuration)
+, m_ListSubCmd(m_Configuration)
+, m_ListBlocksSubCmd(m_Configuration)
+, m_UploadSubCmd(m_Configuration)
+, m_DownloadSubCmd(m_Configuration)
+, m_LsSubCmd(m_Configuration)
+, m_DiffSubCmd(m_Configuration)
+, m_FetchBlobSubCmd(m_Configuration)
+, m_PrimeCacheSubCmd(m_Configuration)
+, m_PauseSubCmd(m_Configuration)
+, m_ResumeSubCmd(m_Configuration)
+, m_AbortSubCmd(m_Configuration)
+, m_ValidatePartSubCmd(m_Configuration)
+, m_TestSubCmd(m_Configuration)
+, m_MultiTestDownloadSubCmd(m_Configuration)
{
m_Options.add_options()("h,help", "Print help");
m_Options.add_option("__hidden__", "", "subcommand", "", cxxopts::value<std::string>(m_SubCommand)->default_value(""), "");
@@ -2246,158 +470,222 @@ BuildsCommand::OnParentOptionsParsed(const ZenCliOptions& /*GlobalOptions*/)
#endif // ZEN_PLATFORM_WINDOWS
// Validate output options
- if (m_Verbose && m_Quiet)
+ if (m_Configuration.Verbose && m_Configuration.Quiet)
{
throw OptionParseException("'--verbose' conflicts with '--quiet'", {});
}
- if (m_LogProgress && m_PlainProgress)
+ if (m_Configuration.LogProgress && m_Configuration.PlainProgress)
{
throw OptionParseException("'--plain-progress' conflicts with '--log-progress'", {});
}
- if (m_LogProgress && m_Quiet)
+ if (m_Configuration.LogProgress && m_Configuration.Quiet)
{
throw OptionParseException("'--quiet' conflicts with '--log-progress'", {});
}
- if (m_PlainProgress && m_Quiet)
+ if (m_Configuration.PlainProgress && m_Configuration.Quiet)
{
throw OptionParseException("'--quiet' conflicts with '--plain-progress'", {});
}
- IsVerbose = m_Verbose;
- IsQuiet = m_Quiet;
- if (m_LogProgress)
+ if (m_Configuration.LogProgress)
{
- ProgressMode = ProgressBar::Mode::Log;
+ m_Configuration.ProgressMode = ConsoleProgressMode::Log;
}
- else if (m_PlainProgress)
+ else if (m_Configuration.PlainProgress)
{
- ProgressMode = ProgressBar::Mode::Plain;
+ m_Configuration.ProgressMode = ConsoleProgressMode::Plain;
}
- else if (IsQuiet)
+ else if (m_Configuration.Quiet)
{
- ProgressMode = ProgressBar::Mode::Quiet;
+ m_Configuration.ProgressMode = ConsoleProgressMode::Quiet;
}
else
{
- ProgressMode = ProgressBar::Mode::Pretty;
+ m_Configuration.ProgressMode = ConsoleProgressMode::Pretty;
}
- if (m_BoostWorkers)
+ if (m_Configuration.BoostWorkers)
{
- m_BoostWorkerCount = true;
- m_BoostWorkerMemory = true;
+ m_Configuration.BoostWorkerCount = true;
+ m_Configuration.BoostWorkerMemory = true;
}
// Parse system options
- if (m_SystemRootDir.empty())
+ if (m_Configuration.SystemRootDir.empty())
{
- m_SystemRootDir = PickDefaultSystemRootDirectory();
+ m_Configuration.SystemRootDir = PickDefaultSystemRootDirectory();
}
- MakeSafeAbsolutePathInPlace(m_SystemRootDir);
-
- UseSparseFiles = m_UseSparseFiles;
+ MakeSafeAbsolutePathInPlace(m_Configuration.SystemRootDir);
+ MakeSafeAbsolutePathInPlace(m_Configuration.ChunkingCachePath);
return true;
}
+std::atomic<bool>&
+BuildsSubCmdBase::AbortFlag() const
+{
+ return builds_impl::AbortFlag;
+}
+
+std::atomic<bool>&
+BuildsSubCmdBase::PauseFlag() const
+{
+ return builds_impl::PauseFlag;
+}
+
+std::unique_ptr<ProgressBase>
+BuildsSubCmdBase::CreateProgress() const
+{
+ return std::unique_ptr<ProgressBase>(CreateConsoleProgress(m_Config.ProgressMode));
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void
+BuildsSubCmdBase::LogBanner()
+{
+ if (!m_Config.Quiet)
+ {
+ ZenCmdBase::LogExecutableVersionAndPid();
+ }
+}
+
+void
+BuildsSubCmdBase::LogWorkersInfo(const TransferThreadWorkers& Workers)
+{
+ if (!m_Config.Quiet)
+ {
+ ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
+ }
+}
+
void
-BuildsCommand::ParseStorageOptions(std::string& BuildId, bool RequireNamespace, bool RequireBucket, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::CleanZenFolder()
{
- if (!m_Url.empty())
+ CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), AbortFlag(), PauseFlag(), GetZenFolderPath());
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+BuildsSubCmdBase::ResolvedStorage
+BuildsSubCmdBase::ParseStorageOptions(std::string& BuildId,
+ const CreateBuildStorageOptions& Options,
+ cxxopts::Options& SubOpts,
+ const std::filesystem::path& SystemRootDirOverride,
+ const std::filesystem::path& StoragePathOverride)
+{
+ ResolvedStorage Resolved{.SystemRootDir = SystemRootDirOverride.empty() ? m_Config.SystemRootDir : SystemRootDirOverride,
+ .Host = m_Config.Host,
+ .Namespace = m_Config.Namespace,
+ .Bucket = m_Config.Bucket,
+ .StoragePath = StoragePathOverride.empty() ? m_Config.StoragePath : StoragePathOverride};
+
+ if (!m_Config.Url.empty())
{
- if (!m_Host.empty())
+ if (!Resolved.Host.empty())
{
- throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", m_Host, m_Url), SubOpts.help());
+ throw OptionParseException(fmt::format("'--host' ('{}') conflicts with '--url' ('{}')", Resolved.Host, m_Config.Url),
+ SubOpts.help());
}
- if (!m_Bucket.empty())
+ if (!Resolved.Bucket.empty())
{
- throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", m_Bucket, m_Url), SubOpts.help());
+ throw OptionParseException(fmt::format("'--bucket' ('{}') conflicts with '--url' ('{}')", Resolved.Bucket, m_Config.Url),
+ SubOpts.help());
}
if (!BuildId.empty())
{
- throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Url), SubOpts.help());
+ throw OptionParseException(fmt::format("'--buildid' ('{}') conflicts with '--url' ('{}')", BuildId, m_Config.Url),
+ SubOpts.help());
}
- if (!ParseBuildStorageUrl(m_Url, m_Host, m_Namespace, m_Bucket, BuildId))
+ if (!ParseBuildStorageUrl(m_Config.Url, Resolved.Host, Resolved.Namespace, Resolved.Bucket, BuildId))
{
throw OptionParseException("'--url' ('{}') is malformed, it does not match the Cloud Artifact URL format", SubOpts.help());
}
}
- if (!m_OverrideHost.empty() || !m_Host.empty())
+ if (!m_Config.OverrideHost.empty() || !Resolved.Host.empty())
{
- if (!m_StoragePath.empty())
+ if (!Resolved.StoragePath.empty())
{
throw OptionParseException(
- fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", m_StoragePath),
+ fmt::format("'--storage-path' ('{}') conflicts with '--host'/'--url'/'--override-host' options", Resolved.StoragePath),
SubOpts.help());
}
- if (RequireNamespace && m_Namespace.empty())
+ if (Options.RequireNamespace && Resolved.Namespace.empty())
{
throw OptionParseException("'--namespace' is required", SubOpts.help());
}
- if (RequireBucket && m_Bucket.empty())
+ if (Options.RequireBucket && Resolved.Bucket.empty())
{
throw OptionParseException("'--bucket' is required", SubOpts.help());
}
}
- else if (m_StoragePath.empty())
+ else if (Resolved.StoragePath.empty())
{
throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help());
}
- MakeSafeAbsolutePathInPlace(m_StoragePath);
+ MakeSafeAbsolutePathInPlace(Resolved.StoragePath);
+ return Resolved;
}
StorageInstance
-BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
- BuildStorageCache::Statistics& StorageCacheStats,
- const std::filesystem::path& TempPath,
- std::string& BuildId,
- bool RequireNamespace,
- bool RequireBucket,
- bool BoostCacheBackgroundWorkerPool,
- std::unique_ptr<AuthMgr>& Auth,
- cxxopts::Options& SubOpts)
+BuildsSubCmdBase::CreateBuildStorage(const std::filesystem::path& ZenFolderDefault,
+ const CreateBuildStorageOptions& Options,
+ cxxopts::Options& SubOpts,
+ BuildStorageBase::Statistics& StorageStats,
+ BuildStorageCache::Statistics& StorageCacheStats,
+ std::unique_ptr<AuthMgr>& Auth,
+ const ResolvedStorage& Resolved)
{
using namespace builds_impl;
- ParseStorageOptions(BuildId, RequireNamespace, RequireBucket, SubOpts);
+ ResolveZenFolderPath(ZenFolderDefault);
+ CreateDirectories(GetZenFolderPath());
+ const std::filesystem::path TempPath = ZenTempFolderPath(GetZenFolderPath());
- HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient",
- .AssumeHttp2 = m_AssumeHttp2,
- .AllowResume = true,
- .RetryCount = 2,
- .Verbose = m_VerboseHttp,
- .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)};
+ HttpClientSettings ClientSettings{
+ .LogCategory = "httpbuildsclient",
+ .AssumeHttp2 = m_Config.AssumeHttp2,
+ .AllowResume = true,
+ .RetryCount = 2,
+ .Verbose = m_Config.VerboseHttp,
+ .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)};
std::string StorageDescription;
std::string CacheDescription;
StorageInstance Result;
- if (!m_Host.empty() || !m_OverrideHost.empty())
- {
- m_AuthOptions.ParseOptions(SubOpts,
- m_SystemRootDir,
- ClientSettings,
- m_Host.empty() ? m_OverrideHost : m_Host,
- Auth,
- IsQuiet,
- /*Hidden*/ false,
- m_Verbose);
-
- BuildStorageResolveResult ResolveRes =
- ResolveBuildStorage(ConsoleLog(), ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All, m_Verbose);
+ if (!Resolved.Host.empty() || !m_Config.OverrideHost.empty())
+ {
+ AuthCommandLineOptions AuthOpts = m_Config.AuthOptions;
+ AuthOpts.ParseOptions(SubOpts,
+ Resolved.SystemRootDir,
+ ClientSettings,
+ Resolved.Host.empty() ? m_Config.OverrideHost : Resolved.Host,
+ Auth,
+ m_Config.Quiet,
+ /*Hidden*/ false,
+ m_Config.Verbose);
+
+ BuildStorageResolveResult ResolveRes = ResolveBuildStorage(ConsoleLog(),
+ ClientSettings,
+ Resolved.Host,
+ m_Config.OverrideHost,
+ m_Config.ZenCacheHost,
+ ZenCacheResolveMode::All,
+ m_Config.Verbose);
if (!ResolveRes.Cloud.Address.empty())
{
ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2;
Result.BuildStorageHttp =
- std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); });
+ std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings, [this]() { return AbortFlag().load(); });
Result.BuildStorage = CreateJupiterBuildStorage(Log(),
*Result.BuildStorageHttp,
StorageStats,
- m_Namespace,
- m_Bucket,
- m_AllowRedirect,
+ Resolved.Namespace,
+ Resolved.Bucket,
+ m_Config.AllowRedirect,
TempPath / "storage");
Result.BuildStorageHost = ResolveRes.Cloud;
@@ -2408,8 +696,8 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
ResolveRes.Cloud.Name,
(ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address),
Result.BuildStorageHttp->GetSessionId(),
- m_Namespace,
- m_Bucket,
+ Resolved.Namespace,
+ Resolved.Bucket,
NiceLatencyNs(HostLatencyNs));
if (!ResolveRes.Cache.Address.empty())
@@ -2423,17 +711,17 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
.AssumeHttp2 = ResolveRes.Cache.AssumeHttp2,
.AllowResume = true,
.RetryCount = 0,
- .Verbose = m_VerboseHttp,
- .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)},
- []() { return AbortFlag.load(); });
+ .Verbose = m_Config.VerboseHttp,
+ .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)},
+ [this]() { return AbortFlag().load(); });
Result.CacheStorage =
CreateZenBuildStorageCache(*Result.CacheHttp,
StorageCacheStats,
- m_Namespace,
- m_Bucket,
+ Resolved.Namespace,
+ Resolved.Bucket,
TempPath / "zencache",
- BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
- : GetTinyWorkerPool(EWorkloadType::Background));
+ Options.BoostCacheBackgroundWorkers ? GetSmallWorkerPool(EWorkloadType::Background)
+ : GetTinyWorkerPool(EWorkloadType::Background));
Result.CacheHost = ResolveRes.Cache;
uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0;
@@ -2445,69 +733,69 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
Result.CacheHttp->GetSessionId(),
NiceLatencyNs(CacheLatencyNs));
- if (!m_Namespace.empty())
+ if (!Resolved.Namespace.empty())
{
- CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
+ CacheDescription += fmt::format(". Namespace '{}'", Resolved.Namespace);
}
- if (!m_Bucket.empty())
+ if (!Resolved.Bucket.empty())
{
- CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
+ CacheDescription += fmt::format(" Bucket '{}'", Resolved.Bucket);
}
}
}
}
- else if (!m_StoragePath.empty())
+ else if (!Resolved.StoragePath.empty())
{
- StorageDescription = fmt::format("folder {}", m_StoragePath);
- Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
+ StorageDescription = fmt::format("folder {}", Resolved.StoragePath);
+ Result.BuildStorage = CreateFileBuildStorage(Resolved.StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
- Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(),
+ Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = Resolved.StoragePath.generic_string(),
.Name = "Disk",
.LatencySec = 1.0 / 100000, // 1 us
.Caps = {.MaxRangeCountPerRequest = 2048u}};
- if (!m_ZenCacheHost.empty())
+ if (!m_Config.ZenCacheHost.empty())
{
- ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp);
+ ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_Config.ZenCacheHost, m_Config.AssumeHttp2, m_Config.VerboseHttp);
if (TestResult.Success)
{
Result.CacheHttp = std::make_unique<HttpClient>(
- m_ZenCacheHost,
+ m_Config.ZenCacheHost,
HttpClientSettings{
.LogCategory = "httpcacheclient",
.ConnectTimeout = std::chrono::milliseconds{3000},
.Timeout = std::chrono::milliseconds{30000},
- .AssumeHttp2 = m_AssumeHttp2,
+ .AssumeHttp2 = m_Config.AssumeHttp2,
.AllowResume = true,
.RetryCount = 0,
- .Verbose = m_VerboseHttp,
- .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)},
- []() { return AbortFlag.load(); });
+ .Verbose = m_Config.VerboseHttp,
+ .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory)},
+ [this]() { return AbortFlag().load(); });
Result.CacheStorage =
CreateZenBuildStorageCache(*Result.CacheHttp,
StorageCacheStats,
- m_Namespace,
- m_Bucket,
+ Resolved.Namespace,
+ Resolved.Bucket,
TempPath / "zencache",
- BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background)
- : GetTinyWorkerPool(EWorkloadType::Background));
- Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_ZenCacheHost,
- .Name = m_ZenCacheHost,
- .AssumeHttp2 = m_AssumeHttp2,
+ Options.BoostCacheBackgroundWorkers ? GetSmallWorkerPool(EWorkloadType::Background)
+ : GetTinyWorkerPool(EWorkloadType::Background));
+ Result.CacheHost = BuildStorageResolveResult::Host{.Address = m_Config.ZenCacheHost,
+ .Name = m_Config.ZenCacheHost,
+ .AssumeHttp2 = m_Config.AssumeHttp2,
.LatencySec = TestResult.LatencySeconds,
.Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}};
CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId());
- if (!m_Namespace.empty())
+ if (!Resolved.Namespace.empty())
{
- CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
+ CacheDescription += fmt::format(". Namespace '{}'", Resolved.Namespace);
}
- if (!m_Bucket.empty())
+ if (!Resolved.Bucket.empty())
{
- CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
+ CacheDescription += fmt::format(" Bucket '{}'", Resolved.Bucket);
}
}
}
@@ -2517,7 +805,7 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOpts.help());
}
- if (!IsQuiet)
+ if (!m_Config.Quiet)
{
ZEN_CONSOLE("Remote: {}", StorageDescription);
if (!Result.CacheHost.Name.empty())
@@ -2529,7 +817,7 @@ BuildsCommand::CreateBuildStorage(BuildStorageBase::Statistics& StorageStats,
}
Oid
-BuildsCommand::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& SubOpts)
{
if (BuildIdStr.length() != Oid::StringLength)
{
@@ -2548,7 +836,7 @@ BuildsCommand::ParseBuildId(const std::string& BuildIdStr, cxxopts::Options& Sub
}
Oid
-BuildsCommand::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Options& SubOpts)
{
if (BuildPartIdStr.length() != Oid::StringLength)
{
@@ -2567,7 +855,7 @@ BuildsCommand::ParseBuildPartId(const std::string& BuildPartIdStr, cxxopts::Opti
}
std::vector<Oid>
-BuildsCommand::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs, cxxopts::Options& SubOpts)
{
std::vector<Oid> BuildPartIds;
for (const std::string& BuildPartId : BuildPartIdStrs)
@@ -2582,7 +870,7 @@ BuildsCommand::ParseBuildPartIds(const std::vector<std::string>& BuildPartIdStrs
}
std::vector<std::string>
-BuildsCommand::ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParseBuildPartNames(const std::vector<std::string>& BuildPartNameStrs, cxxopts::Options& SubOpts)
{
std::vector<std::string> BuildPartNames;
for (const std::string& BuildPartName : BuildPartNameStrs)
@@ -2597,10 +885,10 @@ BuildsCommand::ParseBuildPartNames(const std::vector<std::string>& BuildPartName
}
CbObject
-BuildsCommand::ParseBuildMetadata(bool CreateBuild,
- std::filesystem::path& BuildMetadataPath,
- const std::string& BuildMetadata,
- cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParseBuildMetadata(bool CreateBuild,
+ std::filesystem::path& BuildMetadataPath,
+ const std::string& BuildMetadata,
+ cxxopts::Options& SubOpts)
{
if (CreateBuild)
{
@@ -2659,7 +947,7 @@ BuildsCommand::ParseBuildMetadata(bool CreateBuild,
}
void
-BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts)
{
if (Path.empty())
{
@@ -2669,7 +957,7 @@ BuildsCommand::ParsePath(std::filesystem::path& Path, cxxopts::Options& SubOpts)
}
IoHash
-BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& SubOpts)
{
if (BlobHashStr.empty())
{
@@ -2693,7 +981,7 @@ BuildsCommand::ParseBlobHash(const std::string& BlobHashStr, cxxopts::Options& S
}
void
-BuildsCommand::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards)
+BuildsSubCmdBase::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards)
{
auto SplitAndAppendWildcard = [](const std::string_view Wildcard, std::vector<std::string>& Output) {
ForEachStrTok(Wildcard, ';', [&Output](std::string_view Wildcard) {
@@ -2719,12 +1007,13 @@ BuildsCommand::ParseFileFilters(std::vector<std::string>& OutIncludeWildcards, s
});
};
- SplitAndAppendWildcard(m_IncludeWildcard, OutIncludeWildcards);
- SplitAndAppendWildcard(m_ExcludeWildcard, OutExcludeWildcards);
+ SplitAndAppendWildcard(m_Config.IncludeWildcard, OutIncludeWildcards);
+ SplitAndAppendWildcard(m_Config.ExcludeWildcard, OutExcludeWildcards);
}
void
-BuildsCommand::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions)
+BuildsSubCmdBase::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExcludeFolders,
+ std::vector<std::string>& OutExcludeExtensions)
{
auto SplitAndAppendExclusion = [](const std::string_view Input, std::vector<std::string>& Output) {
ForEachStrTok(Input, ";,", [&Output](std::string_view Exclusion) {
@@ -2741,34 +1030,31 @@ BuildsCommand::ParseExcludeFolderAndExtension(std::vector<std::string>& OutExclu
});
};
- SplitAndAppendExclusion(m_ExcludeFolders, OutExcludeFolders);
- SplitAndAppendExclusion(m_ExcludeExtensions, OutExcludeExtensions);
+ SplitAndAppendExclusion(m_Config.ExcludeFolders, OutExcludeFolders);
+ SplitAndAppendExclusion(m_Config.ExcludeExtensions, OutExcludeExtensions);
}
void
-BuildsCommand::ResolveZenFolderPath(const std::filesystem::path& DefaultPath)
+BuildsSubCmdBase::ResolveZenFolderPath(const std::filesystem::path& DefaultPath)
{
- if (m_ZenFolderPath.empty())
- {
- m_ZenFolderPath = DefaultPath;
- }
- MakeSafeAbsolutePathInPlace(m_ZenFolderPath);
+ m_ResolvedZenFolderPath = m_Config.ZenFolderPath.empty() ? DefaultPath : m_Config.ZenFolderPath;
+ MakeSafeAbsolutePathInPlace(m_ResolvedZenFolderPath);
}
EPartialBlockRequestMode
-BuildsCommand::ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts)
+BuildsSubCmdBase::ParseAllowPartialBlockRequests(cxxopts::Options& SubOpts)
{
- EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests);
+ EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_Config.AllowPartialBlockRequests);
if (Mode == EPartialBlockRequestMode::Invalid)
{
- throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests),
+ throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_Config.AllowPartialBlockRequests),
SubOpts.help());
}
return Mode;
}
void
-BuildsCommand::ParseZenProcessId(int& ZenProcessId)
+BuildsSubCmdBase::ParseZenProcessId(int& ZenProcessId)
{
if (ZenProcessId == -1)
{
@@ -2789,20 +1075,15 @@ BuildsCommand::ParseZenProcessId(int& ZenProcessId)
//////////////////////////////////////////////////////////////////////////
-// ---------------------------------------------------------------------------
-// Subcommand implementations
-// ---------------------------------------------------------------------------
-
-BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("list-namespaces", "List all namespaces and optionally their buckets")
-, m_Parent(Parent)
+BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsConfiguration& Config)
+: BuildsSubCmdBase(Config, "list-namespaces", "List all namespaces and optionally their buckets")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
Opts.add_option("", "", "recursive", "Enable fetch of buckets within namespaces also", cxxopts::value(m_Recursive), "<recursive>");
Opts.add_option("",
"",
@@ -2817,36 +1098,21 @@ BuildsListNamespacesSubCmd::BuildsListNamespacesSubCmd(BuildsCommand& Parent)
void
BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
-
if (!m_ResultPath.empty())
{
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
+ LogBanner();
}
- BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
+ cxxopts::Options& Opts = SubOptions();
- CreateDirectories(m_Parent.GetZenFolderPath());
- auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); });
-
- std::unique_ptr<AuthMgr> Auth;
- std::string DummyBuildId;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- DummyBuildId,
- /*RequireNamespace*/ false,
- /*RequireBucket*/ false,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
+ std::string DummyBuildId;
+ BuildStorageBase::Statistics StorageStats;
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {.RequireNamespace = false, .RequireBucket = false}, Opts);
+ StorageInstance Storage =
+ CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved);
+ auto _ = MakeGuard([this]() { CleanZenFolder(); });
CbObject Response = Storage.BuildStorage->ListNamespaces(m_Recursive);
ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
@@ -2858,29 +1124,20 @@ BuildsListNamespacesSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
}
else
{
- std::filesystem::path ResultPath = MakeSafeAbsolutePath(m_ResultPath);
- if (ToLower(ResultPath.extension().string()) == ".cbo")
- {
- MemoryView ResponseView = Response.GetView();
- WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
- }
- else
- {
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
- }
+ builds_impl::WriteResultObject(MakeSafeAbsolutePath(m_ResultPath), Response);
}
}
-BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list", "List builds matching a query"), m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsListSubCmd::BuildsListSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "list", "List builds matching a query")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
Opts.add_option("",
"",
"query-path",
@@ -2900,19 +1157,16 @@ BuildsListSubCmd::BuildsListSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("list"
void
BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
+ if (!m_ResultPath.empty())
+ {
+ LogBanner();
+ }
+
+ cxxopts::Options& Opts = SubOptions();
MakeSafeAbsolutePathInPlace(m_QueryPath);
MakeSafeAbsolutePathInPlace(m_ResultPath);
- if (!m_ResultPath.empty())
- {
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
- }
std::string JsonQuery;
if (m_QueryPath.empty())
{
@@ -2949,25 +1203,14 @@ BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
}
}
+ std::string DummyBuildId;
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
-
- CreateDirectories(m_Parent.GetZenFolderPath());
- auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); });
-
- std::unique_ptr<AuthMgr> Auth;
- std::string DummyBuildId;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- DummyBuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ false,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {.RequireBucket = false}, Opts);
+ StorageInstance Storage =
+ CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved);
+ auto _ = MakeGuard([this]() { CleanZenFolder(); });
CbObject Response = Storage.BuildStorage->ListBuilds(JsonQuery);
ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
@@ -2979,28 +1222,20 @@ BuildsListSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
}
else
{
- if (ToLower(m_ResultPath.extension().string()) == ".cbo")
- {
- MemoryView ResponseView = Response.GetView();
- WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
- }
- else
- {
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
- }
+ builds_impl::WriteResultObject(m_ResultPath, Response);
}
}
-BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("list-blocks", "List blocks for a build")
-, m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsConfiguration& Config)
+: BuildsSubCmdBase(Config, "list-blocks", "List blocks for a build")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
Opts.add_option("",
"",
@@ -3016,50 +1251,35 @@ BuildsListBlocksSubCmd::BuildsListBlocksSubCmd(BuildsCommand& Parent)
void
BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
-
- MakeSafeAbsolutePathInPlace(m_ResultPath);
-
if (!m_ResultPath.empty())
{
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
+ LogBanner();
}
+ cxxopts::Options& Opts = SubOptions();
+
+ MakeSafeAbsolutePathInPlace(m_ResultPath);
+
if (m_MaxCount == 0)
{
throw OptionParseException(fmt::format("'--max-count' ('{}') is invalid", m_MaxCount), Opts.help());
}
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts);
+ StorageInstance Storage =
+ CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved);
+ auto _ = MakeGuard([this]() { CleanZenFolder(); });
- CreateDirectories(m_Parent.GetZenFolderPath());
- auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_Parent.GetZenFolderPath()); });
-
- std::unique_ptr<AuthMgr> Auth;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- m_BuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
-
- const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
+ const Oid BuildId = ParseBuildId(m_BuildId, Opts);
CbObject Response = Storage.BuildStorage->FindBlocks(BuildId, m_MaxCount);
ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
std::vector<ChunkBlockDescription> BlockDescriptions = ParseChunkBlockDescriptionList(Response);
- if (!IsQuiet)
+ if (!m_Config.Quiet)
{
ZEN_CONSOLE("Response contains {} block", BlockDescriptions.size());
}
@@ -3074,35 +1294,25 @@ BuildsListBlocksSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
}
else
{
- if (ToLower(m_ResultPath.extension().string()) == ".cbo")
- {
- MemoryView ResponseView = Response.GetView();
- WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
- }
- else
- {
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
- }
+ builds_impl::WriteResultObject(m_ResultPath, Response);
}
}
-BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("upload", "Upload a folder to build storage")
-, m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "upload", "Upload a folder to build storage")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddCacheOptions(Opts);
- Parent.AddWorkerOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
- Parent.AddExcludeFolderOption(Opts);
- Parent.AddExcludeExtensionsOption(Opts);
- Parent.AddChunkingCacheOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddCacheOptions(Opts);
+ Config.AddWorkerOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
+ Config.AddExcludeFolderOption(Opts);
+ Config.AddExcludeExtensionsOption(Opts);
+ Config.AddChunkingCacheOptions(Opts);
Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>");
Opts.add_option("",
"",
@@ -3150,7 +1360,7 @@ BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent)
cxxopts::value(m_UploadToZenCache),
"<uploadtozencache>");
- Parent.AddMultipartOptions(Opts);
+ Config.AddMultipartOptions(Opts);
Opts.add_option("",
"",
@@ -3175,50 +1385,30 @@ BuildsUploadSubCmd::BuildsUploadSubCmd(BuildsCommand& Parent)
void
BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
+ LogBanner();
+ TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false);
+ LogWorkersInfo(Workers);
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
+ cxxopts::Options& Opts = SubOptions();
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ ParsePath(m_Path, Opts);
- ZenState InstanceState;
-
- m_Parent.ParsePath(m_Path, Opts);
+ builds_impl::ZenState InstanceState;
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
- MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath);
-
- CreateDirectories(m_Parent.GetZenFolderPath());
- auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); });
-
- std::unique_ptr<AuthMgr> Auth;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- m_BuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts);
+ StorageInstance Storage =
+ CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved);
+ auto _ = MakeGuard([this]() { CleanZenFolder(); });
if (m_BuildPartName.empty() && m_ManifestPath.empty())
{
m_BuildPartName = m_Path.filename().string();
}
- const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : m_Parent.ParseBuildId(m_BuildId, Opts);
+ const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId(m_BuildId, Opts);
if (m_BuildId.empty())
{
m_BuildId = BuildId.ToString();
@@ -3227,29 +1417,31 @@ BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
Oid BuildPartId;
if (!m_BuildPartId.empty())
{
- BuildPartId = m_Parent.ParseBuildPartId(m_BuildPartId, Opts);
+ BuildPartId = ParseBuildPartId(m_BuildPartId, Opts);
}
- CbObject MetaData = m_Parent.ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts);
+ CbObject MetaData = ParseBuildMetadata(m_CreateBuild, m_BuildMetadataPath, m_BuildMetadata, Opts);
- const std::filesystem::path TempDir = ZenTempFolderPath(m_Parent.GetZenFolderPath());
+ const std::filesystem::path TempDir = ZenTempFolderPath(GetZenFolderPath());
std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
- m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
+ ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
- std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty()
+ std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty()
? CreateNullChunkingCache()
- : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u);
+ : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u);
- std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode));
+ std::unique_ptr<ProgressBase> Progress = CreateProgress();
std::vector<std::pair<Oid, std::string>> UploadedParts =
UploadFolder(ConsoleLog(),
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
BuildId,
BuildPartId,
m_BuildPartName,
@@ -3261,68 +1453,79 @@ BuildsUploadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
UploadFolderOptions{.TempDir = TempDir,
.FindBlockMaxCount = m_FindBlockMaxCount,
.BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.CreateBuild = m_CreateBuild,
.IgnoreExistingBlocks = m_Clean,
.UploadToZenCache = m_UploadToZenCache,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
.ExcludeFolders = ExcludeFolders,
.ExcludeExtensions = ExcludeExtensions});
- if (!AbortFlag)
+ if (!AbortFlag())
{
if (m_PostUploadVerify)
{
for (const auto& Part : UploadedParts)
{
- ValidateBuildPart(ConsoleLog(), *Progress, Workers, *Storage.BuildStorage, BuildId, Part.first, Part.second);
+ ValidateBuildPart(ConsoleLog(),
+ *Progress,
+ AbortFlag(),
+ PauseFlag(),
+ m_Config.Quiet,
+ m_Config.Verbose,
+ Workers,
+ *Storage.BuildStorage,
+ BuildId,
+ Part.first,
+ Part.second);
}
}
}
- if (true)
+ if (!m_Config.Quiet)
{
- if (!IsQuiet)
- {
- ZEN_CONSOLE(
- "{}:\n"
- "Read: {}\n"
- "Write: {}\n"
- "Requests: {}\n"
- "Avg Request Time: {}\n"
- "Avg I/O Time: {}",
- Storage.BuildStorageHost.Name,
- NiceBytes(StorageStats.TotalBytesRead.load()),
- NiceBytes(StorageStats.TotalBytesWritten.load()),
- StorageStats.TotalRequestCount.load(),
- StorageStats.TotalExecutionTimeUs.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0,
- StorageStats.TotalRequestCount.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0);
- }
+ const BuildStorageResolveResult::Host& Host = Storage.BuildStorageHost;
+ const BuildStorageBase::Statistics& Stats = StorageStats;
+ ZEN_CONSOLE(
+ "{}:\n"
+ "Read: {}\n"
+ "Write: {}\n"
+ "Requests: {}\n"
+ "Avg Request Time: {}\n"
+ "Avg I/O Time: {}",
+ Host.Name,
+ NiceBytes(Stats.TotalBytesRead.load()),
+ NiceBytes(Stats.TotalBytesWritten.load()),
+ Stats.TotalRequestCount.load(),
+ Stats.TotalExecutionTimeUs.load() > 0
+ ? NiceTimeSpanMs(Stats.TotalExecutionTimeUs.load() / 1000 / Stats.TotalRequestCount.load())
+ : 0,
+ Stats.TotalRequestCount.load() > 0 ? NiceTimeSpanMs(Stats.TotalRequestTimeUs.load() / 1000 / Stats.TotalRequestCount.load())
+ : 0);
}
- if (AbortFlag)
+ if (AbortFlag())
{
throw std::runtime_error("Upload aborted");
}
}
-BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("download", "Download a build to a local folder")
-, m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsConfiguration& Config)
+: BuildsSubCmdBase(Config, "download", "Download a build to a local folder")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddCacheOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
- Parent.AddWorkerOptions(Opts);
- Parent.AddWildcardOptions(Opts);
- Parent.AddAppendNewContentOptions(Opts);
- Parent.AddExcludeFolderOption(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddCacheOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
+ Config.AddWorkerOptions(Opts);
+ Config.AddWildcardOptions(Opts);
+ Config.AddAppendNewContentOptions(Opts);
+ Config.AddExcludeFolderOption(Opts);
Opts.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>");
Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
@@ -3357,9 +1560,9 @@ BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent)
"Upload data downloaded from remote host to zen cache",
cxxopts::value(m_UploadToZenCache),
"<uploadtozencache>");
- Parent.AddMultipartOptions(Opts);
+ Config.AddMultipartOptions(Opts);
- Parent.AddPartialBlockRequestOptions(Opts);
+ Config.AddPartialBlockRequestOptions(Opts);
Opts.add_option(
"",
@@ -3389,76 +1592,61 @@ BuildsDownloadSubCmd::BuildsDownloadSubCmd(BuildsCommand& Parent)
void
BuildsDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
-
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
-
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ LogBanner();
+ TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false);
+ LogWorkersInfo(Workers);
- ZenState InstanceState;
+ cxxopts::Options& Opts = SubOptions();
- m_Parent.ParsePath(m_Path, Opts);
+ ParsePath(m_Path, Opts);
std::vector<std::string> IncludeWildcards;
std::vector<std::string> ExcludeWildcards;
- m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards);
+ ParseFileFilters(IncludeWildcards, ExcludeWildcards);
- m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName);
+ builds_impl::ZenState InstanceState;
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- std::unique_ptr<AuthMgr> Auth;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- m_BuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts);
+ StorageInstance Storage = CreateBuildStorage(m_Path / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved);
+ auto _ = MakeGuard([this]() { CleanZenFolder(); });
- const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
+ const Oid BuildId = ParseBuildId(m_BuildId, Opts);
- std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts);
- std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts);
+ std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts);
+ std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts);
- EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(Opts);
+ EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts);
- if (m_Parent.m_AppendNewContent && m_Clean)
+ if (m_Config.AppendNewContent && m_Clean)
{
throw OptionParseException("'--append' conflicts with '--clean'", Opts.help());
}
std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
- m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
+ ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
- std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode));
+ std::unique_ptr<ProgressBase> Progress = CreateProgress();
DownloadFolder(
ConsoleLog(),
*Progress,
Workers,
Storage,
- StorageCacheStats,
+ AbortFlag(),
+ PauseFlag(),
+ CacheStats,
BuildId,
BuildPartIds,
BuildPartNames,
m_DownloadSpecPath,
m_Path,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
- .ZenFolderPath = m_Parent.GetZenFolderPath(),
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ DownloadOptions{.SystemRootDir = m_Config.SystemRootDir,
+ .ZenFolderPath = GetZenFolderPath(),
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = m_Clean,
.PostDownloadVerify = m_PostDownloadVerify,
@@ -3467,28 +1655,33 @@ BuildsDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
.AllowFileClone = m_AllowFileClone,
.IncludeWildcards = IncludeWildcards,
.ExcludeWildcards = ExcludeWildcards,
- .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Parent.m_BoostWorkerMemory),
+ .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_Config.BoostWorkerMemory),
.PopulateCache = m_UploadToZenCache,
- .AppendNewContent = m_Parent.m_AppendNewContent,
+ .AppendNewContent = m_Config.AppendNewContent,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles,
.ExcludeFolders = ExcludeFolders});
- if (AbortFlag)
+ if (AbortFlag())
{
throw std::runtime_error("Download aborted");
}
}
-BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "List files in a build"), m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsLsSubCmd::BuildsLsSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "ls", "List files in a build")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddCacheOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
- Parent.AddWorkerOptions(Opts);
- Parent.AddWildcardOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddCacheOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
+ Config.AddWorkerOptions(Opts);
+ Config.AddWildcardOptions(Opts);
Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
Opts.add_option("",
@@ -3526,43 +1719,31 @@ BuildsLsSubCmd::BuildsLsSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("ls", "Lis
void
BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
-
if (!m_ResultPath.empty())
{
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
+ LogBanner();
}
- ZenState InstanceState;
+ cxxopts::Options& Opts = SubOptions();
+
+ builds_impl::ZenState InstanceState;
std::vector<std::string> IncludeWildcards;
std::vector<std::string> ExcludeWildcards;
- m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards);
-
- m_Parent.ResolveZenFolderPath(m_Parent.m_StoragePath); // ls uses storage path context
+ ParseFileFilters(IncludeWildcards, ExcludeWildcards);
+ // ls uses storage path context
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- std::unique_ptr<AuthMgr> Auth;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- m_BuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts);
+ StorageInstance Storage = CreateBuildStorage(m_Config.StoragePath, {}, Opts, StorageStats, CacheStats, Auth, Resolved);
+ auto _ = MakeGuard([this]() { CleanZenFolder(); });
- const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
+ const Oid BuildId = ParseBuildId(m_BuildId, Opts);
- std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts);
- std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts);
+ std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts);
+ std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts);
std::unique_ptr<CbObjectWriter> StructuredOutput;
if (!m_ResultPath.empty())
@@ -3571,38 +1752,30 @@ BuildsLsSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
StructuredOutput = std::make_unique<CbObjectWriter>();
}
- ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get());
+ ListBuild(m_Config.Quiet, Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get());
if (StructuredOutput)
{
CbObject Response = StructuredOutput->Save();
- if (ToLower(m_ResultPath.extension().string()) == ".cbo")
- {
- MemoryView ResponseView = Response.GetView();
- WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
- }
- else
- {
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(m_ResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
- }
+ builds_impl::WriteResultObject(m_ResultPath, Response);
}
- if (AbortFlag)
+ if (AbortFlag())
{
throw std::runtime_error("List build aborted");
}
}
-BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff", "Diff two local folders"), m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "diff", "Diff two local folders")
{
- auto& Opts = SubOptions();
- Parent.AddOutputOptions(Opts);
- Parent.AddWorkerOptions(Opts);
- Parent.AddExcludeFolderOption(Opts);
- Parent.AddExcludeExtensionsOption(Opts);
- Parent.AddChunkingCacheOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddOutputOptions(Opts);
+ Config.AddWorkerOptions(Opts);
+ Config.AddExcludeFolderOption(Opts);
+ Config.AddExcludeExtensionsOption(Opts);
+ Config.AddChunkingCacheOptions(Opts);
Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
Opts.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>");
Opts.add_option("",
@@ -3618,38 +1791,28 @@ BuildsDiffSubCmd::BuildsDiffSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("diff"
void
BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
+ LogBanner();
+ TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false);
+ LogWorkersInfo(Workers);
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
+ cxxopts::Options& Opts = SubOptions();
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
-
- m_Parent.ParsePath(m_Path, Opts);
+ ParsePath(m_Path, Opts);
if (m_DiffPath.empty())
{
throw OptionParseException("'--compare-path' is required", Opts.help());
}
MakeSafeAbsolutePathInPlace(m_DiffPath);
- MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath);
-
std::vector<std::string> ExcludeFolders = DefaultExcludeFolders;
std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
- m_Parent.ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
+ ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
StandardChunkingControllerSettings ChunkingSettings;
std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(ChunkingSettings);
- std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty()
+ std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty()
? CreateNullChunkingCache()
- : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u);
+ : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u);
if (m_OnlyChunked)
{
@@ -3661,24 +1824,37 @@ BuildsDiffSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
ChunkingSettings.SplitAndCompressExtensions.end());
}
- DiffFolders(Workers, m_Path, m_DiffPath, *ChunkController, *ChunkCache, ExcludeFolders, ExcludeExtensions);
- if (AbortFlag)
+ std::unique_ptr<ProgressBase> Progress = CreateProgress();
+
+ DiffFolders(*Progress,
+ AbortFlag(),
+ PauseFlag(),
+ m_Config.Quiet,
+ Workers,
+ m_Path,
+ m_DiffPath,
+ *ChunkController,
+ *ChunkCache,
+ ExcludeFolders,
+ ExcludeExtensions);
+ if (AbortFlag())
{
throw std::runtime_error("Diff folders aborted");
}
}
-BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("fetch-blob", "Fetch and validate a specific blob")
-, m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsConfiguration& Config)
+: BuildsSubCmdBase(Config, "fetch-blob", "Fetch and validate a specific blob")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddCacheOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddCacheOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
Opts.add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), "<blob-hash>");
Opts.parse_positional({"build-id", "blob-hash"});
@@ -3688,68 +1864,47 @@ BuildsFetchBlobSubCmd::BuildsFetchBlobSubCmd(BuildsCommand& Parent)
void
BuildsFetchBlobSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
+ LogBanner();
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
-
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ cxxopts::Options& Opts = SubOptions();
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts);
+ StorageInstance Storage =
+ CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved);
+ auto _ = MakeGuard([this]() { CleanZenFolder(); });
- m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
-
- CreateDirectories(m_Parent.GetZenFolderPath());
- auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); });
-
- std::unique_ptr<AuthMgr> Auth;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- m_BuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
-
- IoHash BlobHash = m_Parent.ParseBlobHash(m_BlobHash, Opts);
-
- const Oid BuildId = Oid::FromHexString(m_BuildId);
+ IoHash BlobHash = ParseBlobHash(m_BlobHash, Opts);
+ const Oid BuildId = ParseBuildId(m_BuildId, Opts);
uint64_t CompressedSize;
uint64_t DecompressedSize;
- ValidateBlob(AbortFlag, *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize);
- if (AbortFlag)
+ ValidateBlob(AbortFlag(), *Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize);
+ if (AbortFlag())
{
throw std::runtime_error("Fetch blob aborted");
}
- if (!IsQuiet)
+ if (!m_Config.Quiet)
{
ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", BlobHash, CompressedSize, DecompressedSize);
}
}
-BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("prime-cache", "Prime the zen cache with build data")
-, m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsConfiguration& Config)
+: BuildsSubCmdBase(Config, "prime-cache", "Prime the zen cache with build data")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddCacheOptions(Opts);
- Parent.AddWorkerOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddCacheOptions(Opts);
+ Config.AddWorkerOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
Opts.add_option("",
"",
@@ -3777,47 +1932,33 @@ BuildsPrimeCacheSubCmd::BuildsPrimeCacheSubCmd(BuildsCommand& Parent)
void
BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
+ LogBanner();
+ TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false);
+ LogWorkersInfo(Workers);
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
-
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ cxxopts::Options& Opts = SubOptions();
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
-
- CreateDirectories(m_Parent.GetZenFolderPath());
- auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); });
-
- std::unique_ptr<AuthMgr> Auth;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- m_BuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ true,
- Auth,
- Opts);
-
- const Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
-
- std::vector<Oid> BuildPartIds = m_Parent.ParseBuildPartIds(m_BuildPartIds, Opts);
- std::vector<std::string> BuildPartNames = m_Parent.ParseBuildPartNames(m_BuildPartNames, Opts);
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts);
+ StorageInstance Storage = CreateBuildStorage(std::filesystem::current_path() / ZenFolderName,
+ {.BoostCacheBackgroundWorkers = true},
+ Opts,
+ StorageStats,
+ CacheStats,
+ Auth,
+ Resolved);
+ auto _ = MakeGuard([this]() { CleanZenFolder(); });
+
+ const Oid BuildId = ParseBuildId(m_BuildId, Opts);
+
+ std::vector<Oid> BuildPartIds = ParseBuildPartIds(m_BuildPartIds, Opts);
+ std::vector<std::string> BuildPartNames = ParseBuildPartNames(m_BuildPartNames, Opts);
std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
- CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId);
+ CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId, m_Config.Quiet);
std::vector<std::pair<Oid, std::string>> AllBuildParts =
ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
@@ -3829,101 +1970,101 @@ BuildsPrimeCacheSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
AllBuildPartIds.push_back(BuildPart.first);
}
- ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache");
-
- std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode));
+ std::unique_ptr<ProgressBase> Progress = CreateProgress();
+ Progress->SetLogOperationName("Prime Cache");
BuildsOperationPrimeCache PrimeOp(ConsoleLog(),
*Progress,
Storage,
- AbortFlag,
- PauseFlag,
+ AbortFlag(),
+ PauseFlag(),
Workers.GetNetworkPool(),
BuildId,
AllBuildPartIds,
- BuildsOperationPrimeCache::Options{.IsQuiet = IsQuiet,
- .IsVerbose = IsVerbose,
- .ZenFolderPath = m_Parent.GetZenFolderPath(),
+ BuildsOperationPrimeCache::Options{.IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .ZenFolderPath = GetZenFolderPath(),
.LargeAttachmentSize = PreferredMultipartChunkSize * 4u,
.PreferredMultipartChunkSize = PreferredMultipartChunkSize,
.ForceUpload = m_Force},
- StorageCacheStats);
+ CacheStats);
PrimeOp.Execute();
- if (!IsQuiet)
+ if (!m_Config.Quiet && Storage.CacheStorage)
{
- if (Storage.CacheStorage)
- {
- ZEN_CONSOLE("Uploaded {} ({}) blobs to {}",
- StorageCacheStats.PutBlobCount.load(),
- NiceBytes(StorageCacheStats.PutBlobByteCount),
- Storage.CacheHost.Name);
- }
+ ZEN_CONSOLE("Uploaded {} ({}) blobs to {}",
+ CacheStats.PutBlobCount.load(),
+ NiceBytes(CacheStats.PutBlobByteCount),
+ Storage.CacheHost.Name);
}
}
-BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("pause", "Pause a running zen process"), m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+namespace {
+ void AddProcessIdOption(cxxopts::Options& Opts, int& ZenProcessId)
+ {
+ Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(ZenProcessId), "<pid>");
+ Opts.parse_positional({"process-id"});
+ Opts.positional_help("process-id");
+ }
+} // namespace
+
+BuildsPauseSubCmd::BuildsPauseSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "pause", "Pause a running zen process")
{
- auto& Opts = SubOptions();
- Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>");
- Opts.parse_positional({"process-id"});
- Opts.positional_help("process-id");
+ AddProcessIdOption(SubOptions(), m_ZenProcessId);
}
void
BuildsPauseSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- using namespace builds_impl;
- m_Parent.ParseZenProcessId(m_ZenProcessId);
- ZenState RunningState(m_ZenProcessId);
- RunningState.StateData().Pause.store(true);
+ ParseZenProcessId(m_ZenProcessId);
+ builds_impl::ZenState RunningState(m_ZenProcessId);
+ RunningState.StateData().Pause.store(1);
}
-BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("resume", "Resume a paused zen process"), m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsResumeSubCmd::BuildsResumeSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "resume", "Resume a paused zen process")
{
- auto& Opts = SubOptions();
- Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>");
- Opts.parse_positional({"process-id"});
- Opts.positional_help("process-id");
+ AddProcessIdOption(SubOptions(), m_ZenProcessId);
}
void
BuildsResumeSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- using namespace builds_impl;
- m_Parent.ParseZenProcessId(m_ZenProcessId);
- ZenState RunningState(m_ZenProcessId);
- RunningState.StateData().Pause.store(false);
+ ParseZenProcessId(m_ZenProcessId);
+ builds_impl::ZenState RunningState(m_ZenProcessId);
+ RunningState.StateData().Pause.store(0);
}
-BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("abort", "Abort a running zen process"), m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsAbortSubCmd::BuildsAbortSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "abort", "Abort a running zen process")
{
- auto& Opts = SubOptions();
- Opts.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), "<pid>");
- Opts.parse_positional({"process-id"});
- Opts.positional_help("process-id");
+ AddProcessIdOption(SubOptions(), m_ZenProcessId);
}
void
BuildsAbortSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- using namespace builds_impl;
- m_Parent.ParseZenProcessId(m_ZenProcessId);
- ZenState RunningState(m_ZenProcessId);
- RunningState.StateData().Abort.store(true);
+ ParseZenProcessId(m_ZenProcessId);
+ builds_impl::ZenState RunningState(m_ZenProcessId);
+ RunningState.StateData().Abort.store(1);
}
-BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("validate-part", "Validate a build part")
-, m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsConfiguration& Config)
+: BuildsSubCmdBase(Config, "validate-part", "Validate a build part")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddWorkerOptions(Opts);
- Parent.AddZenFolderOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddWorkerOptions(Opts);
+ Config.AddZenFolderOptions(Opts);
Opts.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>");
Opts.add_option("",
"",
@@ -3944,42 +2085,23 @@ BuildsValidatePartSubCmd::BuildsValidatePartSubCmd(BuildsCommand& Parent)
void
BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
+ LogBanner();
+ TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false);
+ LogWorkersInfo(Workers);
- if (!IsQuiet)
- {
- ZenCmdBase::LogExecutableVersionAndPid();
- }
-
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
-
- ZenState InstanceState;
+ cxxopts::Options& Opts = SubOptions();
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(m_BuildId, {}, Opts);
+ StorageInstance Storage =
+ CreateBuildStorage(std::filesystem::current_path() / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved);
+ auto _ = MakeGuard([this]() { CleanZenFolder(); });
- m_Parent.ResolveZenFolderPath(std::filesystem::current_path() / ZenFolderName);
+ builds_impl::ZenState InstanceState;
- CreateDirectories(m_Parent.GetZenFolderPath());
- auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.GetZenFolderPath()); });
-
- std::unique_ptr<AuthMgr> Auth;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- m_BuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
-
- Oid BuildId = m_Parent.ParseBuildId(m_BuildId, Opts);
+ Oid BuildId = ParseBuildId(m_BuildId, Opts);
if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
{
@@ -3988,33 +2110,45 @@ BuildsValidatePartSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
Opts.help());
}
- const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : m_Parent.ParseBuildPartId(m_BuildPartId, Opts);
+ const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(m_BuildPartId, Opts);
- std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode));
+ std::unique_ptr<ProgressBase> Progress = CreateProgress();
- ValidateBuildPart(ConsoleLog(), *Progress, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+ ValidateBuildPart(ConsoleLog(),
+ *Progress,
+ AbortFlag(),
+ PauseFlag(),
+ m_Config.Quiet,
+ m_Config.Verbose,
+ Workers,
+ *Storage.BuildStorage,
+ BuildId,
+ BuildPartId,
+ m_BuildPartName);
- if (AbortFlag)
+ if (AbortFlag())
{
throw std::runtime_error("Validate build part failed");
}
}
-BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test", "Run an upload/download test cycle"), m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsTestSubCmd::BuildsTestSubCmd(BuildsConfiguration& Config) : BuildsSubCmdBase(Config, "test", "Run an upload/download test cycle")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddCacheOptions(Opts);
- Parent.AddWorkerOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddCacheOptions(Opts);
+ Config.AddWorkerOptions(Opts);
Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
- Parent.AddMultipartOptions(Opts);
- Parent.AddPartialBlockRequestOptions(Opts);
- Parent.AddWildcardOptions(Opts);
- Parent.AddAppendNewContentOptions(Opts);
- Parent.AddChunkingCacheOptions(Opts);
+ Config.AddMultipartOptions(Opts);
+ Config.AddPartialBlockRequestOptions(Opts);
+ Config.AddWildcardOptions(Opts);
+ Config.AddAppendNewContentOptions(Opts);
+ Config.AddChunkingCacheOptions(Opts);
Opts.add_option("",
"",
"enable-scavenge",
@@ -4034,38 +2168,35 @@ BuildsTestSubCmd::BuildsTestSubCmd(BuildsCommand& Parent) : ZenSubCmdBase("test"
void
BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
+ TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false);
+ LogWorkersInfo(Workers);
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
+ cxxopts::Options& Opts = SubOptions();
- m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
- CreateDirectories(m_Parent.m_SystemRootDir);
- CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true);
- auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); });
+ m_TestSystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
+ CreateDirectories(m_TestSystemRootDir);
+ CleanDirectory(m_TestSystemRootDir, /*ForceRemoveReadOnlyFiles*/ true);
+ auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_TestSystemRootDir); });
- m_Parent.ParsePath(m_Path, Opts);
+ ParsePath(m_Path, Opts);
- if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty())
+ const bool CreatedTempStorage = m_Config.OverrideHost.empty() && m_Config.StoragePath.empty();
+ if (CreatedTempStorage)
{
- m_Parent.m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_Parent.m_StoragePath);
- CreateDirectories(m_Parent.m_StoragePath);
- m_Parent.m_StoragePath = m_Parent.m_StoragePath.generic_string();
+ m_TestStoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), m_TestStoragePath);
+ CreateDirectories(m_TestStoragePath);
+ m_TestStoragePath = m_TestStoragePath.generic_string();
}
- auto StorageGuard = MakeGuard([this]() {
- if (m_Parent.m_OverrideHost.empty() && m_Parent.m_StoragePath.empty())
+ auto StorageGuard = MakeGuard([this, CreatedTempStorage]() {
+ if (CreatedTempStorage)
{
- DeleteDirectories(m_Parent.m_StoragePath);
+ DeleteDirectories(m_TestStoragePath);
}
});
- EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(Opts);
+ EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts);
BuildStorageBase::Statistics StorageStats;
BuildStorageCache::Statistics StorageCacheStats;
@@ -4076,30 +2207,20 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2");
const std::filesystem::path DownloadPath3 = m_Path.parent_path() / (m_BuildPartName + "_test3");
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath);
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2);
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3);
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath);
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath2);
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath3);
- auto DownloadGuard = MakeGuard([&Workers, DownloadPath, DownloadPath2, DownloadPath3]() {
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath);
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath2);
- CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath3);
+ auto DownloadGuard = MakeGuard([this, &Workers, DownloadPath, DownloadPath2, DownloadPath3]() {
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath);
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath2);
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath3);
});
- m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName);
- MakeSafeAbsolutePathInPlace(m_Parent.m_ChunkingCachePath);
-
std::unique_ptr<AuthMgr> Auth;
std::string TestBuildId;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- TestBuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
+ const ResolvedStorage Resolved = ParseStorageOptions(TestBuildId, {}, Opts, m_TestSystemRootDir, m_TestStoragePath);
+ StorageInstance Storage = CreateBuildStorage(m_Path / ZenFolderName, {}, Opts, StorageStats, StorageCacheStats, Auth, Resolved);
m_BuildId = Oid::NewOid().ToString();
m_BuildPartId = Oid::NewOid().ToString();
@@ -4129,19 +2250,21 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView());
}
- const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path);
+ const std::filesystem::path UploadTempDir = builds_impl::UploadTempDirectory(m_Path);
std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
- std::unique_ptr<ChunkingCache> ChunkCache = m_Parent.m_ChunkingCachePath.empty()
+ std::unique_ptr<ChunkingCache> ChunkCache = m_Config.ChunkingCachePath.empty()
? CreateNullChunkingCache()
- : CreateDiskChunkingCache(m_Parent.m_ChunkingCachePath, *ChunkController, 256u * 1024u);
+ : CreateDiskChunkingCache(m_Config.ChunkingCachePath, *ChunkController, 256u * 1024u);
- std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode));
+ std::unique_ptr<ProgressBase> Progress = CreateProgress();
UploadFolder(ConsoleLog(),
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
BuildId,
BuildPartId,
m_BuildPartName,
@@ -4153,12 +2276,14 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
UploadFolderOptions{.TempDir = UploadTempDir,
.FindBlockMaxCount = m_FindBlockMaxCount,
.BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.CreateBuild = true,
.IgnoreExistingBlocks = false,
- .UploadToZenCache = m_UploadToZenCache});
+ .UploadToZenCache = m_UploadToZenCache,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose});
- if (AbortFlag)
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Upload build)");
}
@@ -4170,6 +2295,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
Oid::NewOid(),
Oid::NewOid(),
m_BuildPartName,
@@ -4181,42 +2308,56 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
UploadFolderOptions{.TempDir = UploadTempDir,
.FindBlockMaxCount = m_FindBlockMaxCount,
.BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.CreateBuild = true,
.IgnoreExistingBlocks = false,
- .UploadToZenCache = m_UploadToZenCache});
+ .UploadToZenCache = m_UploadToZenCache,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose});
- if (AbortFlag)
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Upload again, chunking is cached)");
}
}
- ValidateBuildPart(ConsoleLog(), *Progress, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+ ValidateBuildPart(ConsoleLog(),
+ *Progress,
+ AbortFlag(),
+ PauseFlag(),
+ m_Config.Quiet,
+ m_Config.Verbose,
+ Workers,
+ *Storage.BuildStorage,
+ BuildId,
+ BuildPartId,
+ m_BuildPartName);
- if (!m_Parent.m_IncludeWildcard.empty() || !m_Parent.m_ExcludeWildcard.empty())
+ if (!m_Config.IncludeWildcard.empty() || !m_Config.ExcludeWildcard.empty())
{
- auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), DownloadPath); });
+ auto WcGuard = MakeGuard([&]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag(), PauseFlag(), DownloadPath); });
ZEN_CONSOLE("\nDownload Filtered Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
std::vector<std::string> IncludeWildcards;
std::vector<std::string> ExcludeWildcards;
- m_Parent.ParseFileFilters(IncludeWildcards, ExcludeWildcards);
+ ParseFileFilters(IncludeWildcards, ExcludeWildcards);
DownloadFolder(ConsoleLog(),
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = true,
.PostDownloadVerify = true,
@@ -4225,8 +2366,11 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
.AllowFileClone = m_AllowFileClone,
.IncludeWildcards = IncludeWildcards,
.ExcludeWildcards = ExcludeWildcards,
- .AppendNewContent = false});
- if (AbortFlag)
+ .AppendNewContent = false,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download build)");
}
@@ -4240,15 +2384,17 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = true,
.PostDownloadVerify = true,
@@ -4257,8 +2403,11 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
.AllowFileClone = m_AllowFileClone,
.IncludeWildcards = ExcludeWildcards,
.ExcludeWildcards = IncludeWildcards,
- .AppendNewContent = true});
- if (AbortFlag)
+ .AppendNewContent = true,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download build)");
}
@@ -4268,15 +2417,17 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
@@ -4285,8 +2436,11 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
.AllowFileClone = m_AllowFileClone,
.IncludeWildcards = {},
.ExcludeWildcards = {},
- .AppendNewContent = false});
- if (AbortFlag)
+ .AppendNewContent = false,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download build)");
}
@@ -4297,6 +2451,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
@@ -4304,16 +2460,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = true,
.PostDownloadVerify = true,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = false,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download build)");
}
@@ -4323,6 +2482,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
@@ -4330,16 +2491,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Re-download identical target)");
}
@@ -4371,7 +2535,7 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
return true;
};
- ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+ ParallelWork Work(AbortFlag(), PauseFlag(), WorkerThreadPool::EMode::EnableBacklog);
uint32_t Randomizer = 0;
auto FileSizeIt = DownloadContent.FileSizes.begin();
@@ -4389,8 +2553,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
Work.ScheduleWork(
Workers.GetIOWorkerPool(),
- [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) {
- if (!AbortFlag)
+ [this, SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) {
+ if (!AbortFlag())
{
bool WasReadOnly = SetFileReadOnly(FilePath, false);
{
@@ -4431,7 +2595,7 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
ZEN_UNUSED(IsAborted, IsPaused);
ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork);
});
- ZEN_ASSERT(!AbortFlag.load());
+ ZEN_ASSERT(!AbortFlag().load());
ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
};
@@ -4441,6 +2605,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
@@ -4448,16 +2614,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Re-download scrambled target)");
}
@@ -4478,6 +2647,8 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
BuildId2,
BuildPartId2,
m_BuildPartName,
@@ -4489,23 +2660,37 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
UploadFolderOptions{.TempDir = UploadTempDir,
.FindBlockMaxCount = m_FindBlockMaxCount,
.BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.CreateBuild = true,
.IgnoreExistingBlocks = false,
- .UploadToZenCache = m_UploadToZenCache});
+ .UploadToZenCache = m_UploadToZenCache,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose});
- if (AbortFlag)
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Upload scrambled)");
}
- ValidateBuildPart(ConsoleLog(), *Progress, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+ ValidateBuildPart(ConsoleLog(),
+ *Progress,
+ AbortFlag(),
+ PauseFlag(),
+ m_Config.Quiet,
+ m_Config.Verbose,
+ Workers,
+ *Storage.BuildStorage,
+ BuildId,
+ BuildPartId,
+ m_BuildPartName);
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
DownloadFolder(ConsoleLog(),
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
@@ -4513,16 +2698,19 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download original)");
}
@@ -4532,22 +2720,27 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId2,
{BuildPartId2},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download scrambled)");
}
@@ -4557,22 +2750,27 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId2,
{BuildPartId2},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Re-download scrambled)");
}
@@ -4582,22 +2780,27 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath2,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath2 / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download original)");
}
@@ -4607,38 +2810,44 @@ BuildsTestSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
*Progress,
Workers,
Storage,
+ AbortFlag(),
+ PauseFlag(),
StorageCacheStats,
BuildId,
{BuildPartId},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
DownloadPath3,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
.ZenFolderPath = DownloadPath3 / ZenFolderName,
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = false,
.PostDownloadVerify = true,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = true,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Test aborted. (Download original)");
}
}
-BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Parent)
-: ZenSubCmdBase("multi-test-download", "Download multiple builds sequentially as a test")
-, m_Parent(Parent)
+//////////////////////////////////////////////////////////////////////////
+
+BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsConfiguration& Config)
+: BuildsSubCmdBase(Config, "multi-test-download", "Download multiple builds sequentially as a test")
{
- auto& Opts = SubOptions();
- Parent.AddSystemOptions(Opts);
- Parent.AddCloudOptions(Opts);
- Parent.AddFileOptions(Opts);
- Parent.AddOutputOptions(Opts);
- Parent.AddCacheOptions(Opts);
- Parent.AddWorkerOptions(Opts);
+ cxxopts::Options& Opts = SubOptions();
+ Config.AddSystemOptions(Opts);
+ Config.AddCloudOptions(Opts);
+ Config.AddFileOptions(Opts);
+ Config.AddOutputOptions(Opts);
+ Config.AddCacheOptions(Opts);
+ Config.AddWorkerOptions(Opts);
Opts.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>");
Opts.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), "<ids>");
Opts.add_option("",
@@ -4660,42 +2869,30 @@ BuildsMultiTestDownloadSubCmd::BuildsMultiTestDownloadSubCmd(BuildsCommand& Pare
void
BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
{
- auto& Opts = SubOptions();
- using namespace builds_impl;
+ LogBanner();
+ TransferThreadWorkers Workers(m_Config.BoostWorkerCount, false);
+ LogWorkersInfo(Workers);
- TransferThreadWorkers Workers(m_Parent.m_BoostWorkerCount, SingleThreaded);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("{}", Workers.GetWorkersInfo());
- }
-
- m_Parent.m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
- CreateDirectories(m_Parent.m_SystemRootDir);
- CleanDirectory(m_Parent.m_SystemRootDir, /*ForceRemoveReadOnlyFiles*/ true);
- auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_Parent.m_SystemRootDir); });
-
- m_Parent.ParsePath(m_Path, Opts);
+ cxxopts::Options& Opts = SubOptions();
- m_Parent.ResolveZenFolderPath(m_Path / ZenFolderName);
+ m_TestSystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
+ CreateDirectories(m_TestSystemRootDir);
+ CleanDirectory(m_TestSystemRootDir, /*ForceRemoveReadOnlyFiles*/ true);
+ auto SystemGuard = MakeGuard([this]() { DeleteDirectories(m_TestSystemRootDir); });
- EPartialBlockRequestMode PartialBlockRequestMode = m_Parent.ParseAllowPartialBlockRequests(Opts);
+ ParsePath(m_Path, Opts);
+ std::string DummyBuildId;
BuildStorageBase::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ BuildStorageCache::Statistics CacheStats;
+ std::unique_ptr<AuthMgr> Auth;
+ const ResolvedStorage Resolved = ParseStorageOptions(DummyBuildId, {}, Opts, m_TestSystemRootDir);
+ StorageInstance Storage = CreateBuildStorage(m_Path / ZenFolderName, {}, Opts, StorageStats, CacheStats, Auth, Resolved);
+ auto _ = MakeGuard([this]() { CleanZenFolder(); });
- std::unique_ptr<AuthMgr> Auth;
- std::string DummyBuildId;
- StorageInstance Storage = m_Parent.CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_Parent.GetZenFolderPath()),
- DummyBuildId,
- /*RequireNamespace*/ true,
- /*RequireBucket*/ true,
- /*BoostCacheBackgroundWorkerPool*/ false,
- Auth,
- Opts);
-
- std::unique_ptr<ProgressBase> Progress(CreateConsoleProgress(ProgressMode));
+ EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(Opts);
+
+ std::unique_ptr<ProgressBase> Progress = CreateProgress();
Stopwatch Timer;
for (const std::string& BuildIdString : m_BuildIds)
@@ -4709,31 +2906,36 @@ BuildsMultiTestDownloadSubCmd::Run(const ZenCliOptions& /*GlobalOptions*/)
*Progress,
Workers,
Storage,
- StorageCacheStats,
+ AbortFlag(),
+ PauseFlag(),
+ CacheStats,
BuildId,
/*BuildPartIds,*/ {},
/*BuildPartNames*/ {},
/*ManifestPath*/ {},
m_Path,
- DownloadOptions{.SystemRootDir = m_Parent.m_SystemRootDir,
- .ZenFolderPath = m_Parent.GetZenFolderPath(),
- .AllowMultiparts = m_Parent.m_AllowMultiparts,
+ DownloadOptions{.SystemRootDir = m_TestSystemRootDir,
+ .ZenFolderPath = GetZenFolderPath(),
+ .AllowMultiparts = m_Config.AllowMultiparts,
.PartialBlockRequestMode = PartialBlockRequestMode,
.CleanTargetFolder = BuildIdString == m_BuildIds.front(),
.PostDownloadVerify = true,
.EnableOtherDownloadsScavenging = m_EnableScavenging,
.EnableTargetFolderScavenging = false,
- .AllowFileClone = m_AllowFileClone});
- if (AbortFlag)
+ .AllowFileClone = m_AllowFileClone,
+ .IsQuiet = m_Config.Quiet,
+ .IsVerbose = m_Config.Verbose,
+ .UseSparseFiles = m_Config.UseSparseFiles});
+ if (AbortFlag())
{
throw std::runtime_error("Multitest aborted");
}
- if (!IsQuiet)
+ if (!m_Config.Quiet)
{
ZEN_CONSOLE("\n");
}
}
- if (!IsQuiet)
+ if (!m_Config.Quiet)
{
ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
}
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-07 11:30:38 +0000