aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2025-09-17 15:42:26 +0200
committerGitHub Enterprise <[email protected]>2025-09-17 15:42:26 +0200
commit08d7ab0fa1e8deff6857fdba0a14fea48f717b5a (patch)
treebde758f0380b6f1a86811e970064ba53429e84bd
parentrpmalloc fixes (#499) (diff)
downloadzen-08d7ab0fa1e8deff6857fdba0a14fea48f717b5a.tar.xz
zen-08d7ab0fa1e8deff6857fdba0a14fea48f717b5a.zip
add builds download force option (#498)
* add --force option to builds download command
-rw-r--r--CHANGELOG.md1
-rw-r--r--src/zen/cmds/builds_cmd.cpp566
-rw-r--r--src/zen/cmds/builds_cmd.h1
3 files changed, 298 insertions, 270 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 39786b3b5..b398efe0e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
##
+- Feature: Added `--force` option to `zen builds download` command that forces download of all content ignoring any local state
- Improvement: Change BadAlloc exceptions in GC to warnings
- Improvement: Add explict ASSERT exception catch in http plugin request processing
- Improvement: Make exceptions handled in http request processing to warnings
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 1e097edb7..179f34193 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -5758,21 +5758,27 @@ namespace {
return Result;
}
- void UpdateFolder(const std::filesystem::path& SystemRootDir,
- StorageInstance& Storage,
+ struct UpdateOptions
+ {
+ std::filesystem::path SystemRootDir;
+ std::filesystem::path ZenFolderPath;
+ std::uint64_t LargeAttachmentSize = DefaultPreferredMultipartChunkSize * 4u;
+ std::uint64_t PreferredMultipartChunkSize = DefaultPreferredMultipartChunkSize;
+ bool AllowPartialBlockRequests = true;
+ bool WipeTargetFolder = false;
+ bool PrimeCacheOnly = false;
+ bool EnableOtherDownloadsScavenging = true;
+ bool EnableTargetFolderScavenging = true;
+ };
+
+ void UpdateFolder(StorageInstance& Storage,
const Oid& BuildId,
const std::filesystem::path& Path,
- const std::filesystem::path& ZenFolderPath,
- const std::uint64_t LargeAttachmentSize,
- const std::uint64_t PreferredMultipartChunkSize,
const ChunkedFolderContent& LocalContent,
const ChunkedFolderContent& RemoteContent,
const std::vector<ChunkBlockDescription>& BlockDescriptions,
const std::vector<IoHash>& LooseChunkHashes,
- bool AllowPartialBlockRequests,
- bool WipeTargetFolder,
- bool PrimeCacheOnly,
- bool EnableScavenging,
+ const UpdateOptions& Options,
FolderContent& OutLocalFolderState,
DiskStatistics& DiskStats,
CacheMappingStatistics& CacheMappingStats,
@@ -5782,7 +5788,7 @@ namespace {
{
ZEN_TRACE_CPU("UpdateFolder");
- ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests)));
+ ZEN_ASSERT((!Options.PrimeCacheOnly) || (Options.PrimeCacheOnly && (!Options.AllowPartialBlockRequests)));
Stopwatch IndexTimer;
@@ -5795,7 +5801,7 @@ namespace {
ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
}
- const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(ZenFolderPath);
+ const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(Options.ZenFolderPath);
Stopwatch CacheMappingTimer;
@@ -5805,7 +5811,7 @@ namespace {
tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedChunkHashesFound;
tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedSequenceHashesFound;
- if (!PrimeCacheOnly)
+ if (!Options.PrimeCacheOnly)
{
ZEN_TRACE_CPU("UpdateFolder_CheckChunkCache");
@@ -5817,40 +5823,43 @@ namespace {
CacheDirContent);
for (size_t Index = 0; Index < CacheDirContent.Files.size(); Index++)
{
- IoHash FileHash;
- if (IoHash::TryParse(CacheDirContent.Files[Index].filename().string(), FileHash))
+ if (Options.EnableTargetFolderScavenging)
{
- if (auto ChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(FileHash);
- ChunkIt != RemoteLookup.ChunkHashToChunkIndex.end())
+ IoHash FileHash;
+ if (IoHash::TryParse(CacheDirContent.Files[Index].filename().string(), FileHash))
{
- const uint32_t ChunkIndex = ChunkIt->second;
- const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
- if (ChunkSize == CacheDirContent.FileSizes[Index])
+ if (auto ChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(FileHash);
+ ChunkIt != RemoteLookup.ChunkHashToChunkIndex.end())
{
- CachedChunkHashesFound.insert({FileHash, ChunkIndex});
- CacheMappingStats.CacheChunkCount++;
- CacheMappingStats.CacheChunkByteCount += ChunkSize;
- continue;
+ const uint32_t ChunkIndex = ChunkIt->second;
+ const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ if (ChunkSize == CacheDirContent.FileSizes[Index])
+ {
+ CachedChunkHashesFound.insert({FileHash, ChunkIndex});
+ CacheMappingStats.CacheChunkCount++;
+ CacheMappingStats.CacheChunkByteCount += ChunkSize;
+ continue;
+ }
}
- }
- else if (auto SequenceIt = RemoteLookup.RawHashToSequenceIndex.find(FileHash);
- SequenceIt != RemoteLookup.RawHashToSequenceIndex.end())
- {
- const uint32_t SequenceIndex = SequenceIt->second;
- const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
- const uint64_t SequenceSize = RemoteContent.RawSizes[PathIndex];
- if (SequenceSize == CacheDirContent.FileSizes[Index])
+ else if (auto SequenceIt = RemoteLookup.RawHashToSequenceIndex.find(FileHash);
+ SequenceIt != RemoteLookup.RawHashToSequenceIndex.end())
{
- CachedSequenceHashesFound.insert({FileHash, SequenceIndex});
- CacheMappingStats.CacheSequenceHashesCount++;
- CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize;
+ const uint32_t SequenceIndex = SequenceIt->second;
+ const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
+ const uint64_t SequenceSize = RemoteContent.RawSizes[PathIndex];
+ if (SequenceSize == CacheDirContent.FileSizes[Index])
+ {
+ CachedSequenceHashesFound.insert({FileHash, SequenceIndex});
+ CacheMappingStats.CacheSequenceHashesCount++;
+ CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize;
- const std::filesystem::path CacheFilePath =
- GetFinalChunkedSequenceFileName(CacheFolderPath,
- RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
- ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
+ const std::filesystem::path CacheFilePath =
+ GetFinalChunkedSequenceFileName(CacheFolderPath,
+ RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
+ ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
- continue;
+ continue;
+ }
}
}
}
@@ -5860,7 +5869,7 @@ namespace {
}
tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedBlocksFound;
- if (!PrimeCacheOnly)
+ if (!Options.PrimeCacheOnly)
{
ZEN_TRACE_CPU("UpdateFolder_CheckBlockCache");
@@ -5875,31 +5884,34 @@ namespace {
}
DirectoryContent BlockDirContent;
- GetDirectoryContent(ZenTempBlockFolderPath(ZenFolderPath),
+ GetDirectoryContent(ZenTempBlockFolderPath(Options.ZenFolderPath),
DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes,
BlockDirContent);
CachedBlocksFound.reserve(BlockDirContent.Files.size());
for (size_t Index = 0; Index < BlockDirContent.Files.size(); Index++)
{
- IoHash FileHash;
- if (IoHash::TryParse(BlockDirContent.Files[Index].filename().string(), FileHash))
+ if (Options.EnableTargetFolderScavenging)
{
- if (auto BlockIt = AllBlockSizes.find(FileHash); BlockIt != AllBlockSizes.end())
+ IoHash FileHash;
+ if (IoHash::TryParse(BlockDirContent.Files[Index].filename().string(), FileHash))
{
- const uint32_t BlockIndex = BlockIt->second;
- const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
- uint64_t BlockSize = CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize;
- for (uint64_t ChunkSize : BlockDescription.ChunkCompressedLengths)
+ if (auto BlockIt = AllBlockSizes.find(FileHash); BlockIt != AllBlockSizes.end())
{
- BlockSize += ChunkSize;
- }
+ const uint32_t BlockIndex = BlockIt->second;
+ const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+ uint64_t BlockSize = CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize;
+ for (uint64_t ChunkSize : BlockDescription.ChunkCompressedLengths)
+ {
+ BlockSize += ChunkSize;
+ }
- if (BlockSize == BlockDirContent.FileSizes[Index])
- {
- CachedBlocksFound.insert({FileHash, BlockIndex});
- CacheMappingStats.CacheBlockCount++;
- CacheMappingStats.CacheBlocksByteCount += BlockSize;
- continue;
+ if (BlockSize == BlockDirContent.FileSizes[Index])
+ {
+ CachedBlocksFound.insert({FileHash, BlockIndex});
+ CacheMappingStats.CacheBlockCount++;
+ CacheMappingStats.CacheBlocksByteCount += BlockSize;
+ continue;
+ }
}
}
}
@@ -5912,7 +5924,7 @@ namespace {
std::vector<uint32_t> LocalPathIndexesMatchingSequenceIndexes;
tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> SequenceIndexesLeftToFindToRemoteIndex;
- if (!PrimeCacheOnly)
+ if (!Options.PrimeCacheOnly && Options.EnableTargetFolderScavenging)
{
// Pick up all whole files we can use from current local state
ZEN_TRACE_CPU("UpdateFolder_GetLocalSequences");
@@ -5987,7 +5999,7 @@ namespace {
std::vector<ScavengeCopyOperation> ScavengeCopyOperations;
uint64_t ScavengedPathsCount = 0;
- if (!PrimeCacheOnly && EnableScavenging)
+ if (!Options.PrimeCacheOnly && Options.EnableOtherDownloadsScavenging)
{
ZEN_TRACE_CPU("UpdateFolder_GetScavengedSequences");
@@ -5995,7 +6007,7 @@ namespace {
if (!SequenceIndexesLeftToFindToRemoteIndex.empty())
{
- std::vector<ScavengeSource> ScavengeSources = GetDownloadedStatePaths(SystemRootDir);
+ std::vector<ScavengeSource> ScavengeSources = GetDownloadedStatePaths(Options.SystemRootDir);
auto EraseIt = std::remove_if(ScavengeSources.begin(), ScavengeSources.end(), [&Path](const ScavengeSource& Source) {
return Source.Path == Path;
});
@@ -6234,7 +6246,7 @@ namespace {
tsl::robin_map<IoHash, size_t, IoHash::Hasher> RawHashToCacheCopyDataIndex;
std::vector<CacheCopyData> CacheCopyDatas;
- if (!PrimeCacheOnly)
+ if (!Options.PrimeCacheOnly && Options.EnableTargetFolderScavenging)
{
ZEN_TRACE_CPU("UpdateFolder_GetLocalChunks");
@@ -6315,7 +6327,7 @@ namespace {
CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs();
}
- if (!PrimeCacheOnly)
+ if (!Options.PrimeCacheOnly && Options.EnableOtherDownloadsScavenging)
{
ZEN_TRACE_CPU("UpdateFolder_GetScavengeChunks");
@@ -6475,7 +6487,7 @@ namespace {
WorkerThreadPool& NetworkPool = GetNetworkPool();
WorkerThreadPool& WritePool = GetIOWorkerPool();
- ProgressBar WriteProgressBar(ProgressMode, PrimeCacheOnly ? "Downloading" : "Writing");
+ ProgressBar WriteProgressBar(ProgressMode, Options.PrimeCacheOnly ? "Downloading" : "Writing");
ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
struct LooseChunkHashWorkData
@@ -6562,11 +6574,12 @@ namespace {
for (uint32_t BlockIndex = 0; BlockIndex < BlockCount; BlockIndex++)
{
- const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
- const std::vector<uint32_t> BlockChunkIndexNeeded = GetNeededChunkBlockIndexes(BlockDescription);
+ const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+
+ const std::vector<uint32_t> BlockChunkIndexNeeded = GetNeededChunkBlockIndexes(BlockDescription);
if (!BlockChunkIndexNeeded.empty())
{
- if (PrimeCacheOnly)
+ if (Options.PrimeCacheOnly)
{
TotalRequestCount++;
TotalPartWriteCount++;
@@ -6583,7 +6596,7 @@ namespace {
TotalPartWriteCount++;
std::filesystem::path BlockPath =
- ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
+ ZenTempBlockFolderPath(Options.ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
if (IsFile(BlockPath))
{
CachedChunkBlockIndexes.push_back(BlockIndex);
@@ -6597,7 +6610,7 @@ namespace {
bool CanDoPartialBlockDownload =
(BlockDescription.HeaderSize > 0) &&
(BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size());
- if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload)
+ if (Options.AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload)
{
std::vector<BlockRangeDescriptor> BlockRanges;
@@ -6608,10 +6621,9 @@ namespace {
uint32_t CurrentOffset =
gsl::narrow<uint32_t>(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize);
- const uint64_t TotalBlockSize = std::accumulate(BlockDescription.ChunkCompressedLengths.begin(),
- BlockDescription.ChunkCompressedLengths.end(),
- std::uint64_t(CurrentOffset));
-
+ const uint64_t TotalBlockSize = std::accumulate(BlockDescription.ChunkCompressedLengths.begin(),
+ BlockDescription.ChunkCompressedLengths.end(),
+ std::uint64_t(CurrentOffset));
BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex};
while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() &&
ChunkBlockIndex < BlockDescription.ChunkRawHashes.size())
@@ -6843,7 +6855,7 @@ namespace {
{
break;
}
- if (!PrimeCacheOnly)
+ if (!Options.PrimeCacheOnly)
{
Work.ScheduleWork(
WritePool,
@@ -6906,7 +6918,8 @@ namespace {
std::move(LooseChunkHashWork.ChunkTargetPtrs);
const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex;
- if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]))
+ if (Options.PrimeCacheOnly &&
+ ExistsResult.ExistingBlobs.contains(RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]))
{
DownloadStats.RequestsCompleteCount++;
continue;
@@ -6916,7 +6929,7 @@ namespace {
WritePool,
[&Storage,
&Path,
- &ZenFolderPath,
+ &Options,
&RemoteContent,
&RemoteLookup,
&CacheFolderPath,
@@ -6924,7 +6937,6 @@ namespace {
&Work,
&WritePool,
&NetworkPool,
- PrimeCacheOnly,
&ExistsResult,
&DiskStats,
&DownloadStats,
@@ -6933,8 +6945,6 @@ namespace {
RemoteChunkIndex,
ChunkTargetPtrs,
BuildId = Oid(BuildId),
- LargeAttachmentSize,
- PreferredMultipartChunkSize,
TotalRequestCount,
TotalPartWriteCount,
&WriteCache,
@@ -6944,11 +6954,11 @@ namespace {
{
ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded");
std::filesystem::path ExistingCompressedChunkPath;
- if (!PrimeCacheOnly)
+ if (!Options.PrimeCacheOnly)
{
const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
std::filesystem::path CompressedChunkPath =
- ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString();
+ ZenTempDownloadFolderPath(Options.ZenFolderPath) / ChunkHash.ToHexString();
if (IsFile(CompressedChunkPath))
{
IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath);
@@ -6981,7 +6991,7 @@ namespace {
Work.ScheduleWork(
WritePool,
[&Path,
- &ZenFolderPath,
+ &Options,
&RemoteContent,
&RemoteLookup,
&CacheFolderPath,
@@ -7014,7 +7024,7 @@ namespace {
CompressedChunkPath));
}
- std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath);
+ std::filesystem::path TargetFolder = ZenTempCacheFolderPath(Options.ZenFolderPath);
bool NeedHashVerify = WriteCompressedChunk(TargetFolder,
RemoteContent,
RemoteLookup,
@@ -7059,10 +7069,9 @@ namespace {
Work.ScheduleWork(
NetworkPool,
[&Path,
- &ZenFolderPath,
+ &Options,
&Storage,
BuildId = Oid(BuildId),
- PrimeCacheOnly,
&RemoteContent,
&RemoteLookup,
&ExistsResult,
@@ -7078,8 +7087,6 @@ namespace {
TotalRequestCount,
&FilteredDownloadedBytesPerSecond,
&FilteredWrittenBytesPerSecond,
- LargeAttachmentSize,
- PreferredMultipartChunkSize,
RemoteChunkIndex,
ChunkTargetPtrs,
&DownloadStats](std::atomic<bool>&) mutable {
@@ -7104,7 +7111,7 @@ namespace {
{
FilteredDownloadedBytesPerSecond.Stop();
}
- AsyncWriteDownloadedChunk(ZenFolderPath,
+ AsyncWriteDownloadedChunk(Options.ZenFolderPath,
RemoteContent,
RemoteLookup,
RemoteChunkIndex,
@@ -7121,23 +7128,23 @@ namespace {
}
else
{
- if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize)
+ if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >=
+ Options.LargeAttachmentSize)
{
ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk");
DownloadLargeBlob(*Storage.BuildStorage,
- ZenTempDownloadFolderPath(ZenFolderPath),
+ ZenTempDownloadFolderPath(Options.ZenFolderPath),
BuildId,
ChunkHash,
- PreferredMultipartChunkSize,
+ Options.PreferredMultipartChunkSize,
Work,
NetworkPool,
DownloadStats,
[&Storage,
- &ZenFolderPath,
+ &Options,
&RemoteContent,
&RemoteLookup,
BuildId,
- PrimeCacheOnly,
&SequenceIndexChunksLeftToWriteCounters,
&WriteCache,
&Work,
@@ -7165,12 +7172,12 @@ namespace {
ZenContentType::kCompressedBinary,
CompositeBuffer(SharedBuffer(Payload)));
}
- if (!PrimeCacheOnly)
+ if (!Options.PrimeCacheOnly)
{
if (!AbortFlag)
{
AsyncWriteDownloadedChunk(
- ZenFolderPath,
+ Options.ZenFolderPath,
RemoteContent,
RemoteLookup,
RemoteChunkIndex,
@@ -7204,7 +7211,7 @@ namespace {
{
throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash));
}
- if (!PrimeCacheOnly)
+ if (!Options.PrimeCacheOnly)
{
if (!AbortFlag)
{
@@ -7216,7 +7223,7 @@ namespace {
{
FilteredDownloadedBytesPerSecond.Stop();
}
- AsyncWriteDownloadedChunk(ZenFolderPath,
+ AsyncWriteDownloadedChunk(Options.ZenFolderPath,
RemoteContent,
RemoteLookup,
RemoteChunkIndex,
@@ -7244,7 +7251,7 @@ namespace {
for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++)
{
- ZEN_ASSERT(!PrimeCacheOnly);
+ ZEN_ASSERT(!Options.PrimeCacheOnly);
if (AbortFlag)
{
break;
@@ -7450,7 +7457,7 @@ namespace {
for (uint32_t BlockIndex : CachedChunkBlockIndexes)
{
- ZEN_ASSERT(!PrimeCacheOnly);
+ ZEN_ASSERT(!Options.PrimeCacheOnly);
if (AbortFlag)
{
break;
@@ -7458,7 +7465,7 @@ namespace {
Work.ScheduleWork(
WritePool,
- [&ZenFolderPath,
+ [&Options,
&CacheFolderPath,
&RemoteContent,
&RemoteLookup,
@@ -7481,7 +7488,7 @@ namespace {
FilteredWrittenBytesPerSecond.Start();
std::filesystem::path BlockChunkPath =
- ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
+ ZenTempBlockFolderPath(Options.ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath);
if (!BlockBuffer)
{
@@ -7523,7 +7530,7 @@ namespace {
for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++)
{
- ZEN_ASSERT(!PrimeCacheOnly);
+ ZEN_ASSERT(!Options.PrimeCacheOnly);
if (AbortFlag)
{
break;
@@ -7534,7 +7541,7 @@ namespace {
Work.ScheduleWork(
NetworkPool,
[&Storage,
- &ZenFolderPath,
+ &Options,
BuildId,
&RemoteLookup,
&BlockDescriptions,
@@ -7610,11 +7617,12 @@ namespace {
if (!Ec)
{
BlockBuffer.SetDeleteOnClose(false);
- BlockBuffer = {};
- BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}",
- BlockDescription.BlockHash,
- BlockRange.RangeStart,
- BlockRange.RangeLength);
+ BlockBuffer = {};
+ BlockChunkPath =
+ ZenTempBlockFolderPath(Options.ZenFolderPath) / fmt::format("{}_{:x}_{:x}",
+ BlockDescription.BlockHash,
+ BlockRange.RangeStart,
+ BlockRange.RangeLength);
RenameFile(TempBlobPath, BlockChunkPath, Ec);
if (Ec)
{
@@ -7633,10 +7641,10 @@ namespace {
{
ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock");
// Could not be moved and rather large, lets store it on disk
- BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}",
- BlockDescription.BlockHash,
- BlockRange.RangeStart,
- BlockRange.RangeLength);
+ BlockChunkPath = ZenTempBlockFolderPath(Options.ZenFolderPath) / fmt::format("{}_{:x}_{:x}",
+ BlockDescription.BlockHash,
+ BlockRange.RangeStart,
+ BlockRange.RangeLength);
TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer);
BlockBuffer = {};
}
@@ -7732,7 +7740,7 @@ namespace {
break;
}
- if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(BlockDescriptions[BlockIndex].BlockHash))
+ if (Options.PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(BlockDescriptions[BlockIndex].BlockHash))
{
DownloadStats.RequestsCompleteCount++;
continue;
@@ -7741,9 +7749,8 @@ namespace {
Work.ScheduleWork(
NetworkPool,
[&Storage,
- &ZenFolderPath,
+ &Options,
BuildId,
- PrimeCacheOnly,
&BlockDescriptions,
&WritePartsComplete,
TotalPartWriteCount,
@@ -7804,7 +7811,7 @@ namespace {
FilteredDownloadedBytesPerSecond.Stop();
}
- if (!PrimeCacheOnly)
+ if (!Options.PrimeCacheOnly)
{
std::filesystem::path BlockChunkPath;
@@ -7820,9 +7827,9 @@ namespace {
if (!Ec)
{
BlockBuffer.SetDeleteOnClose(false);
- BlockBuffer = {};
- BlockChunkPath =
- ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
+ BlockBuffer = {};
+ BlockChunkPath = ZenTempBlockFolderPath(Options.ZenFolderPath) /
+ BlockDescription.BlockHash.ToHexString();
RenameFile(TempBlobPath, BlockChunkPath, Ec);
if (Ec)
{
@@ -7841,7 +7848,8 @@ namespace {
{
ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock");
// Could not be moved and rather large, lets store it on disk
- BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
+ BlockChunkPath =
+ ZenTempBlockFolderPath(Options.ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer);
BlockBuffer = {};
}
@@ -7943,11 +7951,11 @@ namespace {
(DownloadStats.RequestsCompleteCount == TotalRequestCount)
? ""
: fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8));
- std::string WriteDetails = PrimeCacheOnly ? ""
- : fmt::format(" {}/{} ({}B/s) written.",
- NiceBytes(DiskStats.WriteByteCount.load()),
- NiceBytes(BytesToWrite),
- NiceNum(FilteredWrittenBytesPerSecond.GetCurrent()));
+ std::string WriteDetails = Options.PrimeCacheOnly ? ""
+ : fmt::format(" {}/{} ({}B/s) written.",
+ NiceBytes(DiskStats.WriteByteCount.load()),
+ NiceBytes(BytesToWrite),
+ NiceNum(FilteredWrittenBytesPerSecond.GetCurrent()));
std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}",
DownloadStats.RequestsCompleteCount.load(),
TotalRequestCount,
@@ -7955,11 +7963,11 @@ namespace {
DownloadRateString,
WriteDetails);
WriteProgressBar.UpdateState(
- {.Task = PrimeCacheOnly ? "Downloading " : "Writing chunks ",
+ {.Task = Options.PrimeCacheOnly ? "Downloading " : "Writing chunks ",
.Details = Details,
- .TotalCount = PrimeCacheOnly ? TotalRequestCount : BytesToWrite,
- .RemainingCount = PrimeCacheOnly ? (TotalRequestCount - DownloadStats.RequestsCompleteCount.load())
- : (BytesToWrite - DiskStats.WriteByteCount.load()),
+ .TotalCount = Options.PrimeCacheOnly ? TotalRequestCount : BytesToWrite,
+ .RemainingCount = Options.PrimeCacheOnly ? (TotalRequestCount - DownloadStats.RequestsCompleteCount.load())
+ : (BytesToWrite - DiskStats.WriteByteCount.load()),
.Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
false);
});
@@ -7974,7 +7982,7 @@ namespace {
return;
}
- if (!PrimeCacheOnly)
+ if (!Options.PrimeCacheOnly)
{
uint32_t RawSequencesMissingWriteCount = 0;
for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++)
@@ -8019,7 +8027,7 @@ namespace {
WriteChunkStats.WriteTimeUs = FilteredWrittenBytesPerSecond.GetElapsedTimeUS();
}
- if (PrimeCacheOnly)
+ if (Options.PrimeCacheOnly)
{
return;
}
@@ -8066,7 +8074,7 @@ namespace {
ZEN_ASSERT_SLOW(IsFile((Path / LocalContent.Paths[LocalPathIndex]).make_preferred()));
- if (!WipeTargetFolder)
+ if (!Options.WipeTargetFolder)
{
if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string());
RemotePathIt != RemotePathToRemoteIndex.end())
@@ -8105,7 +8113,7 @@ namespace {
SkippedCount++;
}
}
- else if (!WipeTargetFolder)
+ else if (!Options.WipeTargetFolder)
{
// We don't need it
RemoveLocalPathIndexes.push_back(LocalPathIndex);
@@ -8188,7 +8196,7 @@ namespace {
}
}
- if (WipeTargetFolder)
+ if (Options.WipeTargetFolder)
{
ZEN_TRACE_CPU("UpdateFolder_WipeTarget");
Stopwatch Timer;
@@ -9386,21 +9394,27 @@ namespace {
return Result;
};
+ struct DownloadOptions
+ {
+ std::filesystem::path SystemRootDir;
+ std::filesystem::path ZenFolderPath;
+ bool AllowMultiparts = true;
+ bool AllowPartialBlockRequests = true;
+ bool CleanTargetFolder = false;
+ bool PostDownloadVerify = false;
+ bool PrimeCacheOnly = false;
+ bool EnableOtherDownloadsScavenging = true;
+ bool EnableTargetFolderScavenging = true;
+ std::string IncludeWildcard;
+ std::string ExcludeWildcard;
+ };
+
void DownloadFolder(StorageInstance& Storage,
const Oid& BuildId,
const std::vector<Oid>& BuildPartIds,
std::span<const std::string> BuildPartNames,
const std::filesystem::path& Path,
- const std::filesystem::path& ZenFolderPath,
- const std::filesystem::path SystemRootDir,
- bool AllowMultiparts,
- bool AllowPartialBlockRequests,
- bool WipeTargetFolder,
- bool PostDownloadVerify,
- bool PrimeCacheOnly,
- bool EnableScavenging,
- std::string_view IncludeWildcard,
- std::string_view ExcludeWildcard)
+ const DownloadOptions& Options)
{
ZEN_TRACE_CPU("DownloadFolder");
@@ -9419,18 +9433,18 @@ namespace {
auto EndProgress =
MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); });
- ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests)));
+ ZEN_ASSERT((!Options.PrimeCacheOnly) || (Options.PrimeCacheOnly && (!Options.AllowPartialBlockRequests)));
Stopwatch DownloadTimer;
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount);
- const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath);
+ const std::filesystem::path ZenTempFolder = ZenTempFolderPath(Options.ZenFolderPath);
CreateDirectories(ZenTempFolder);
- CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath));
- CreateDirectories(ZenTempCacheFolderPath(ZenFolderPath));
- CreateDirectories(ZenTempDownloadFolderPath(ZenFolderPath));
+ CreateDirectories(ZenTempBlockFolderPath(Options.ZenFolderPath));
+ CreateDirectories(ZenTempCacheFolderPath(Options.ZenFolderPath));
+ CreateDirectories(ZenTempDownloadFolderPath(Options.ZenFolderPath));
std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
@@ -9449,19 +9463,19 @@ namespace {
ChunkedFolderContent RemoteContent = GetRemoteContent(Storage,
BuildId,
AllBuildParts,
- IncludeWildcard,
- ExcludeWildcard,
+ Options.IncludeWildcard,
+ Options.ExcludeWildcard,
ChunkController,
PartContents,
BlockDescriptions,
LooseChunkHashes);
- const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
+ const std::uint64_t LargeAttachmentSize = Options.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
GetFolderContentStatistics LocalFolderScanStats;
ChunkingStatistics ChunkingStats;
ChunkedFolderContent LocalContent;
FolderContent LocalFolderContent;
- if (!PrimeCacheOnly)
+ if (!Options.PrimeCacheOnly)
{
if (IsDir(Path))
{
@@ -9474,11 +9488,11 @@ namespace {
LocalContent = GetLocalContent(LocalFolderScanStats,
ChunkingStats,
Path,
- ZenStateFilePath(ZenFolderPath),
+ ZenStateFilePath(Options.ZenFolderPath),
*ChunkController,
RemoteContent.Paths,
- IncludeWildcard,
- ExcludeWildcard,
+ Options.IncludeWildcard,
+ Options.ExcludeWildcard,
LocalFolderContent);
}
else
@@ -9532,7 +9546,7 @@ namespace {
return true;
};
- if (CompareContent(RemoteContent, LocalContent) && !WipeTargetFolder)
+ if (Options.EnableTargetFolderScavenging && !Options.CleanTargetFolder && CompareContent(RemoteContent, LocalContent))
{
if (!IsQuiet)
{
@@ -9542,14 +9556,14 @@ namespace {
Stopwatch WriteStateTimer;
CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent, Path);
- CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path());
- TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView());
+ CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path());
+ TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView());
if (!IsQuiet)
{
ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
}
- AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path);
+ AddDownloadedPath(Options.SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(Options.ZenFolderPath), Path);
}
else
{
@@ -9576,21 +9590,22 @@ namespace {
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount);
- UpdateFolder(SystemRootDir,
- Storage,
+ UpdateFolder(Storage,
BuildId,
Path,
- ZenFolderPath,
- LargeAttachmentSize,
- PreferredMultipartChunkSize,
LocalContent,
RemoteContent,
BlockDescriptions,
LooseChunkHashes,
- AllowPartialBlockRequests,
- WipeTargetFolder,
- PrimeCacheOnly,
- EnableScavenging,
+ UpdateOptions{.SystemRootDir = Options.SystemRootDir,
+ .ZenFolderPath = Options.ZenFolderPath,
+ .LargeAttachmentSize = LargeAttachmentSize,
+ .PreferredMultipartChunkSize = PreferredMultipartChunkSize,
+ .AllowPartialBlockRequests = Options.AllowPartialBlockRequests,
+ .WipeTargetFolder = Options.CleanTargetFolder,
+ .PrimeCacheOnly = Options.PrimeCacheOnly,
+ .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging,
+ .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging},
LocalFolderState,
DiskStats,
CacheMappingStats,
@@ -9600,28 +9615,28 @@ namespace {
if (!AbortFlag)
{
- if (!PrimeCacheOnly)
+ if (!Options.PrimeCacheOnly)
{
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount);
- VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats);
+ VerifyFolder(RemoteContent, Path, Options.PostDownloadVerify, VerifyFolderStats);
Stopwatch WriteStateTimer;
CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState, Path);
- CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path());
- TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView());
+ CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path());
+ TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView());
if (!IsQuiet)
{
ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
}
- AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path);
+ AddDownloadedPath(Options.SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(Options.ZenFolderPath), Path);
#if 0
ExtendableStringBuilder<1024> SB;
CompactBinaryToJson(StateObject, SB);
- WriteFile(ZenStateFileJsonPath(ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
+ WriteFile(ZenStateFileJsonPath(Options.ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
#endif // 0
}
const uint64_t DownloadCount = DownloadStats.DownloadedChunkCount.load() + DownloadStats.DownloadedBlockCount.load() +
@@ -9660,7 +9675,7 @@ namespace {
}
}
}
- if (PrimeCacheOnly)
+ if (Options.PrimeCacheOnly)
{
if (Storage.BuildCacheStorage)
{
@@ -10299,8 +10314,18 @@ BuildsCommand::BuildsCommand()
"all parts will be downloaded",
cxxopts::value(m_BuildPartNames),
"<name>");
- m_DownloadOptions
- .add_option("", "", "clean", "Delete all data in target folder before downloading", cxxopts::value(m_Clean), "<clean>");
+ m_DownloadOptions.add_option("",
+ "",
+ "clean",
+ "Delete all data in target folder that is not part of the downloaded content",
+ cxxopts::value(m_Clean),
+ "<clean>");
+ m_DownloadOptions.add_option("",
+ "",
+ "force",
+ "Force download of all content by ignoring any existing local content",
+ cxxopts::value(m_Force),
+ "<force>");
m_DownloadOptions.add_option("",
"",
"allow-multipart",
@@ -11405,7 +11430,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
m_Clean,
m_PostUploadVerify);
- if (false)
+ if (true)
{
if (!IsQuiet)
{
@@ -11470,33 +11495,38 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (m_Clean && m_PrimeCacheOnly)
{
- ZEN_CONSOLE_WARN("Ignoring 'clean' option when 'cache-prime-only' is enabled");
+ ZEN_CONSOLE_WARN("Ignoring '--clean' option when '--cache-prime-only' is enabled");
+ }
+
+ if (m_Force && m_PrimeCacheOnly)
+ {
+ ZEN_CONSOLE_WARN("Ignoring '--force' option when '--cache-prime-only' is enabled");
}
if (m_AllowPartialBlockRequests && m_PrimeCacheOnly)
{
- ZEN_CONSOLE_WARN("Ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled");
+ ZEN_CONSOLE_WARN("Ignoring '--allow-partial-block-requests' option when '--cache-prime-only' is enabled");
}
std::vector<Oid> BuildPartIds = ParseBuildPartIds();
std::vector<std::string> BuildPartNames = ParseBuildPartNames();
- // TODO: Add file filters
DownloadFolder(Storage,
BuildId,
BuildPartIds,
BuildPartNames,
m_Path,
- m_ZenFolderPath,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests && !m_PrimeCacheOnly,
- m_Clean,
- m_PostDownloadVerify,
- m_PrimeCacheOnly,
- m_EnableScavenging,
- m_IncludeWildcard,
- m_ExcludeWildcard);
+ DownloadOptions{.SystemRootDir = m_SystemRootDir,
+ .ZenFolderPath = m_ZenFolderPath,
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests && !m_PrimeCacheOnly,
+ .CleanTargetFolder = m_Clean,
+ .PostDownloadVerify = m_PostDownloadVerify,
+ .PrimeCacheOnly = m_PrimeCacheOnly,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging && !m_Force,
+ .EnableTargetFolderScavenging = !m_Force,
+ .IncludeWildcard = m_IncludeWildcard,
+ .ExcludeWildcard = m_ExcludeWildcard});
if (AbortFlag)
{
@@ -11698,16 +11728,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{},
{},
m_Path,
- m_ZenFolderPath,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- BuildIdString == m_BuildIds.front(),
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
+ DownloadOptions{.SystemRootDir = m_SystemRootDir,
+ .ZenFolderPath = m_ZenFolderPath,
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .CleanTargetFolder = BuildIdString == m_BuildIds.front(),
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = false});
if (AbortFlag)
{
throw std::runtime_error("Multitest aborted");
@@ -11870,16 +11899,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{BuildPartId},
{},
DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- true,
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
+
+ DownloadOptions{.SystemRootDir = m_SystemRootDir,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .CleanTargetFolder = true,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = false});
if (AbortFlag)
{
throw std::runtime_error("Test aborted. (Download build)");
@@ -11891,16 +11920,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{BuildPartId},
{},
DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
+
+ DownloadOptions{.SystemRootDir = m_SystemRootDir,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .CleanTargetFolder = false,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = true});
if (AbortFlag)
{
throw std::runtime_error("Test aborted. (Re-download identical target)");
@@ -12004,16 +12033,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{BuildPartId},
{},
DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
+
+ DownloadOptions{.SystemRootDir = m_SystemRootDir,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .CleanTargetFolder = false,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = true});
if (AbortFlag)
{
throw std::runtime_error("Test aborted. (Re-download scrambled target)");
@@ -12056,16 +12085,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{BuildPartId},
{},
DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
+
+ DownloadOptions{.SystemRootDir = m_SystemRootDir,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .CleanTargetFolder = false,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = true});
if (AbortFlag)
{
throw std::runtime_error("Test aborted. (Download original)");
@@ -12077,16 +12106,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{BuildPartId2},
{},
DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
+ DownloadOptions{.SystemRootDir = m_SystemRootDir,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .CleanTargetFolder = false,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = true});
if (AbortFlag)
{
throw std::runtime_error("Test aborted. (Download scrambled)");
@@ -12098,16 +12126,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{BuildPartId2},
{},
DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
+ DownloadOptions{.SystemRootDir = m_SystemRootDir,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .CleanTargetFolder = false,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = true});
if (AbortFlag)
{
throw std::runtime_error("Test aborted. (Re-download scrambled)");
@@ -12119,16 +12146,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{BuildPartId},
{},
DownloadPath2,
- DownloadPath2 / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
+ DownloadOptions{.SystemRootDir = m_SystemRootDir,
+ .ZenFolderPath = DownloadPath2 / ZenFolderName,
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .CleanTargetFolder = false,
+ .PostDownloadVerify = true,
+ .PrimeCacheOnly = false,
+ .EnableOtherDownloadsScavenging = m_EnableScavenging,
+ .EnableTargetFolderScavenging = true});
if (AbortFlag)
{
throw std::runtime_error("Test aborted. (Download original)");
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index 79d69b16c..26f804fa5 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -61,6 +61,7 @@ private:
std::string m_BuildPartName; // Defaults to name of leaf folder in m_Path
std::string m_BuildPartId; // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName
bool m_Clean = false;
+ bool m_Force = false;
uint8_t m_BlockReuseMinPercentLimit = 85;
bool m_AllowMultiparts = true;
bool m_AllowPartialBlockRequests = true;