diff options
| author | Dan Engelbrecht <[email protected]> | 2025-09-17 15:42:26 +0200 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-09-17 15:42:26 +0200 |
| commit | 08d7ab0fa1e8deff6857fdba0a14fea48f717b5a (patch) | |
| tree | bde758f0380b6f1a86811e970064ba53429e84bd | |
| parent | rpmalloc fixes (#499) (diff) | |
| download | zen-08d7ab0fa1e8deff6857fdba0a14fea48f717b5a.tar.xz zen-08d7ab0fa1e8deff6857fdba0a14fea48f717b5a.zip | |
add builds download force option (#498)
* add --force option to builds download command
| -rw-r--r-- | CHANGELOG.md | 1 | ||||
| -rw-r--r-- | src/zen/cmds/builds_cmd.cpp | 566 | ||||
| -rw-r--r-- | src/zen/cmds/builds_cmd.h | 1 |
3 files changed, 298 insertions, 270 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 39786b3b5..b398efe0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## +- Feature: Added `--force` option to `zen builds download` command that forces download of all content ignoring any local state - Improvement: Change BadAlloc exceptions in GC to warnings - Improvement: Add explict ASSERT exception catch in http plugin request processing - Improvement: Make exceptions handled in http request processing to warnings diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 1e097edb7..179f34193 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -5758,21 +5758,27 @@ namespace { return Result; } - void UpdateFolder(const std::filesystem::path& SystemRootDir, - StorageInstance& Storage, + struct UpdateOptions + { + std::filesystem::path SystemRootDir; + std::filesystem::path ZenFolderPath; + std::uint64_t LargeAttachmentSize = DefaultPreferredMultipartChunkSize * 4u; + std::uint64_t PreferredMultipartChunkSize = DefaultPreferredMultipartChunkSize; + bool AllowPartialBlockRequests = true; + bool WipeTargetFolder = false; + bool PrimeCacheOnly = false; + bool EnableOtherDownloadsScavenging = true; + bool EnableTargetFolderScavenging = true; + }; + + void UpdateFolder(StorageInstance& Storage, const Oid& BuildId, const std::filesystem::path& Path, - const std::filesystem::path& ZenFolderPath, - const std::uint64_t LargeAttachmentSize, - const std::uint64_t PreferredMultipartChunkSize, const ChunkedFolderContent& LocalContent, const ChunkedFolderContent& RemoteContent, const std::vector<ChunkBlockDescription>& BlockDescriptions, const std::vector<IoHash>& LooseChunkHashes, - bool AllowPartialBlockRequests, - bool WipeTargetFolder, - bool PrimeCacheOnly, - bool EnableScavenging, + const UpdateOptions& Options, FolderContent& OutLocalFolderState, DiskStatistics& DiskStats, CacheMappingStatistics& CacheMappingStats, @@ -5782,7 +5788,7 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder"); - ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests))); + ZEN_ASSERT((!Options.PrimeCacheOnly) || (Options.PrimeCacheOnly && (!Options.AllowPartialBlockRequests))); Stopwatch IndexTimer; @@ -5795,7 +5801,7 @@ namespace { ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); } - const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(ZenFolderPath); + const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(Options.ZenFolderPath); Stopwatch CacheMappingTimer; @@ -5805,7 +5811,7 @@ namespace { tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedChunkHashesFound; tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedSequenceHashesFound; - if (!PrimeCacheOnly) + if (!Options.PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_CheckChunkCache"); @@ -5817,40 +5823,43 @@ namespace { CacheDirContent); for (size_t Index = 0; Index < CacheDirContent.Files.size(); Index++) { - IoHash FileHash; - if (IoHash::TryParse(CacheDirContent.Files[Index].filename().string(), FileHash)) + if (Options.EnableTargetFolderScavenging) { - if (auto ChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(FileHash); - ChunkIt != RemoteLookup.ChunkHashToChunkIndex.end()) + IoHash FileHash; + if (IoHash::TryParse(CacheDirContent.Files[Index].filename().string(), FileHash)) { - const uint32_t ChunkIndex = ChunkIt->second; - const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - if (ChunkSize == CacheDirContent.FileSizes[Index]) + if (auto ChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(FileHash); + ChunkIt != RemoteLookup.ChunkHashToChunkIndex.end()) { - CachedChunkHashesFound.insert({FileHash, ChunkIndex}); - CacheMappingStats.CacheChunkCount++; - CacheMappingStats.CacheChunkByteCount += ChunkSize; - continue; + const uint32_t ChunkIndex = ChunkIt->second; + const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + if (ChunkSize == CacheDirContent.FileSizes[Index]) + { + CachedChunkHashesFound.insert({FileHash, ChunkIndex}); + CacheMappingStats.CacheChunkCount++; + CacheMappingStats.CacheChunkByteCount += ChunkSize; + continue; + } } - } - else if (auto SequenceIt = RemoteLookup.RawHashToSequenceIndex.find(FileHash); - SequenceIt != RemoteLookup.RawHashToSequenceIndex.end()) - { - const uint32_t SequenceIndex = SequenceIt->second; - const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; - const uint64_t SequenceSize = RemoteContent.RawSizes[PathIndex]; - if (SequenceSize == CacheDirContent.FileSizes[Index]) + else if (auto SequenceIt = RemoteLookup.RawHashToSequenceIndex.find(FileHash); + SequenceIt != RemoteLookup.RawHashToSequenceIndex.end()) { - CachedSequenceHashesFound.insert({FileHash, SequenceIndex}); - CacheMappingStats.CacheSequenceHashesCount++; - CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize; + const uint32_t SequenceIndex = SequenceIt->second; + const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + const uint64_t SequenceSize = RemoteContent.RawSizes[PathIndex]; + if (SequenceSize == CacheDirContent.FileSizes[Index]) + { + CachedSequenceHashesFound.insert({FileHash, SequenceIndex}); + CacheMappingStats.CacheSequenceHashesCount++; + CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize; - const std::filesystem::path CacheFilePath = - GetFinalChunkedSequenceFileName(CacheFolderPath, - RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); - ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(CacheFolderPath, + RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); - continue; + continue; + } } } } @@ -5860,7 +5869,7 @@ namespace { } tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedBlocksFound; - if (!PrimeCacheOnly) + if (!Options.PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_CheckBlockCache"); @@ -5875,31 +5884,34 @@ namespace { } DirectoryContent BlockDirContent; - GetDirectoryContent(ZenTempBlockFolderPath(ZenFolderPath), + GetDirectoryContent(ZenTempBlockFolderPath(Options.ZenFolderPath), DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, BlockDirContent); CachedBlocksFound.reserve(BlockDirContent.Files.size()); for (size_t Index = 0; Index < BlockDirContent.Files.size(); Index++) { - IoHash FileHash; - if (IoHash::TryParse(BlockDirContent.Files[Index].filename().string(), FileHash)) + if (Options.EnableTargetFolderScavenging) { - if (auto BlockIt = AllBlockSizes.find(FileHash); BlockIt != AllBlockSizes.end()) + IoHash FileHash; + if (IoHash::TryParse(BlockDirContent.Files[Index].filename().string(), FileHash)) { - const uint32_t BlockIndex = BlockIt->second; - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - uint64_t BlockSize = CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize; - for (uint64_t ChunkSize : BlockDescription.ChunkCompressedLengths) + if (auto BlockIt = AllBlockSizes.find(FileHash); BlockIt != AllBlockSizes.end()) { - BlockSize += ChunkSize; - } + const uint32_t BlockIndex = BlockIt->second; + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + uint64_t BlockSize = CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize; + for (uint64_t ChunkSize : BlockDescription.ChunkCompressedLengths) + { + BlockSize += ChunkSize; + } - if (BlockSize == BlockDirContent.FileSizes[Index]) - { - CachedBlocksFound.insert({FileHash, BlockIndex}); - CacheMappingStats.CacheBlockCount++; - CacheMappingStats.CacheBlocksByteCount += BlockSize; - continue; + if (BlockSize == BlockDirContent.FileSizes[Index]) + { + CachedBlocksFound.insert({FileHash, BlockIndex}); + CacheMappingStats.CacheBlockCount++; + CacheMappingStats.CacheBlocksByteCount += BlockSize; + continue; + } } } } @@ -5912,7 +5924,7 @@ namespace { std::vector<uint32_t> LocalPathIndexesMatchingSequenceIndexes; tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> SequenceIndexesLeftToFindToRemoteIndex; - if (!PrimeCacheOnly) + if (!Options.PrimeCacheOnly && Options.EnableTargetFolderScavenging) { // Pick up all whole files we can use from current local state ZEN_TRACE_CPU("UpdateFolder_GetLocalSequences"); @@ -5987,7 +5999,7 @@ namespace { std::vector<ScavengeCopyOperation> ScavengeCopyOperations; uint64_t ScavengedPathsCount = 0; - if (!PrimeCacheOnly && EnableScavenging) + if (!Options.PrimeCacheOnly && Options.EnableOtherDownloadsScavenging) { ZEN_TRACE_CPU("UpdateFolder_GetScavengedSequences"); @@ -5995,7 +6007,7 @@ namespace { if (!SequenceIndexesLeftToFindToRemoteIndex.empty()) { - std::vector<ScavengeSource> ScavengeSources = GetDownloadedStatePaths(SystemRootDir); + std::vector<ScavengeSource> ScavengeSources = GetDownloadedStatePaths(Options.SystemRootDir); auto EraseIt = std::remove_if(ScavengeSources.begin(), ScavengeSources.end(), [&Path](const ScavengeSource& Source) { return Source.Path == Path; }); @@ -6234,7 +6246,7 @@ namespace { tsl::robin_map<IoHash, size_t, IoHash::Hasher> RawHashToCacheCopyDataIndex; std::vector<CacheCopyData> CacheCopyDatas; - if (!PrimeCacheOnly) + if (!Options.PrimeCacheOnly && Options.EnableTargetFolderScavenging) { ZEN_TRACE_CPU("UpdateFolder_GetLocalChunks"); @@ -6315,7 +6327,7 @@ namespace { CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs(); } - if (!PrimeCacheOnly) + if (!Options.PrimeCacheOnly && Options.EnableOtherDownloadsScavenging) { ZEN_TRACE_CPU("UpdateFolder_GetScavengeChunks"); @@ -6475,7 +6487,7 @@ namespace { WorkerThreadPool& NetworkPool = GetNetworkPool(); WorkerThreadPool& WritePool = GetIOWorkerPool(); - ProgressBar WriteProgressBar(ProgressMode, PrimeCacheOnly ? "Downloading" : "Writing"); + ProgressBar WriteProgressBar(ProgressMode, Options.PrimeCacheOnly ? "Downloading" : "Writing"); ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); struct LooseChunkHashWorkData @@ -6562,11 +6574,12 @@ namespace { for (uint32_t BlockIndex = 0; BlockIndex < BlockCount; BlockIndex++) { - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - const std::vector<uint32_t> BlockChunkIndexNeeded = GetNeededChunkBlockIndexes(BlockDescription); + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + + const std::vector<uint32_t> BlockChunkIndexNeeded = GetNeededChunkBlockIndexes(BlockDescription); if (!BlockChunkIndexNeeded.empty()) { - if (PrimeCacheOnly) + if (Options.PrimeCacheOnly) { TotalRequestCount++; TotalPartWriteCount++; @@ -6583,7 +6596,7 @@ namespace { TotalPartWriteCount++; std::filesystem::path BlockPath = - ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + ZenTempBlockFolderPath(Options.ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); if (IsFile(BlockPath)) { CachedChunkBlockIndexes.push_back(BlockIndex); @@ -6597,7 +6610,7 @@ namespace { bool CanDoPartialBlockDownload = (BlockDescription.HeaderSize > 0) && (BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size()); - if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) + if (Options.AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) { std::vector<BlockRangeDescriptor> BlockRanges; @@ -6608,10 +6621,9 @@ namespace { uint32_t CurrentOffset = gsl::narrow<uint32_t>(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); - const uint64_t TotalBlockSize = std::accumulate(BlockDescription.ChunkCompressedLengths.begin(), - BlockDescription.ChunkCompressedLengths.end(), - std::uint64_t(CurrentOffset)); - + const uint64_t TotalBlockSize = std::accumulate(BlockDescription.ChunkCompressedLengths.begin(), + BlockDescription.ChunkCompressedLengths.end(), + std::uint64_t(CurrentOffset)); BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex}; while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() && ChunkBlockIndex < BlockDescription.ChunkRawHashes.size()) @@ -6843,7 +6855,7 @@ namespace { { break; } - if (!PrimeCacheOnly) + if (!Options.PrimeCacheOnly) { Work.ScheduleWork( WritePool, @@ -6906,7 +6918,8 @@ namespace { std::move(LooseChunkHashWork.ChunkTargetPtrs); const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex; - if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex])) + if (Options.PrimeCacheOnly && + ExistsResult.ExistingBlobs.contains(RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex])) { DownloadStats.RequestsCompleteCount++; continue; @@ -6916,7 +6929,7 @@ namespace { WritePool, [&Storage, &Path, - &ZenFolderPath, + &Options, &RemoteContent, &RemoteLookup, &CacheFolderPath, @@ -6924,7 +6937,6 @@ namespace { &Work, &WritePool, &NetworkPool, - PrimeCacheOnly, &ExistsResult, &DiskStats, &DownloadStats, @@ -6933,8 +6945,6 @@ namespace { RemoteChunkIndex, ChunkTargetPtrs, BuildId = Oid(BuildId), - LargeAttachmentSize, - PreferredMultipartChunkSize, TotalRequestCount, TotalPartWriteCount, &WriteCache, @@ -6944,11 +6954,11 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded"); std::filesystem::path ExistingCompressedChunkPath; - if (!PrimeCacheOnly) + if (!Options.PrimeCacheOnly) { const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; std::filesystem::path CompressedChunkPath = - ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); + ZenTempDownloadFolderPath(Options.ZenFolderPath) / ChunkHash.ToHexString(); if (IsFile(CompressedChunkPath)) { IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath); @@ -6981,7 +6991,7 @@ namespace { Work.ScheduleWork( WritePool, [&Path, - &ZenFolderPath, + &Options, &RemoteContent, &RemoteLookup, &CacheFolderPath, @@ -7014,7 +7024,7 @@ namespace { CompressedChunkPath)); } - std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath); + std::filesystem::path TargetFolder = ZenTempCacheFolderPath(Options.ZenFolderPath); bool NeedHashVerify = WriteCompressedChunk(TargetFolder, RemoteContent, RemoteLookup, @@ -7059,10 +7069,9 @@ namespace { Work.ScheduleWork( NetworkPool, [&Path, - &ZenFolderPath, + &Options, &Storage, BuildId = Oid(BuildId), - PrimeCacheOnly, &RemoteContent, &RemoteLookup, &ExistsResult, @@ -7078,8 +7087,6 @@ namespace { TotalRequestCount, &FilteredDownloadedBytesPerSecond, &FilteredWrittenBytesPerSecond, - LargeAttachmentSize, - PreferredMultipartChunkSize, RemoteChunkIndex, ChunkTargetPtrs, &DownloadStats](std::atomic<bool>&) mutable { @@ -7104,7 +7111,7 @@ namespace { { FilteredDownloadedBytesPerSecond.Stop(); } - AsyncWriteDownloadedChunk(ZenFolderPath, + AsyncWriteDownloadedChunk(Options.ZenFolderPath, RemoteContent, RemoteLookup, RemoteChunkIndex, @@ -7121,23 +7128,23 @@ namespace { } else { - if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) + if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= + Options.LargeAttachmentSize) { ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); DownloadLargeBlob(*Storage.BuildStorage, - ZenTempDownloadFolderPath(ZenFolderPath), + ZenTempDownloadFolderPath(Options.ZenFolderPath), BuildId, ChunkHash, - PreferredMultipartChunkSize, + Options.PreferredMultipartChunkSize, Work, NetworkPool, DownloadStats, [&Storage, - &ZenFolderPath, + &Options, &RemoteContent, &RemoteLookup, BuildId, - PrimeCacheOnly, &SequenceIndexChunksLeftToWriteCounters, &WriteCache, &Work, @@ -7165,12 +7172,12 @@ namespace { ZenContentType::kCompressedBinary, CompositeBuffer(SharedBuffer(Payload))); } - if (!PrimeCacheOnly) + if (!Options.PrimeCacheOnly) { if (!AbortFlag) { AsyncWriteDownloadedChunk( - ZenFolderPath, + Options.ZenFolderPath, RemoteContent, RemoteLookup, RemoteChunkIndex, @@ -7204,7 +7211,7 @@ namespace { { throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); } - if (!PrimeCacheOnly) + if (!Options.PrimeCacheOnly) { if (!AbortFlag) { @@ -7216,7 +7223,7 @@ namespace { { FilteredDownloadedBytesPerSecond.Stop(); } - AsyncWriteDownloadedChunk(ZenFolderPath, + AsyncWriteDownloadedChunk(Options.ZenFolderPath, RemoteContent, RemoteLookup, RemoteChunkIndex, @@ -7244,7 +7251,7 @@ namespace { for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) { - ZEN_ASSERT(!PrimeCacheOnly); + ZEN_ASSERT(!Options.PrimeCacheOnly); if (AbortFlag) { break; @@ -7450,7 +7457,7 @@ namespace { for (uint32_t BlockIndex : CachedChunkBlockIndexes) { - ZEN_ASSERT(!PrimeCacheOnly); + ZEN_ASSERT(!Options.PrimeCacheOnly); if (AbortFlag) { break; @@ -7458,7 +7465,7 @@ namespace { Work.ScheduleWork( WritePool, - [&ZenFolderPath, + [&Options, &CacheFolderPath, &RemoteContent, &RemoteLookup, @@ -7481,7 +7488,7 @@ namespace { FilteredWrittenBytesPerSecond.Start(); std::filesystem::path BlockChunkPath = - ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + ZenTempBlockFolderPath(Options.ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); if (!BlockBuffer) { @@ -7523,7 +7530,7 @@ namespace { for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++) { - ZEN_ASSERT(!PrimeCacheOnly); + ZEN_ASSERT(!Options.PrimeCacheOnly); if (AbortFlag) { break; @@ -7534,7 +7541,7 @@ namespace { Work.ScheduleWork( NetworkPool, [&Storage, - &ZenFolderPath, + &Options, BuildId, &RemoteLookup, &BlockDescriptions, @@ -7610,11 +7617,12 @@ namespace { if (!Ec) { BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); + BlockBuffer = {}; + BlockChunkPath = + ZenTempBlockFolderPath(Options.ZenFolderPath) / fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); RenameFile(TempBlobPath, BlockChunkPath, Ec); if (Ec) { @@ -7633,10 +7641,10 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); // Could not be moved and rather large, lets store it on disk - BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); + BlockChunkPath = ZenTempBlockFolderPath(Options.ZenFolderPath) / fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); BlockBuffer = {}; } @@ -7732,7 +7740,7 @@ namespace { break; } - if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(BlockDescriptions[BlockIndex].BlockHash)) + if (Options.PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(BlockDescriptions[BlockIndex].BlockHash)) { DownloadStats.RequestsCompleteCount++; continue; @@ -7741,9 +7749,8 @@ namespace { Work.ScheduleWork( NetworkPool, [&Storage, - &ZenFolderPath, + &Options, BuildId, - PrimeCacheOnly, &BlockDescriptions, &WritePartsComplete, TotalPartWriteCount, @@ -7804,7 +7811,7 @@ namespace { FilteredDownloadedBytesPerSecond.Stop(); } - if (!PrimeCacheOnly) + if (!Options.PrimeCacheOnly) { std::filesystem::path BlockChunkPath; @@ -7820,9 +7827,9 @@ namespace { if (!Ec) { BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = - ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + BlockBuffer = {}; + BlockChunkPath = ZenTempBlockFolderPath(Options.ZenFolderPath) / + BlockDescription.BlockHash.ToHexString(); RenameFile(TempBlobPath, BlockChunkPath, Ec); if (Ec) { @@ -7841,7 +7848,8 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); // Could not be moved and rather large, lets store it on disk - BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + BlockChunkPath = + ZenTempBlockFolderPath(Options.ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); BlockBuffer = {}; } @@ -7943,11 +7951,11 @@ namespace { (DownloadStats.RequestsCompleteCount == TotalRequestCount) ? "" : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8)); - std::string WriteDetails = PrimeCacheOnly ? "" - : fmt::format(" {}/{} ({}B/s) written.", - NiceBytes(DiskStats.WriteByteCount.load()), - NiceBytes(BytesToWrite), - NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); + std::string WriteDetails = Options.PrimeCacheOnly ? "" + : fmt::format(" {}/{} ({}B/s) written.", + NiceBytes(DiskStats.WriteByteCount.load()), + NiceBytes(BytesToWrite), + NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}", DownloadStats.RequestsCompleteCount.load(), TotalRequestCount, @@ -7955,11 +7963,11 @@ namespace { DownloadRateString, WriteDetails); WriteProgressBar.UpdateState( - {.Task = PrimeCacheOnly ? "Downloading " : "Writing chunks ", + {.Task = Options.PrimeCacheOnly ? "Downloading " : "Writing chunks ", .Details = Details, - .TotalCount = PrimeCacheOnly ? TotalRequestCount : BytesToWrite, - .RemainingCount = PrimeCacheOnly ? (TotalRequestCount - DownloadStats.RequestsCompleteCount.load()) - : (BytesToWrite - DiskStats.WriteByteCount.load()), + .TotalCount = Options.PrimeCacheOnly ? TotalRequestCount : BytesToWrite, + .RemainingCount = Options.PrimeCacheOnly ? (TotalRequestCount - DownloadStats.RequestsCompleteCount.load()) + : (BytesToWrite - DiskStats.WriteByteCount.load()), .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }); @@ -7974,7 +7982,7 @@ namespace { return; } - if (!PrimeCacheOnly) + if (!Options.PrimeCacheOnly) { uint32_t RawSequencesMissingWriteCount = 0; for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++) @@ -8019,7 +8027,7 @@ namespace { WriteChunkStats.WriteTimeUs = FilteredWrittenBytesPerSecond.GetElapsedTimeUS(); } - if (PrimeCacheOnly) + if (Options.PrimeCacheOnly) { return; } @@ -8066,7 +8074,7 @@ namespace { ZEN_ASSERT_SLOW(IsFile((Path / LocalContent.Paths[LocalPathIndex]).make_preferred())); - if (!WipeTargetFolder) + if (!Options.WipeTargetFolder) { if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string()); RemotePathIt != RemotePathToRemoteIndex.end()) @@ -8105,7 +8113,7 @@ namespace { SkippedCount++; } } - else if (!WipeTargetFolder) + else if (!Options.WipeTargetFolder) { // We don't need it RemoveLocalPathIndexes.push_back(LocalPathIndex); @@ -8188,7 +8196,7 @@ namespace { } } - if (WipeTargetFolder) + if (Options.WipeTargetFolder) { ZEN_TRACE_CPU("UpdateFolder_WipeTarget"); Stopwatch Timer; @@ -9386,21 +9394,27 @@ namespace { return Result; }; + struct DownloadOptions + { + std::filesystem::path SystemRootDir; + std::filesystem::path ZenFolderPath; + bool AllowMultiparts = true; + bool AllowPartialBlockRequests = true; + bool CleanTargetFolder = false; + bool PostDownloadVerify = false; + bool PrimeCacheOnly = false; + bool EnableOtherDownloadsScavenging = true; + bool EnableTargetFolderScavenging = true; + std::string IncludeWildcard; + std::string ExcludeWildcard; + }; + void DownloadFolder(StorageInstance& Storage, const Oid& BuildId, const std::vector<Oid>& BuildPartIds, std::span<const std::string> BuildPartNames, const std::filesystem::path& Path, - const std::filesystem::path& ZenFolderPath, - const std::filesystem::path SystemRootDir, - bool AllowMultiparts, - bool AllowPartialBlockRequests, - bool WipeTargetFolder, - bool PostDownloadVerify, - bool PrimeCacheOnly, - bool EnableScavenging, - std::string_view IncludeWildcard, - std::string_view ExcludeWildcard) + const DownloadOptions& Options) { ZEN_TRACE_CPU("DownloadFolder"); @@ -9419,18 +9433,18 @@ namespace { auto EndProgress = MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); - ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests))); + ZEN_ASSERT((!Options.PrimeCacheOnly) || (Options.PrimeCacheOnly && (!Options.AllowPartialBlockRequests))); Stopwatch DownloadTimer; ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount); - const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath); + const std::filesystem::path ZenTempFolder = ZenTempFolderPath(Options.ZenFolderPath); CreateDirectories(ZenTempFolder); - CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath)); - CreateDirectories(ZenTempCacheFolderPath(ZenFolderPath)); - CreateDirectories(ZenTempDownloadFolderPath(ZenFolderPath)); + CreateDirectories(ZenTempBlockFolderPath(Options.ZenFolderPath)); + CreateDirectories(ZenTempCacheFolderPath(Options.ZenFolderPath)); + CreateDirectories(ZenTempDownloadFolderPath(Options.ZenFolderPath)); std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; @@ -9449,19 +9463,19 @@ namespace { ChunkedFolderContent RemoteContent = GetRemoteContent(Storage, BuildId, AllBuildParts, - IncludeWildcard, - ExcludeWildcard, + Options.IncludeWildcard, + Options.ExcludeWildcard, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes); - const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; + const std::uint64_t LargeAttachmentSize = Options.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; GetFolderContentStatistics LocalFolderScanStats; ChunkingStatistics ChunkingStats; ChunkedFolderContent LocalContent; FolderContent LocalFolderContent; - if (!PrimeCacheOnly) + if (!Options.PrimeCacheOnly) { if (IsDir(Path)) { @@ -9474,11 +9488,11 @@ namespace { LocalContent = GetLocalContent(LocalFolderScanStats, ChunkingStats, Path, - ZenStateFilePath(ZenFolderPath), + ZenStateFilePath(Options.ZenFolderPath), *ChunkController, RemoteContent.Paths, - IncludeWildcard, - ExcludeWildcard, + Options.IncludeWildcard, + Options.ExcludeWildcard, LocalFolderContent); } else @@ -9532,7 +9546,7 @@ namespace { return true; }; - if (CompareContent(RemoteContent, LocalContent) && !WipeTargetFolder) + if (Options.EnableTargetFolderScavenging && !Options.CleanTargetFolder && CompareContent(RemoteContent, LocalContent)) { if (!IsQuiet) { @@ -9542,14 +9556,14 @@ namespace { Stopwatch WriteStateTimer; CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent, Path); - CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); - TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); + CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path()); + TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView()); if (!IsQuiet) { ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); } - AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path); + AddDownloadedPath(Options.SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(Options.ZenFolderPath), Path); } else { @@ -9576,21 +9590,22 @@ namespace { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount); - UpdateFolder(SystemRootDir, - Storage, + UpdateFolder(Storage, BuildId, Path, - ZenFolderPath, - LargeAttachmentSize, - PreferredMultipartChunkSize, LocalContent, RemoteContent, BlockDescriptions, LooseChunkHashes, - AllowPartialBlockRequests, - WipeTargetFolder, - PrimeCacheOnly, - EnableScavenging, + UpdateOptions{.SystemRootDir = Options.SystemRootDir, + .ZenFolderPath = Options.ZenFolderPath, + .LargeAttachmentSize = LargeAttachmentSize, + .PreferredMultipartChunkSize = PreferredMultipartChunkSize, + .AllowPartialBlockRequests = Options.AllowPartialBlockRequests, + .WipeTargetFolder = Options.CleanTargetFolder, + .PrimeCacheOnly = Options.PrimeCacheOnly, + .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging, + .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging}, LocalFolderState, DiskStats, CacheMappingStats, @@ -9600,28 +9615,28 @@ namespace { if (!AbortFlag) { - if (!PrimeCacheOnly) + if (!Options.PrimeCacheOnly) { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount); - VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats); + VerifyFolder(RemoteContent, Path, Options.PostDownloadVerify, VerifyFolderStats); Stopwatch WriteStateTimer; CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState, Path); - CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); - TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); + CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path()); + TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView()); if (!IsQuiet) { ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); } - AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path); + AddDownloadedPath(Options.SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(Options.ZenFolderPath), Path); #if 0 ExtendableStringBuilder<1024> SB; CompactBinaryToJson(StateObject, SB); - WriteFile(ZenStateFileJsonPath(ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); + WriteFile(ZenStateFileJsonPath(Options.ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); #endif // 0 } const uint64_t DownloadCount = DownloadStats.DownloadedChunkCount.load() + DownloadStats.DownloadedBlockCount.load() + @@ -9660,7 +9675,7 @@ namespace { } } } - if (PrimeCacheOnly) + if (Options.PrimeCacheOnly) { if (Storage.BuildCacheStorage) { @@ -10299,8 +10314,18 @@ BuildsCommand::BuildsCommand() "all parts will be downloaded", cxxopts::value(m_BuildPartNames), "<name>"); - m_DownloadOptions - .add_option("", "", "clean", "Delete all data in target folder before downloading", cxxopts::value(m_Clean), "<clean>"); + m_DownloadOptions.add_option("", + "", + "clean", + "Delete all data in target folder that is not part of the downloaded content", + cxxopts::value(m_Clean), + "<clean>"); + m_DownloadOptions.add_option("", + "", + "force", + "Force download of all content by ignoring any existing local content", + cxxopts::value(m_Force), + "<force>"); m_DownloadOptions.add_option("", "", "allow-multipart", @@ -11405,7 +11430,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_Clean, m_PostUploadVerify); - if (false) + if (true) { if (!IsQuiet) { @@ -11470,33 +11495,38 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_Clean && m_PrimeCacheOnly) { - ZEN_CONSOLE_WARN("Ignoring 'clean' option when 'cache-prime-only' is enabled"); + ZEN_CONSOLE_WARN("Ignoring '--clean' option when '--cache-prime-only' is enabled"); + } + + if (m_Force && m_PrimeCacheOnly) + { + ZEN_CONSOLE_WARN("Ignoring '--force' option when '--cache-prime-only' is enabled"); } if (m_AllowPartialBlockRequests && m_PrimeCacheOnly) { - ZEN_CONSOLE_WARN("Ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled"); + ZEN_CONSOLE_WARN("Ignoring '--allow-partial-block-requests' option when '--cache-prime-only' is enabled"); } std::vector<Oid> BuildPartIds = ParseBuildPartIds(); std::vector<std::string> BuildPartNames = ParseBuildPartNames(); - // TODO: Add file filters DownloadFolder(Storage, BuildId, BuildPartIds, BuildPartNames, m_Path, - m_ZenFolderPath, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests && !m_PrimeCacheOnly, - m_Clean, - m_PostDownloadVerify, - m_PrimeCacheOnly, - m_EnableScavenging, - m_IncludeWildcard, - m_ExcludeWildcard); + DownloadOptions{.SystemRootDir = m_SystemRootDir, + .ZenFolderPath = m_ZenFolderPath, + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests && !m_PrimeCacheOnly, + .CleanTargetFolder = m_Clean, + .PostDownloadVerify = m_PostDownloadVerify, + .PrimeCacheOnly = m_PrimeCacheOnly, + .EnableOtherDownloadsScavenging = m_EnableScavenging && !m_Force, + .EnableTargetFolderScavenging = !m_Force, + .IncludeWildcard = m_IncludeWildcard, + .ExcludeWildcard = m_ExcludeWildcard}); if (AbortFlag) { @@ -11698,16 +11728,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {}, {}, m_Path, - m_ZenFolderPath, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - BuildIdString == m_BuildIds.front(), - true, - false, - m_EnableScavenging, - ""sv, - ""sv); + DownloadOptions{.SystemRootDir = m_SystemRootDir, + .ZenFolderPath = m_ZenFolderPath, + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .CleanTargetFolder = BuildIdString == m_BuildIds.front(), + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = false}); if (AbortFlag) { throw std::runtime_error("Multitest aborted"); @@ -11870,16 +11899,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - true, - true, - false, - m_EnableScavenging, - ""sv, - ""sv); + + DownloadOptions{.SystemRootDir = m_SystemRootDir, + .ZenFolderPath = DownloadPath / ZenFolderName, + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .CleanTargetFolder = true, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = false}); if (AbortFlag) { throw std::runtime_error("Test aborted. (Download build)"); @@ -11891,16 +11920,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging, - ""sv, - ""sv); + + DownloadOptions{.SystemRootDir = m_SystemRootDir, + .ZenFolderPath = DownloadPath / ZenFolderName, + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .CleanTargetFolder = false, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = true}); if (AbortFlag) { throw std::runtime_error("Test aborted. (Re-download identical target)"); @@ -12004,16 +12033,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging, - ""sv, - ""sv); + + DownloadOptions{.SystemRootDir = m_SystemRootDir, + .ZenFolderPath = DownloadPath / ZenFolderName, + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .CleanTargetFolder = false, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = true}); if (AbortFlag) { throw std::runtime_error("Test aborted. (Re-download scrambled target)"); @@ -12056,16 +12085,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging, - ""sv, - ""sv); + + DownloadOptions{.SystemRootDir = m_SystemRootDir, + .ZenFolderPath = DownloadPath / ZenFolderName, + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .CleanTargetFolder = false, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = true}); if (AbortFlag) { throw std::runtime_error("Test aborted. (Download original)"); @@ -12077,16 +12106,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId2}, {}, DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging, - ""sv, - ""sv); + DownloadOptions{.SystemRootDir = m_SystemRootDir, + .ZenFolderPath = DownloadPath / ZenFolderName, + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .CleanTargetFolder = false, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = true}); if (AbortFlag) { throw std::runtime_error("Test aborted. (Download scrambled)"); @@ -12098,16 +12126,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId2}, {}, DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging, - ""sv, - ""sv); + DownloadOptions{.SystemRootDir = m_SystemRootDir, + .ZenFolderPath = DownloadPath / ZenFolderName, + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .CleanTargetFolder = false, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = true}); if (AbortFlag) { throw std::runtime_error("Test aborted. (Re-download scrambled)"); @@ -12119,16 +12146,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath2, - DownloadPath2 / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging, - ""sv, - ""sv); + DownloadOptions{.SystemRootDir = m_SystemRootDir, + .ZenFolderPath = DownloadPath2 / ZenFolderName, + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .CleanTargetFolder = false, + .PostDownloadVerify = true, + .PrimeCacheOnly = false, + .EnableOtherDownloadsScavenging = m_EnableScavenging, + .EnableTargetFolderScavenging = true}); if (AbortFlag) { throw std::runtime_error("Test aborted. (Download original)"); diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 79d69b16c..26f804fa5 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -61,6 +61,7 @@ private: std::string m_BuildPartName; // Defaults to name of leaf folder in m_Path std::string m_BuildPartId; // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName bool m_Clean = false; + bool m_Force = false; uint8_t m_BlockReuseMinPercentLimit = 85; bool m_AllowMultiparts = true; bool m_AllowPartialBlockRequests = true; |