diff options
| -rw-r--r-- | src/zen/cmds/builds_cmd.cpp | 1698 |
1 files changed, 858 insertions, 840 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index d998ed3e8..a8ec3be52 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -838,6 +838,8 @@ namespace { uint64_t LocalChunkMatchingRemoteCount = 0; uint64_t LocalChunkMatchingRemoteByteCount = 0; + uint64_t LocalPathIndexesMatchingSequenceIndexCount = 0; + uint64_t LocalScanElapsedWallTimeUs = 0; uint32_t ScavengedPathsMatchingSequencesCount = 0; @@ -5435,58 +5437,101 @@ namespace { return Result; } - void UpdateFolder(const std::filesystem::path& SystemRootDir, - StorageInstance& Storage, - const Oid& BuildId, - const std::filesystem::path& Path, - const std::filesystem::path& ZenFolderPath, - const std::uint64_t LargeAttachmentSize, - const std::uint64_t PreferredMultipartChunkSize, - const ChunkedFolderContent& LocalContent, - const ChunkedFolderContent& RemoteContent, - const std::vector<ChunkBlockDescription>& BlockDescriptions, - const std::vector<IoHash>& LooseChunkHashes, - bool AllowPartialBlockRequests, - bool WipeTargetFolder, - bool PrimeCacheOnly, - bool EnableScavenging, - FolderContent& OutLocalFolderState, - DiskStatistics& DiskStats, - CacheMappingStatistics& CacheMappingStats, - DownloadStatistics& DownloadStats, - WriteChunkStatistics& WriteChunkStats, - RebuildFolderStateStatistics& RebuildFolderStateStats) + struct DownloadFolderContext + { + const std::filesystem::path SystemRootDir; + + const Oid BuildId; + std::span<const Oid> BuildPartIds; + std::span<const std::string> BuildPartNames; + + const std::filesystem::path Path; + const std::filesystem::path ZenFolderPath; + + const bool AllowMultiparts; + const bool AllowPartialBlockRequests; + const bool WipeTargetFolder; + const bool PrimeCacheOnly; + const bool EnableScavenging; + const bool PostDownloadVerify; + }; + + struct UpdateFolderContext + { + const DownloadFolderContext& Base; + + const uint64_t LargeAttachmentSize; + const uint64_t PreferredMultipartChunkSize; + + const ChunkedFolderContent& LocalContent; + const ChunkedFolderContent& RemoteContent; + + std::span<const ChunkBlockDescription> BlockDescriptions; + std::span<const IoHash> LooseChunkHashes; + }; + + struct UpdateFolderStats + { + DiskStatistics& DiskStats; + CacheMappingStatistics& CacheMappingStats; + DownloadStatistics& DownloadStats; + WriteChunkStatistics& WriteChunkStats; + RebuildFolderStateStatistics& RebuildFolderStateStats; + }; + + void UpdateFolder(StorageInstance& Storage, + const UpdateFolderContext& InContext, + FolderContent& OutLocalFolderState, + UpdateFolderStats& Stats) { ZEN_TRACE_CPU("UpdateFolder"); - ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests))); + ZEN_ASSERT((!InContext.Base.PrimeCacheOnly) || (InContext.Base.PrimeCacheOnly && (!InContext.Base.AllowPartialBlockRequests))); Stopwatch IndexTimer; - const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent); + struct Context + { + const ChunkedContentLookup LocalLookup; + const ChunkedContentLookup RemoteLookup; + const std::filesystem::path CacheFolderPath; - const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); + std::vector<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters; + std::vector<bool> RemoteChunkIndexNeedsCopyFromLocalFileFlags; + std::vector<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags; - ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); + uint64_t TotalRequestCount; + uint64_t TotalPartWriteCount; + std::atomic<uint64_t> WritePartsComplete; + }; - const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(ZenFolderPath); + Context Context = { + .LocalLookup = BuildChunkedContentLookup(InContext.LocalContent), + .RemoteLookup = BuildChunkedContentLookup(InContext.RemoteContent), + .CacheFolderPath = ZenTempCacheFolderPath(InContext.Base.ZenFolderPath), + .SequenceIndexChunksLeftToWriteCounters = + std::vector<std::atomic<uint32_t>>(InContext.RemoteContent.ChunkedContent.SequenceRawHashes.size()), + .RemoteChunkIndexNeedsCopyFromLocalFileFlags = std::vector<bool>(InContext.RemoteContent.ChunkedContent.ChunkHashes.size()), + .RemoteChunkIndexNeedsCopyFromSourceFlags = + std::vector<std::atomic<bool>>(InContext.RemoteContent.ChunkedContent.ChunkHashes.size()), + .TotalRequestCount = 0, + .TotalPartWriteCount = 0, + .WritePartsComplete = 0}; - Stopwatch CacheMappingTimer; + ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); - std::vector<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters(RemoteContent.ChunkedContent.SequenceRawHashes.size()); - std::vector<bool> RemoteChunkIndexNeedsCopyFromLocalFileFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); - std::vector<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); + Stopwatch CacheMappingTimer; tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedChunkHashesFound; tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedSequenceHashesFound; - if (!PrimeCacheOnly) + if (!InContext.Base.PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_CheckChunkCache"); Stopwatch CacheTimer; DirectoryContent CacheDirContent; - GetDirectoryContent(CacheFolderPath, + GetDirectoryContent(Context.CacheFolderPath, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, CacheDirContent); for (size_t Index = 0; Index < CacheDirContent.Files.size(); Index++) @@ -5494,34 +5539,34 @@ namespace { IoHash FileHash; if (IoHash::TryParse(CacheDirContent.Files[Index].filename().string(), FileHash)) { - if (auto ChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(FileHash); - ChunkIt != RemoteLookup.ChunkHashToChunkIndex.end()) + if (auto ChunkIt = Context.RemoteLookup.ChunkHashToChunkIndex.find(FileHash); + ChunkIt != Context.RemoteLookup.ChunkHashToChunkIndex.end()) { const uint32_t ChunkIndex = ChunkIt->second; - const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + const uint64_t ChunkSize = InContext.RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; if (ChunkSize == CacheDirContent.FileSizes[Index]) { CachedChunkHashesFound.insert({FileHash, ChunkIndex}); - CacheMappingStats.CacheChunkCount++; - CacheMappingStats.CacheChunkByteCount += ChunkSize; + Stats.CacheMappingStats.CacheChunkCount++; + Stats.CacheMappingStats.CacheChunkByteCount += ChunkSize; continue; } } - else if (auto SequenceIt = RemoteLookup.RawHashToSequenceIndex.find(FileHash); - SequenceIt != RemoteLookup.RawHashToSequenceIndex.end()) + else if (auto SequenceIt = Context.RemoteLookup.RawHashToSequenceIndex.find(FileHash); + SequenceIt != Context.RemoteLookup.RawHashToSequenceIndex.end()) { const uint32_t SequenceIndex = SequenceIt->second; - const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; - const uint64_t SequenceSize = RemoteContent.RawSizes[PathIndex]; + const uint32_t PathIndex = Context.RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + const uint64_t SequenceSize = InContext.RemoteContent.RawSizes[PathIndex]; if (SequenceSize == CacheDirContent.FileSizes[Index]) { CachedSequenceHashesFound.insert({FileHash, SequenceIndex}); - CacheMappingStats.CacheSequenceHashesCount++; - CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize; + Stats.CacheMappingStats.CacheSequenceHashesCount++; + Stats.CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize; const std::filesystem::path CacheFilePath = - GetFinalChunkedSequenceFileName(CacheFolderPath, - RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + GetFinalChunkedSequenceFileName(Context.CacheFolderPath, + InContext.RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); continue; @@ -5530,26 +5575,26 @@ namespace { } TryRemoveFile(CacheDirContent.Files[Index]); } - CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs(); + Stats.CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs(); } tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedBlocksFound; - if (!PrimeCacheOnly) + if (!InContext.Base.PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_CheckBlockCache"); Stopwatch CacheTimer; tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> AllBlockSizes; - AllBlockSizes.reserve(BlockDescriptions.size()); - for (uint32_t BlockIndex = 0; BlockIndex < BlockDescriptions.size(); BlockIndex++) + AllBlockSizes.reserve(InContext.BlockDescriptions.size()); + for (uint32_t BlockIndex = 0; BlockIndex < InContext.BlockDescriptions.size(); BlockIndex++) { - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex]; AllBlockSizes.insert({BlockDescription.BlockHash, BlockIndex}); } DirectoryContent BlockDirContent; - GetDirectoryContent(ZenTempBlockFolderPath(ZenFolderPath), + GetDirectoryContent(ZenTempBlockFolderPath(InContext.Base.ZenFolderPath), DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, BlockDirContent); CachedBlocksFound.reserve(BlockDirContent.Files.size()); @@ -5561,7 +5606,7 @@ namespace { if (auto BlockIt = AllBlockSizes.find(FileHash); BlockIt != AllBlockSizes.end()) { const uint32_t BlockIndex = BlockIt->second; - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex]; uint64_t BlockSize = CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize; for (uint64_t ChunkSize : BlockDescription.ChunkCompressedLengths) { @@ -5571,8 +5616,8 @@ namespace { if (BlockSize == BlockDirContent.FileSizes[Index]) { CachedBlocksFound.insert({FileHash, BlockIndex}); - CacheMappingStats.CacheBlockCount++; - CacheMappingStats.CacheBlocksByteCount += BlockSize; + Stats.CacheMappingStats.CacheBlockCount++; + Stats.CacheMappingStats.CacheBlocksByteCount += BlockSize; continue; } } @@ -5580,69 +5625,71 @@ namespace { TryRemoveFile(BlockDirContent.Files[Index]); } - CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs(); + Stats.CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs(); } - std::vector<uint32_t> LocalPathIndexesMatchingSequenceIndexes; tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> SequenceIndexesLeftToFindToRemoteIndex; - if (!PrimeCacheOnly) + if (!InContext.Base.PrimeCacheOnly) { // Pick up all whole files we can use from current local state ZEN_TRACE_CPU("UpdateFolder_GetLocalSequences"); Stopwatch LocalTimer; - for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); + for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < InContext.RemoteContent.ChunkedContent.SequenceRawHashes.size(); RemoteSequenceIndex++) { - const IoHash& RemoteSequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); - const uint64_t RemoteRawSize = RemoteContent.RawSizes[RemotePathIndex]; + const IoHash& RemoteSequenceRawHash = InContext.RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(Context.RemoteLookup, RemoteSequenceIndex); + const uint64_t RemoteRawSize = InContext.RemoteContent.RawSizes[RemotePathIndex]; if (auto CacheSequenceIt = CachedSequenceHashesFound.find(RemoteSequenceRawHash); CacheSequenceIt != CachedSequenceHashesFound.end()) { - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(Context.CacheFolderPath, RemoteSequenceRawHash); ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); ZEN_CONSOLE_VERBOSE("Found sequence {} at {} ({})", RemoteSequenceRawHash, CacheFilePath, NiceBytes(RemoteRawSize)); } else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash); CacheChunkIt != CachedChunkHashesFound.end()) { - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(Context.CacheFolderPath, RemoteSequenceRawHash); ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); ZEN_CONSOLE_VERBOSE("Found chunk {} at {} ({})", RemoteSequenceRawHash, CacheFilePath, NiceBytes(RemoteRawSize)); } - else if (auto It = LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash); - It != LocalLookup.RawHashToSequenceIndex.end()) + else if (auto It = Context.LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash); + It != Context.LocalLookup.RawHashToSequenceIndex.end()) { - const uint32_t LocalSequenceIndex = It->second; - const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(LocalLookup, LocalSequenceIndex); - const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + const uint32_t LocalSequenceIndex = It->second; + const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(Context.LocalLookup, LocalSequenceIndex); + const std::filesystem::path LocalFilePath = + (InContext.Base.Path / InContext.LocalContent.Paths[LocalPathIndex]).make_preferred(); ZEN_ASSERT_SLOW(IsFile(LocalFilePath)); - LocalPathIndexesMatchingSequenceIndexes.push_back(LocalPathIndex); - CacheMappingStats.LocalPathsMatchingSequencesCount++; - CacheMappingStats.LocalPathsMatchingSequencesByteCount += RemoteRawSize; + Stats.CacheMappingStats.LocalPathIndexesMatchingSequenceIndexCount++; + Stats.CacheMappingStats.LocalPathsMatchingSequencesCount++; + Stats.CacheMappingStats.LocalPathsMatchingSequencesByteCount += RemoteRawSize; ZEN_CONSOLE_VERBOSE("Found sequence {} at {} ({})", RemoteSequenceRawHash, LocalFilePath, NiceBytes(RemoteRawSize)); } else { // We must write the sequence - const uint32_t ChunkCount = RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; - SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount; + const uint32_t ChunkCount = InContext.RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; + Context.SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount; SequenceIndexesLeftToFindToRemoteIndex.insert({RemoteSequenceRawHash, RemoteSequenceIndex}); } } - CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs(); + Stats.CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs(); } else { - for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); + for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < InContext.RemoteContent.ChunkedContent.SequenceRawHashes.size(); RemoteSequenceIndex++) { - const uint32_t ChunkCount = RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; - SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount; + const uint32_t ChunkCount = InContext.RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; + Context.SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount; } } @@ -5661,7 +5708,7 @@ namespace { std::vector<ScavengeCopyOperation> ScavengeCopyOperations; uint64_t ScavengedPathsCount = 0; - if (!PrimeCacheOnly && EnableScavenging) + if (!InContext.Base.PrimeCacheOnly && InContext.Base.EnableScavenging) { ZEN_TRACE_CPU("UpdateFolder_GetScavengedSequences"); @@ -5669,10 +5716,10 @@ namespace { if (!SequenceIndexesLeftToFindToRemoteIndex.empty()) { - std::vector<ScavengeSource> ScavengeSources = GetDownloadedStatePaths(SystemRootDir); - auto EraseIt = std::remove_if(ScavengeSources.begin(), ScavengeSources.end(), [&Path](const ScavengeSource& Source) { - return Source.Path == Path; - }); + std::vector<ScavengeSource> ScavengeSources = GetDownloadedStatePaths(InContext.Base.SystemRootDir); + auto EraseIt = std::remove_if(ScavengeSources.begin(), + ScavengeSources.end(), + [Path = InContext.Base.Path](const ScavengeSource& Source) { return Source.Path == Path; }); ScavengeSources.erase(EraseIt, ScavengeSources.end()); const size_t ScavengePathCount = ScavengeSources.size(); @@ -5736,7 +5783,7 @@ namespace { { const uint32_t RemoteSequenceIndex = It->second; const uint64_t RawSize = - RemoteContent.RawSizes[RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]]; + InContext.RemoteContent.RawSizes[Context.RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]]; ZEN_ASSERT(RawSize > 0); const uint32_t ScavengedPathIndex = ScavengedLookup.SequenceIndexFirstPathIndex[ScavengedSequenceIndex]; @@ -5748,23 +5795,25 @@ namespace { .RawSize = RawSize}); SequenceIndexesLeftToFindToRemoteIndex.erase(SequenceRawHash); - SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = 0; + Context.SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = 0; - CacheMappingStats.ScavengedPathsMatchingSequencesCount++; - CacheMappingStats.ScavengedPathsMatchingSequencesByteCount += RawSize; + Stats.CacheMappingStats.ScavengedPathsMatchingSequencesCount++; + Stats.CacheMappingStats.ScavengedPathsMatchingSequencesByteCount += RawSize; } } ScavengedPathsCount++; } } } - CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs(); + Stats.CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs(); } uint32_t RemainingChunkCount = 0; - for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++) + for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < InContext.RemoteContent.ChunkedContent.ChunkHashes.size(); + RemoteChunkIndex++) { - uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); + uint64_t ChunkWriteCount = + GetChunkWriteCount(Context.SequenceIndexChunksLeftToWriteCounters, Context.RemoteLookup, RemoteChunkIndex); if (ChunkWriteCount > 0) { RemainingChunkCount++; @@ -5790,36 +5839,39 @@ namespace { tsl::robin_map<IoHash, size_t, IoHash::Hasher> RawHashToCacheCopyDataIndex; std::vector<CacheCopyData> CacheCopyDatas; - if (!PrimeCacheOnly) + if (!InContext.Base.PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_GetLocalChunks"); Stopwatch LocalTimer; for (uint32_t LocalSequenceIndex = 0; - LocalSequenceIndex < LocalContent.ChunkedContent.SequenceRawHashes.size() && (RemainingChunkCount > 0); + LocalSequenceIndex < InContext.LocalContent.ChunkedContent.SequenceRawHashes.size() && (RemainingChunkCount > 0); LocalSequenceIndex++) { - const IoHash& LocalSequenceRawHash = LocalContent.ChunkedContent.SequenceRawHashes[LocalSequenceIndex]; - const uint32_t LocalOrderOffset = LocalLookup.SequenceIndexChunkOrderOffset[LocalSequenceIndex]; + const IoHash& LocalSequenceRawHash = InContext.LocalContent.ChunkedContent.SequenceRawHashes[LocalSequenceIndex]; + const uint32_t LocalOrderOffset = Context.LocalLookup.SequenceIndexChunkOrderOffset[LocalSequenceIndex]; { uint64_t SourceOffset = 0; - const uint32_t LocalChunkCount = LocalContent.ChunkedContent.ChunkCounts[LocalSequenceIndex]; + const uint32_t LocalChunkCount = InContext.LocalContent.ChunkedContent.ChunkCounts[LocalSequenceIndex]; for (uint32_t LocalOrderIndex = 0; LocalOrderIndex < LocalChunkCount; LocalOrderIndex++) { - const uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[LocalOrderOffset + LocalOrderIndex]; - const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; - const uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex]; + const uint32_t LocalChunkIndex = + InContext.LocalContent.ChunkedContent.ChunkOrders[LocalOrderOffset + LocalOrderIndex]; + const IoHash& LocalChunkHash = InContext.LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; + const uint64_t LocalChunkRawSize = InContext.LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex]; - if (auto RemoteChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(LocalChunkHash); - RemoteChunkIt != RemoteLookup.ChunkHashToChunkIndex.end()) + if (auto RemoteChunkIt = Context.RemoteLookup.ChunkHashToChunkIndex.find(LocalChunkHash); + RemoteChunkIt != Context.RemoteLookup.ChunkHashToChunkIndex.end()) { const uint32_t RemoteChunkIndex = RemoteChunkIt->second; - if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) + if (!Context.RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) { std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = - GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); + GetRemainingChunkTargets(Context.SequenceIndexChunksLeftToWriteCounters, + Context.RemoteLookup, + RemoteChunkIndex); if (!ChunkTargetPtrs.empty()) { @@ -5857,9 +5909,9 @@ namespace { .TargetChunkLocationPtrs = ChunkTargetPtrs, .ChunkTargets = std::vector<CacheCopyData::ChunkTarget>{Target}}); } - CacheMappingStats.LocalChunkMatchingRemoteCount++; - CacheMappingStats.LocalChunkMatchingRemoteByteCount += LocalChunkRawSize; - RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true; + Stats.CacheMappingStats.LocalChunkMatchingRemoteCount++; + Stats.CacheMappingStats.LocalChunkMatchingRemoteByteCount += LocalChunkRawSize; + Context.RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true; RemainingChunkCount--; } } @@ -5868,10 +5920,10 @@ namespace { } } } - CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs(); + Stats.CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs(); } - if (!PrimeCacheOnly) + if (!InContext.Base.PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_GetScavengeChunks"); @@ -5901,14 +5953,16 @@ namespace { const IoHash& ScavengedChunkHash = ScavengedContent.ChunkedContent.ChunkHashes[ScavengedChunkIndex]; const uint64_t ScavengedChunkRawSize = ScavengedContent.ChunkedContent.ChunkRawSizes[ScavengedChunkIndex]; - if (auto RemoteChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(ScavengedChunkHash); - RemoteChunkIt != RemoteLookup.ChunkHashToChunkIndex.end()) + if (auto RemoteChunkIt = Context.RemoteLookup.ChunkHashToChunkIndex.find(ScavengedChunkHash); + RemoteChunkIt != Context.RemoteLookup.ChunkHashToChunkIndex.end()) { const uint32_t RemoteChunkIndex = RemoteChunkIt->second; - if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) + if (!Context.RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) { std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = - GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); + GetRemainingChunkTargets(Context.SequenceIndexChunksLeftToWriteCounters, + Context.RemoteLookup, + RemoteChunkIndex); if (!ChunkTargetPtrs.empty()) { @@ -5947,9 +6001,9 @@ namespace { .TargetChunkLocationPtrs = ChunkTargetPtrs, .ChunkTargets = std::vector<CacheCopyData::ChunkTarget>{Target}}); } - CacheMappingStats.ScavengedChunkMatchingRemoteCount++; - CacheMappingStats.ScavengedChunkMatchingRemoteByteCount += ScavengedChunkRawSize; - RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true; + Stats.CacheMappingStats.ScavengedChunkMatchingRemoteCount++; + Stats.CacheMappingStats.ScavengedChunkMatchingRemoteByteCount += ScavengedChunkRawSize; + Context.RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true; RemainingChunkCount--; } } @@ -5959,52 +6013,56 @@ namespace { } } } - CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs(); + Stats.CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs(); } if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty()) { ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}", CachedSequenceHashesFound.size(), - NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount), + NiceBytes(Stats.CacheMappingStats.CacheSequenceHashesByteCount), CachedChunkHashesFound.size(), - NiceBytes(CacheMappingStats.CacheChunkByteCount), + NiceBytes(Stats.CacheMappingStats.CacheChunkByteCount), CachedBlocksFound.size(), - NiceBytes(CacheMappingStats.CacheBlocksByteCount), - NiceTimeSpanMs(CacheMappingStats.CacheScanElapsedWallTimeUs / 1000)); + NiceBytes(Stats.CacheMappingStats.CacheBlocksByteCount), + NiceTimeSpanMs(Stats.CacheMappingStats.CacheScanElapsedWallTimeUs / 1000)); } - if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0) + if (Stats.CacheMappingStats.LocalPathIndexesMatchingSequenceIndexCount > 0 || + Stats.CacheMappingStats.LocalChunkMatchingRemoteCount > 0) { ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}", - LocalPathIndexesMatchingSequenceIndexes.size(), - NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount), - CacheMappingStats.LocalChunkMatchingRemoteCount, - NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount), - NiceTimeSpanMs(CacheMappingStats.LocalScanElapsedWallTimeUs / 1000)); + Stats.CacheMappingStats.LocalPathIndexesMatchingSequenceIndexCount, + NiceBytes(Stats.CacheMappingStats.LocalPathsMatchingSequencesByteCount), + Stats.CacheMappingStats.LocalChunkMatchingRemoteCount, + NiceBytes(Stats.CacheMappingStats.LocalChunkMatchingRemoteByteCount), + NiceTimeSpanMs(Stats.CacheMappingStats.LocalScanElapsedWallTimeUs / 1000)); } - if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0) + if (Stats.CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || + Stats.CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0) { ZEN_CONSOLE("Scavenge of {} paths found {} ({}) chunk sequences, {} ({}) chunks in {}", ScavengedPathsCount, - CacheMappingStats.ScavengedPathsMatchingSequencesCount, - NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount), - CacheMappingStats.ScavengedChunkMatchingRemoteCount, - NiceBytes(CacheMappingStats.ScavengedChunkMatchingRemoteByteCount), - NiceTimeSpanMs(CacheMappingStats.ScavengeElapsedWallTimeUs / 1000)); + Stats.CacheMappingStats.ScavengedPathsMatchingSequencesCount, + NiceBytes(Stats.CacheMappingStats.ScavengedPathsMatchingSequencesByteCount), + Stats.CacheMappingStats.ScavengedChunkMatchingRemoteCount, + NiceBytes(Stats.CacheMappingStats.ScavengedChunkMatchingRemoteByteCount), + NiceTimeSpanMs(Stats.CacheMappingStats.ScavengeElapsedWallTimeUs / 1000)); } uint64_t BytesToWrite = 0; - for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++) + for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < InContext.RemoteContent.ChunkedContent.ChunkHashes.size(); + RemoteChunkIndex++) { - uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); + uint64_t ChunkWriteCount = + GetChunkWriteCount(Context.SequenceIndexChunksLeftToWriteCounters, Context.RemoteLookup, RemoteChunkIndex); if (ChunkWriteCount > 0) { - BytesToWrite += RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] * ChunkWriteCount; - if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) + BytesToWrite += InContext.RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] * ChunkWriteCount; + if (!Context.RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) { - RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex] = true; + Context.RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex] = true; } } } @@ -6014,10 +6072,6 @@ namespace { BytesToWrite += ScavengeCopyOp.RawSize; } - uint64_t TotalRequestCount = 0; - uint64_t TotalPartWriteCount = 0; - std::atomic<uint64_t> WritePartsComplete = 0; - { ZEN_TRACE_CPU("WriteChunks"); @@ -6029,7 +6083,7 @@ namespace { WorkerThreadPool& NetworkPool = GetNetworkPool(); WorkerThreadPool& WritePool = GetIOWorkerPool(); - ProgressBar WriteProgressBar(ProgressMode, PrimeCacheOnly ? "Downloading" : "Writing"); + ProgressBar WriteProgressBar(ProgressMode, InContext.Base.PrimeCacheOnly ? "Downloading" : "Writing"); ParallelWork Work(AbortFlag); struct LooseChunkHashWorkData @@ -6039,24 +6093,24 @@ namespace { }; std::vector<LooseChunkHashWorkData> LooseChunkHashWorks; - TotalPartWriteCount += CacheCopyDatas.size(); - TotalPartWriteCount += ScavengeCopyOperations.size(); + Context.TotalPartWriteCount += CacheCopyDatas.size(); + Context.TotalPartWriteCount += ScavengeCopyOperations.size(); - for (const IoHash ChunkHash : LooseChunkHashes) + for (const IoHash ChunkHash : InContext.LooseChunkHashes) { - auto RemoteChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); - ZEN_ASSERT(RemoteChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end()); + auto RemoteChunkIndexIt = Context.RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); + ZEN_ASSERT(RemoteChunkIndexIt != Context.RemoteLookup.ChunkHashToChunkIndex.end()); const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second; - if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) + if (Context.RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) { ZEN_CONSOLE_VERBOSE("Skipping chunk {} due to cache reuse", ChunkHash); continue; } bool NeedsCopy = true; - if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false)) + if (Context.RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false)) { std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = - GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); + GetRemainingChunkTargets(Context.SequenceIndexChunksLeftToWriteCounters, Context.RemoteLookup, RemoteChunkIndex); if (ChunkTargetPtrs.empty()) { @@ -6064,41 +6118,40 @@ namespace { } else { - TotalRequestCount++; - TotalPartWriteCount++; + Context.TotalRequestCount++; + Context.TotalPartWriteCount++; LooseChunkHashWorks.push_back( LooseChunkHashWorkData{.ChunkTargetPtrs = ChunkTargetPtrs, .RemoteChunkIndex = RemoteChunkIndex}); } } } - uint32_t BlockCount = gsl::narrow<uint32_t>(BlockDescriptions.size()); - - std::vector<bool> ChunkIsPickedUpByBlock(RemoteContent.ChunkedContent.ChunkHashes.size(), false); - auto GetNeededChunkBlockIndexes = [&RemoteContent, - &RemoteLookup, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &ChunkIsPickedUpByBlock](const ChunkBlockDescription& BlockDescription) { - ZEN_TRACE_CPU("UpdateFolder_GetNeededChunkBlockIndexes"); - std::vector<uint32_t> NeededBlockChunkIndexes; - for (uint32_t ChunkBlockIndex = 0; ChunkBlockIndex < BlockDescription.ChunkRawHashes.size(); ChunkBlockIndex++) - { - const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex]; - if (auto It = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != RemoteLookup.ChunkHashToChunkIndex.end()) - { - const uint32_t RemoteChunkIndex = It->second; - if (!ChunkIsPickedUpByBlock[RemoteChunkIndex]) - { - if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]) - { - ChunkIsPickedUpByBlock[RemoteChunkIndex] = true; - NeededBlockChunkIndexes.push_back(ChunkBlockIndex); - } - } - } - } - return NeededBlockChunkIndexes; - }; + uint32_t BlockCount = gsl::narrow<uint32_t>(InContext.BlockDescriptions.size()); + + std::vector<bool> ChunkIsPickedUpByBlock(InContext.RemoteContent.ChunkedContent.ChunkHashes.size(), false); + auto GetNeededChunkBlockIndexes = + [&InContext, &Context, &ChunkIsPickedUpByBlock](const ChunkBlockDescription& BlockDescription) { + ZEN_TRACE_CPU("UpdateFolder_GetNeededChunkBlockIndexes"); + std::vector<uint32_t> NeededBlockChunkIndexes; + for (uint32_t ChunkBlockIndex = 0; ChunkBlockIndex < BlockDescription.ChunkRawHashes.size(); ChunkBlockIndex++) + { + const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex]; + if (auto It = Context.RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); + It != Context.RemoteLookup.ChunkHashToChunkIndex.end()) + { + const uint32_t RemoteChunkIndex = It->second; + if (!ChunkIsPickedUpByBlock[RemoteChunkIndex]) + { + if (Context.RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]) + { + ChunkIsPickedUpByBlock[RemoteChunkIndex] = true; + NeededBlockChunkIndexes.push_back(ChunkBlockIndex); + } + } + } + } + return NeededBlockChunkIndexes; + }; std::vector<uint32_t> CachedChunkBlockIndexes; @@ -6116,14 +6169,14 @@ namespace { for (uint32_t BlockIndex = 0; BlockIndex < BlockCount; BlockIndex++) { - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex]; const std::vector<uint32_t> BlockChunkIndexNeeded = GetNeededChunkBlockIndexes(BlockDescription); if (!BlockChunkIndexNeeded.empty()) { - if (PrimeCacheOnly) + if (InContext.Base.PrimeCacheOnly) { - TotalRequestCount++; - TotalPartWriteCount++; + Context.TotalRequestCount++; + Context.TotalPartWriteCount++; FullBlockWorks.push_back(BlockIndex); } @@ -6134,10 +6187,10 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_CacheGet"); - TotalPartWriteCount++; + Context.TotalPartWriteCount++; std::filesystem::path BlockPath = - ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + ZenTempBlockFolderPath(InContext.Base.ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); if (IsFile(BlockPath)) { CachedChunkBlockIndexes.push_back(BlockIndex); @@ -6151,7 +6204,7 @@ namespace { bool CanDoPartialBlockDownload = (BlockDescription.HeaderSize > 0) && (BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size()); - if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) + if (InContext.Base.AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) { std::vector<BlockRangeDescriptor> BlockRanges; @@ -6239,8 +6292,8 @@ namespace { NiceBytes(WantedSize), BlockDescription.BlockHash, NiceBytes(TotalBlockSize)); - TotalRequestCount++; - TotalPartWriteCount++; + Context.TotalRequestCount++; + Context.TotalPartWriteCount++; FullBlockWorks.push_back(BlockIndex); } @@ -6252,8 +6305,8 @@ namespace { BlockDescription.BlockHash, NiceBytes(TotalBlockSize), CollapsedBlockRanges.size()); - TotalRequestCount++; - TotalPartWriteCount++; + Context.TotalRequestCount++; + Context.TotalPartWriteCount++; FullBlockWorks.push_back(BlockIndex); } @@ -6265,8 +6318,8 @@ namespace { BlockDescription.BlockHash, NiceBytes(TotalBlockSize), CollapsedBlockRanges.size()); - TotalRequestCount++; - TotalPartWriteCount++; + Context.TotalRequestCount++; + Context.TotalPartWriteCount++; FullBlockWorks.push_back(BlockIndex); } @@ -6278,8 +6331,8 @@ namespace { BlockDescription.BlockHash, NiceBytes(TotalBlockSize), CollapsedBlockRanges.size()); - TotalRequestCount++; - TotalPartWriteCount++; + Context.TotalRequestCount++; + Context.TotalPartWriteCount++; FullBlockWorks.push_back(BlockIndex); } @@ -6291,8 +6344,8 @@ namespace { BlockDescription.BlockHash, NiceBytes(TotalBlockSize), CollapsedBlockRanges.size()); - TotalRequestCount++; - TotalPartWriteCount++; + Context.TotalRequestCount++; + Context.TotalPartWriteCount++; FullBlockWorks.push_back(BlockIndex); } @@ -6307,16 +6360,16 @@ namespace { NiceBytes(TotalBlockSize), CollapsedBlockRanges.size()); } - TotalRequestCount += CollapsedBlockRanges.size(); - TotalPartWriteCount += CollapsedBlockRanges.size(); + Context.TotalRequestCount += CollapsedBlockRanges.size(); + Context.TotalPartWriteCount += CollapsedBlockRanges.size(); BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end()); } } else { - TotalRequestCount++; - TotalPartWriteCount++; + Context.TotalRequestCount++; + Context.TotalPartWriteCount++; FullBlockWorks.push_back(BlockIndex); } @@ -6325,7 +6378,7 @@ namespace { } else { - ZEN_CONSOLE_VERBOSE("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash); + ZEN_CONSOLE_VERBOSE("Skipping block {} due to cache reuse", InContext.BlockDescriptions[BlockIndex].BlockHash); } } @@ -6347,17 +6400,17 @@ namespace { BlobHashesSet.reserve(LooseChunkHashWorks.size() + FullBlockWorks.size()); for (LooseChunkHashWorkData& LooseChunkHashWork : LooseChunkHashWorks) { - BlobHashesSet.insert(RemoteContent.ChunkedContent.ChunkHashes[LooseChunkHashWork.RemoteChunkIndex]); + BlobHashesSet.insert(InContext.RemoteContent.ChunkedContent.ChunkHashes[LooseChunkHashWork.RemoteChunkIndex]); } for (const BlockRangeDescriptor& BlockRange : BlockRangeWorks) { const uint32_t BlockIndex = BlockRange.BlockIndex; - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex]; BlobHashesSet.insert(BlockDescription.BlockHash); } for (uint32_t BlockIndex : FullBlockWorks) { - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex]; BlobHashesSet.insert(BlockDescription.BlockHash); } @@ -6365,7 +6418,7 @@ namespace { { const std::vector<IoHash> BlobHashes(BlobHashesSet.begin(), BlobHashesSet.end()); const std::vector<BuildStorageCache::BlobExistsResult> CacheExistsResult = - Storage.BuildCacheStorage->BlobsExists(BuildId, BlobHashes); + Storage.BuildCacheStorage->BlobsExists(InContext.Base.BuildId, BlobHashes); if (CacheExistsResult.size() == BlobHashes.size()) { @@ -6395,20 +6448,18 @@ namespace { { break; } - if (!PrimeCacheOnly) + if (!InContext.Base.PrimeCacheOnly) { Work.ScheduleWork( WritePool, - [&RemoteContent, - &CacheFolderPath, + [&InContext, + &Context, &ScavengedPaths, &ScavengeCopyOperations, &ScavengedContents, &FilteredWrittenBytesPerSecond, ScavengeOpIndex, - &WritePartsComplete, - TotalPartWriteCount, - &DiskStats](std::atomic<bool>&) mutable { + &Stats](std::atomic<bool>&) mutable { if (!AbortFlag) { ZEN_TRACE_CPU("UpdateFolder_WriteScavanged"); @@ -6424,19 +6475,23 @@ namespace { ZEN_ASSERT_SLOW(FileSizeFromPath(ScavengedFilePath) == ScavengeOp.RawSize); const IoHash& RemoteSequenceRawHash = - RemoteContent.ChunkedContent.SequenceRawHashes[ScavengeOp.RemoteSequenceIndex]; + InContext.RemoteContent.ChunkedContent.SequenceRawHashes[ScavengeOp.RemoteSequenceIndex]; const std::filesystem::path TempFilePath = - GetTempChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + GetTempChunkedSequenceFileName(Context.CacheFolderPath, RemoteSequenceRawHash); const uint64_t RawSize = ScavengedContent.RawSizes[ScavengeOp.ScavengedContentIndex]; - CopyFile(ScavengedFilePath, TempFilePath, RawSize, DiskStats.WriteCount, DiskStats.WriteByteCount); + CopyFile(ScavengedFilePath, + TempFilePath, + RawSize, + Stats.DiskStats.WriteCount, + Stats.DiskStats.WriteByteCount); const std::filesystem::path CacheFilePath = - GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + GetFinalChunkedSequenceFileName(Context.CacheFolderPath, RemoteSequenceRawHash); RenameFile(TempFilePath, CacheFilePath); - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) + Context.WritePartsComplete++; + if (Context.WritePartsComplete == Context.TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); } @@ -6458,48 +6513,36 @@ namespace { std::move(LooseChunkHashWork.ChunkTargetPtrs); const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex; - if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex])) + if (InContext.Base.PrimeCacheOnly && + ExistsResult.ExistingBlobs.contains(InContext.RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex])) { - DownloadStats.RequestsCompleteCount++; + Stats.DownloadStats.RequestsCompleteCount++; continue; } Work.ScheduleWork( WritePool, [&Storage, - &Path, - &ZenFolderPath, - &RemoteContent, - &RemoteLookup, - &CacheFolderPath, - &SequenceIndexChunksLeftToWriteCounters, + &InContext, + &Context, &Work, &WritePool, &NetworkPool, - PrimeCacheOnly, &ExistsResult, - &DiskStats, - &DownloadStats, - &WriteChunkStats, - &WritePartsComplete, + &Stats, RemoteChunkIndex, ChunkTargetPtrs, - BuildId = Oid(BuildId), - LargeAttachmentSize, - PreferredMultipartChunkSize, - TotalRequestCount, - TotalPartWriteCount, &FilteredDownloadedBytesPerSecond, &FilteredWrittenBytesPerSecond](std::atomic<bool>&) mutable { if (!AbortFlag) { ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded"); std::filesystem::path ExistingCompressedChunkPath; - if (!PrimeCacheOnly) + if (!InContext.Base.PrimeCacheOnly) { - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + const IoHash& ChunkHash = InContext.RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; std::filesystem::path CompressedChunkPath = - ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); + ZenTempDownloadFolderPath(InContext.Base.ZenFolderPath) / ChunkHash.ToHexString(); if (IsFile(CompressedChunkPath)) { IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath); @@ -6509,8 +6552,8 @@ namespace { uint64_t RawSize; if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart, RawHash, RawSize)) { - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + Stats.DownloadStats.RequestsCompleteCount++; + if (Stats.DownloadStats.RequestsCompleteCount == Context.TotalRequestCount) { FilteredDownloadedBytesPerSecond.Stop(); } @@ -6531,18 +6574,11 @@ namespace { { Work.ScheduleWork( WritePool, - [&Path, - &ZenFolderPath, - &RemoteContent, - &RemoteLookup, - &CacheFolderPath, - &SequenceIndexChunksLeftToWriteCounters, + [&InContext, + &Context, &Work, &WritePool, - &DiskStats, - &WriteChunkStats, - &WritePartsComplete, - TotalPartWriteCount, + &Stats, &FilteredWrittenBytesPerSecond, RemoteChunkIndex, ChunkTargetPtrs, @@ -6553,7 +6589,8 @@ namespace { FilteredWrittenBytesPerSecond.Start(); - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + const IoHash& ChunkHash = + InContext.RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); if (!CompressedPart) @@ -6564,19 +6601,18 @@ namespace { CompressedChunkPath)); } - std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath); - bool NeedHashVerify = WriteCompressedChunk(TargetFolder, - RemoteContent, - RemoteLookup, - ChunkHash, - ChunkTargetPtrs, - std::move(CompressedPart), - DiskStats); - WritePartsComplete++; + bool NeedHashVerify = WriteCompressedChunk(Context.CacheFolderPath, + InContext.RemoteContent, + Context.RemoteLookup, + ChunkHash, + ChunkTargetPtrs, + std::move(CompressedPart), + Stats.DiskStats); + Context.WritePartsComplete++; if (!AbortFlag) { - if (WritePartsComplete == TotalPartWriteCount) + if (Context.WritePartsComplete == Context.TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); } @@ -6584,19 +6620,22 @@ namespace { TryRemoveFile(CompressedChunkPath); std::vector<uint32_t> CompletedSequences = - CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); + CompleteChunkTargets(ChunkTargetPtrs, + Context.SequenceIndexChunksLeftToWriteCounters); if (NeedHashVerify) { - VerifyAndCompleteChunkSequencesAsync(TargetFolder, - RemoteContent, - RemoteLookup, + VerifyAndCompleteChunkSequencesAsync(Context.CacheFolderPath, + InContext.RemoteContent, + Context.RemoteLookup, CompletedSequences, Work, WritePool); } else { - FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences); + FinalizeChunkSequences(Context.CacheFolderPath, + InContext.RemoteContent, + CompletedSequences); } } } @@ -6606,140 +6645,123 @@ namespace { { Work.ScheduleWork( NetworkPool, - [&Path, - &ZenFolderPath, - &Storage, - BuildId = Oid(BuildId), - &PrimeCacheOnly, - &RemoteContent, - &RemoteLookup, + [&Storage, + &InContext, + &Context, &ExistsResult, - &SequenceIndexChunksLeftToWriteCounters, &Work, &WritePool, &NetworkPool, - &DiskStats, - &WriteChunkStats, - &WritePartsComplete, - TotalPartWriteCount, - TotalRequestCount, + &Stats, &FilteredDownloadedBytesPerSecond, &FilteredWrittenBytesPerSecond, - LargeAttachmentSize, - PreferredMultipartChunkSize, RemoteChunkIndex, - ChunkTargetPtrs, - &DownloadStats](std::atomic<bool>&) mutable { + ChunkTargetPtrs](std::atomic<bool>&) mutable { if (!AbortFlag) { - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + const IoHash& ChunkHash = + InContext.RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; FilteredDownloadedBytesPerSecond.Start(); IoBuffer BuildBlob; const bool ExistsInCache = Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash); if (ExistsInCache) { - BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash); + BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(InContext.Base.BuildId, ChunkHash); } if (BuildBlob) { uint64_t BlobSize = BuildBlob.GetSize(); - DownloadStats.DownloadedChunkCount++; - DownloadStats.DownloadedChunkByteCount += BlobSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + Stats.DownloadStats.DownloadedChunkCount++; + Stats.DownloadStats.DownloadedChunkByteCount += BlobSize; + Stats.DownloadStats.RequestsCompleteCount++; + if (Stats.DownloadStats.RequestsCompleteCount == Context.TotalRequestCount) { FilteredDownloadedBytesPerSecond.Stop(); } - AsyncWriteDownloadedChunk(ZenFolderPath, - RemoteContent, - RemoteLookup, + AsyncWriteDownloadedChunk(InContext.Base.ZenFolderPath, + InContext.RemoteContent, + Context.RemoteLookup, RemoteChunkIndex, std::move(ChunkTargetPtrs), Work, WritePool, std::move(BuildBlob), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, + Context.SequenceIndexChunksLeftToWriteCounters, + Context.WritePartsComplete, + Context.TotalPartWriteCount, FilteredWrittenBytesPerSecond, - DiskStats); + Stats.DiskStats); } else { - if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) + if (InContext.RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= + InContext.LargeAttachmentSize) { ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); - DownloadLargeBlob(*Storage.BuildStorage, - ZenTempDownloadFolderPath(ZenFolderPath), - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - Work, - NetworkPool, - DownloadStats, - [&Storage, - &ZenFolderPath, - &RemoteContent, - &RemoteLookup, - BuildId, - PrimeCacheOnly, - &SequenceIndexChunksLeftToWriteCounters, - &Work, - &WritePool, - ChunkHash, - TotalPartWriteCount, - TotalRequestCount, - &WritePartsComplete, - &FilteredWrittenBytesPerSecond, - &FilteredDownloadedBytesPerSecond, - &DownloadStats, - &DiskStats, - RemoteChunkIndex, - ChunkTargetPtrs](IoBuffer&& Payload) mutable { - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - if (Payload && Storage.BuildCacheStorage) - { - Storage.BuildCacheStorage->PutBuildBlob( - BuildId, - ChunkHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(SharedBuffer(Payload))); - } - if (!PrimeCacheOnly) - { - if (!AbortFlag) - { - AsyncWriteDownloadedChunk( - ZenFolderPath, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(Payload), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats); - } - } - }); + DownloadLargeBlob( + *Storage.BuildStorage, + ZenTempDownloadFolderPath(InContext.Base.ZenFolderPath), + InContext.Base.BuildId, + ChunkHash, + InContext.PreferredMultipartChunkSize, + Work, + NetworkPool, + Stats.DownloadStats, + [&Storage, + &InContext, + &Context, + &Work, + &WritePool, + ChunkHash, + &FilteredWrittenBytesPerSecond, + &FilteredDownloadedBytesPerSecond, + &Stats, + RemoteChunkIndex, + ChunkTargetPtrs](IoBuffer&& Payload) mutable { + Stats.DownloadStats.RequestsCompleteCount++; + if (Stats.DownloadStats.RequestsCompleteCount == Context.TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + if (Payload && Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob( + InContext.Base.BuildId, + ChunkHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(Payload))); + } + if (!InContext.Base.PrimeCacheOnly) + { + if (!AbortFlag) + { + AsyncWriteDownloadedChunk( + InContext.Base.ZenFolderPath, + InContext.RemoteContent, + Context.RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(Payload), + Context.SequenceIndexChunksLeftToWriteCounters, + Context.WritePartsComplete, + Context.TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + Stats.DiskStats); + } + } + }); } else { ZEN_TRACE_CPU("UpdateFolder_GetChunk"); - BuildBlob = Storage.BuildStorage->GetBuildBlob(BuildId, ChunkHash); + BuildBlob = Storage.BuildStorage->GetBuildBlob(InContext.Base.BuildId, ChunkHash); if (BuildBlob && Storage.BuildCacheStorage) { Storage.BuildCacheStorage->PutBuildBlob( - BuildId, + InContext.Base.BuildId, ChunkHash, BuildBlob.GetContentType(), CompositeBuffer(SharedBuffer(BuildBlob))); @@ -6748,31 +6770,31 @@ namespace { { throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); } - if (!PrimeCacheOnly) + if (!InContext.Base.PrimeCacheOnly) { if (!AbortFlag) { uint64_t BlobSize = BuildBlob.GetSize(); - DownloadStats.DownloadedChunkCount++; - DownloadStats.DownloadedChunkByteCount += BlobSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + Stats.DownloadStats.DownloadedChunkCount++; + Stats.DownloadStats.DownloadedChunkByteCount += BlobSize; + Stats.DownloadStats.RequestsCompleteCount++; + if (Stats.DownloadStats.RequestsCompleteCount == Context.TotalRequestCount) { FilteredDownloadedBytesPerSecond.Stop(); } - AsyncWriteDownloadedChunk(ZenFolderPath, - RemoteContent, - RemoteLookup, + AsyncWriteDownloadedChunk(InContext.Base.ZenFolderPath, + InContext.RemoteContent, + Context.RemoteLookup, RemoteChunkIndex, std::move(ChunkTargetPtrs), Work, WritePool, std::move(BuildBlob), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, + Context.SequenceIndexChunksLeftToWriteCounters, + Context.WritePartsComplete, + Context.TotalPartWriteCount, FilteredWrittenBytesPerSecond, - DiskStats); + Stats.DiskStats); } } } @@ -6787,7 +6809,7 @@ namespace { for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) { - ZEN_ASSERT(!PrimeCacheOnly); + ZEN_ASSERT(!InContext.Base.PrimeCacheOnly); if (AbortFlag) { break; @@ -6795,13 +6817,8 @@ namespace { Work.ScheduleWork( WritePool, - [&Path, - &LocalContent, - &RemoteContent, - &RemoteLookup, - &CacheFolderPath, - &LocalLookup, - &SequenceIndexChunksLeftToWriteCounters, + [&InContext, + &Context, &Work, &WritePool, &FilteredWrittenBytesPerSecond, @@ -6809,9 +6826,7 @@ namespace { &ScavengedContents, &ScavengedLookups, &ScavengedPaths, - &WritePartsComplete, - TotalPartWriteCount, - &DiskStats, + &Stats, CopyDataIndex](std::atomic<bool>&) { if (!AbortFlag) { @@ -6824,8 +6839,9 @@ namespace { if (CopyData.ScavengeSourceIndex == (uint32_t)-1) { - const uint32_t LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; - SourceFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + const uint32_t LocalPathIndex = + Context.LocalLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; + SourceFilePath = (InContext.Base.Path / InContext.LocalContent.Paths[LocalPathIndex]).make_preferred(); } else { @@ -6894,8 +6910,8 @@ namespace { tsl::robin_set<uint32_t> ChunkIndexesWritten; - BufferedOpenFile SourceFile(SourceFilePath, DiskStats); - WriteFileCache OpenFileCache(DiskStats); + BufferedOpenFile SourceFile(SourceFilePath, Stats.DiskStats); + WriteFileCache OpenFileCache(Stats.DiskStats); for (size_t WriteOpIndex = 0; WriteOpIndex < WriteOps.size();) { if (AbortFlag) @@ -6905,11 +6921,11 @@ namespace { const WriteOp& Op = WriteOps[WriteOpIndex]; const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() <= - RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]); - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0); - const uint32_t RemotePathIndex = RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; - const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex]; + ZEN_ASSERT(Context.SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() <= + InContext.RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]); + ZEN_ASSERT(Context.SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0); + const uint32_t RemotePathIndex = Context.RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; + const uint64_t ChunkSize = InContext.RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex]; uint64_t ReadLength = ChunkSize; size_t WriteCount = 1; @@ -6930,7 +6946,8 @@ namespace { { break; } - const uint64_t NextChunkLength = RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex]; + const uint64_t NextChunkLength = + InContext.RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex]; if (ReadLength + NextChunkLength > 512u * 1024u) { break; @@ -6942,18 +6959,19 @@ namespace { } CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ReadLength); - ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); + ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= + InContext.RemoteContent.RawSizes[RemotePathIndex]); OpenFileCache.WriteToFile<CompositeBuffer>( RemoteSequenceIndex, - [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { + [&InContext, &Context](uint32_t SequenceIndex) { return GetTempChunkedSequenceFileName( - CacheFolderPath, - RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + Context.CacheFolderPath, + InContext.RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); }, ChunkSource, Op.Target->Offset, - RemoteContent.RawSizes[RemotePathIndex]); + InContext.RemoteContent.RawSizes[RemotePathIndex]); CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes? @@ -6967,21 +6985,21 @@ namespace { for (const WriteOp& Op : WriteOps) { const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; - if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters)) + if (CompleteSequenceChunk(RemoteSequenceIndex, Context.SequenceIndexChunksLeftToWriteCounters)) { CompletedChunkSequences.push_back(RemoteSequenceIndex); } } - VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, - RemoteContent, - RemoteLookup, + VerifyAndCompleteChunkSequencesAsync(Context.CacheFolderPath, + InContext.RemoteContent, + Context.RemoteLookup, CompletedChunkSequences, Work, WritePool); ZEN_CONSOLE_VERBOSE("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), SourceFilePath); } - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) + Context.WritePartsComplete++; + if (Context.WritePartsComplete == Context.TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); } @@ -6991,7 +7009,7 @@ namespace { for (uint32_t BlockIndex : CachedChunkBlockIndexes) { - ZEN_ASSERT(!PrimeCacheOnly); + ZEN_ASSERT(!InContext.Base.PrimeCacheOnly); if (AbortFlag) { break; @@ -6999,29 +7017,17 @@ namespace { Work.ScheduleWork( WritePool, - [&ZenFolderPath, - &CacheFolderPath, - &RemoteContent, - &RemoteLookup, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &SequenceIndexChunksLeftToWriteCounters, - &Work, - &WritePool, - &BlockDescriptions, - &FilteredWrittenBytesPerSecond, - &DiskStats, - &WritePartsComplete, - TotalPartWriteCount, - BlockIndex](std::atomic<bool>&) mutable { + [&InContext, &Context, &Work, &WritePool, &FilteredWrittenBytesPerSecond, &Stats, BlockIndex]( + std::atomic<bool>&) mutable { if (!AbortFlag) { ZEN_TRACE_CPU("UpdateFolder_WriteCachedBlock"); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex]; FilteredWrittenBytesPerSecond.Start(); std::filesystem::path BlockChunkPath = - ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + ZenTempBlockFolderPath(InContext.Base.ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); if (!BlockBuffer) { @@ -7031,16 +7037,16 @@ namespace { if (!AbortFlag) { - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, + if (!WriteBlockToDisk(Context.CacheFolderPath, + InContext.RemoteContent, BlockDescription, - SequenceIndexChunksLeftToWriteCounters, + Context.SequenceIndexChunksLeftToWriteCounters, Work, WritePool, CompositeBuffer(std::move(BlockBuffer)), - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats)) + Context.RemoteLookup, + Context.RemoteChunkIndexNeedsCopyFromSourceFlags, + Stats.DiskStats)) { std::error_code DummyEc; RemoveFile(BlockChunkPath, DummyEc); @@ -7049,9 +7055,9 @@ namespace { TryRemoveFile(BlockChunkPath); - WritePartsComplete++; + Context.WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) + if (Context.WritePartsComplete == Context.TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); } @@ -7062,7 +7068,7 @@ namespace { for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++) { - ZEN_ASSERT(!PrimeCacheOnly); + ZEN_ASSERT(!InContext.Base.PrimeCacheOnly); if (AbortFlag) { break; @@ -7073,22 +7079,12 @@ namespace { Work.ScheduleWork( NetworkPool, [&Storage, - &ZenFolderPath, - BuildId, - &RemoteLookup, - &BlockDescriptions, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &CacheFolderPath, - &RemoteContent, - &SequenceIndexChunksLeftToWriteCounters, + &InContext, + &Context, &ExistsResult, &FilteredDownloadedBytesPerSecond, - TotalRequestCount, - &WritePartsComplete, - TotalPartWriteCount, &FilteredWrittenBytesPerSecond, - &DiskStats, - &DownloadStats, + &Stats, &Work, &WritePool, BlockIndex, @@ -7097,20 +7093,20 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_GetPartialBlock"); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex]; FilteredDownloadedBytesPerSecond.Start(); IoBuffer BlockBuffer; if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) { - BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, + BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(InContext.Base.BuildId, BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); } if (!BlockBuffer) { - BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, + BlockBuffer = Storage.BuildStorage->GetBuildBlob(InContext.Base.BuildId, BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); @@ -7122,10 +7118,10 @@ namespace { if (!AbortFlag) { uint64_t BlockSize = BlockBuffer.GetSize(); - DownloadStats.DownloadedBlockCount++; - DownloadStats.DownloadedBlockByteCount += BlockSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + Stats.DownloadStats.DownloadedBlockCount++; + Stats.DownloadStats.DownloadedBlockByteCount += BlockSize; + Stats.DownloadStats.RequestsCompleteCount++; + if (Stats.DownloadStats.RequestsCompleteCount == Context.TotalRequestCount) { FilteredDownloadedBytesPerSecond.Stop(); } @@ -7146,10 +7142,11 @@ namespace { { BlockBuffer.SetDeleteOnClose(false); BlockBuffer = {}; - BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); + BlockChunkPath = ZenTempBlockFolderPath(InContext.Base.ZenFolderPath) / + fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); RenameFile(TempBlobPath, BlockChunkPath, Ec); if (Ec) { @@ -7168,10 +7165,11 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); // Could not be moved and rather large, lets store it on disk - BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); + BlockChunkPath = + ZenTempBlockFolderPath(InContext.Base.ZenFolderPath) / fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); BlockBuffer = {}; } @@ -7180,17 +7178,11 @@ namespace { { Work.ScheduleWork( WritePool, - [&CacheFolderPath, - &RemoteContent, - &RemoteLookup, - &BlockDescriptions, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &SequenceIndexChunksLeftToWriteCounters, - &WritePartsComplete, + [&InContext, + &Context, &Work, - TotalPartWriteCount, &WritePool, - &DiskStats, + &Stats, &FilteredWrittenBytesPerSecond, BlockIndex, BlockRange, @@ -7200,7 +7192,7 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock"); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex]; if (BlockChunkPath.empty()) { @@ -7221,18 +7213,18 @@ namespace { FilteredWrittenBytesPerSecond.Start(); if (!WritePartialBlockToDisk( - CacheFolderPath, - RemoteContent, + Context.CacheFolderPath, + InContext.RemoteContent, BlockDescription, - SequenceIndexChunksLeftToWriteCounters, + Context.SequenceIndexChunksLeftToWriteCounters, Work, WritePool, CompositeBuffer(std::move(BlockPartialBuffer)), BlockRange.ChunkBlockIndexStart, BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats)) + Context.RemoteLookup, + Context.RemoteChunkIndexNeedsCopyFromSourceFlags, + Stats.DiskStats)) { std::error_code DummyEc; RemoveFile(BlockChunkPath, DummyEc); @@ -7245,8 +7237,8 @@ namespace { TryRemoveFile(BlockChunkPath); } - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) + Context.WritePartsComplete++; + if (Context.WritePartsComplete == Context.TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); } @@ -7265,41 +7257,29 @@ namespace { break; } - if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(BlockDescriptions[BlockIndex].BlockHash)) + if (InContext.Base.PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(InContext.BlockDescriptions[BlockIndex].BlockHash)) { - DownloadStats.RequestsCompleteCount++; + Stats.DownloadStats.RequestsCompleteCount++; continue; } Work.ScheduleWork( NetworkPool, [&Storage, - &ZenFolderPath, - BuildId, - PrimeCacheOnly, - &BlockDescriptions, - &WritePartsComplete, - TotalPartWriteCount, + &InContext, + &Context, &FilteredWrittenBytesPerSecond, &ExistsResult, &Work, &WritePool, - &RemoteContent, - &RemoteLookup, - &CacheFolderPath, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &SequenceIndexChunksLeftToWriteCounters, &FilteredDownloadedBytesPerSecond, - &WriteChunkStats, - &DiskStats, - &DownloadStats, - TotalRequestCount, + &Stats, BlockIndex](std::atomic<bool>&) { if (!AbortFlag) { ZEN_TRACE_CPU("UpdateFolder_GetFullBlock"); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex]; FilteredDownloadedBytesPerSecond.Start(); @@ -7308,14 +7288,14 @@ namespace { Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); if (ExistsInCache) { - BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); + BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(InContext.Base.BuildId, BlockDescription.BlockHash); } if (!BlockBuffer) { - BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); + BlockBuffer = Storage.BuildStorage->GetBuildBlob(InContext.Base.BuildId, BlockDescription.BlockHash); if (BlockBuffer && Storage.BuildCacheStorage) { - Storage.BuildCacheStorage->PutBuildBlob(BuildId, + Storage.BuildCacheStorage->PutBuildBlob(InContext.Base.BuildId, BlockDescription.BlockHash, BlockBuffer.GetContentType(), CompositeBuffer(SharedBuffer(BlockBuffer))); @@ -7328,15 +7308,15 @@ namespace { if (!AbortFlag) { uint64_t BlockSize = BlockBuffer.GetSize(); - DownloadStats.DownloadedBlockCount++; - DownloadStats.DownloadedBlockByteCount += BlockSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + Stats.DownloadStats.DownloadedBlockCount++; + Stats.DownloadStats.DownloadedBlockByteCount += BlockSize; + Stats.DownloadStats.RequestsCompleteCount++; + if (Stats.DownloadStats.RequestsCompleteCount == Context.TotalRequestCount) { FilteredDownloadedBytesPerSecond.Stop(); } - if (!PrimeCacheOnly) + if (!InContext.Base.PrimeCacheOnly) { std::filesystem::path BlockChunkPath; @@ -7352,9 +7332,9 @@ namespace { if (!Ec) { BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = - ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + BlockBuffer = {}; + BlockChunkPath = ZenTempBlockFolderPath(InContext.Base.ZenFolderPath) / + BlockDescription.BlockHash.ToHexString(); RenameFile(TempBlobPath, BlockChunkPath, Ec); if (Ec) { @@ -7373,7 +7353,8 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); // Could not be moved and rather large, lets store it on disk - BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + BlockChunkPath = + ZenTempBlockFolderPath(InContext.Base.ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); BlockBuffer = {}; } @@ -7383,17 +7364,10 @@ namespace { Work.ScheduleWork(WritePool, [&Work, &WritePool, - &RemoteContent, - &RemoteLookup, - CacheFolderPath, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &SequenceIndexChunksLeftToWriteCounters, + &InContext, + &Context, BlockIndex, - &BlockDescriptions, - &WriteChunkStats, - &DiskStats, - &WritePartsComplete, - TotalPartWriteCount, + &Stats, &FilteredWrittenBytesPerSecond, BlockChunkPath, BlockBuffer = std::move(BlockBuffer)](std::atomic<bool>&) mutable { @@ -7402,7 +7376,7 @@ namespace { ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); const ChunkBlockDescription& BlockDescription = - BlockDescriptions[BlockIndex]; + InContext.BlockDescriptions[BlockIndex]; if (BlockChunkPath.empty()) { @@ -7422,16 +7396,16 @@ namespace { } FilteredWrittenBytesPerSecond.Start(); - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, + if (!WriteBlockToDisk(Context.CacheFolderPath, + InContext.RemoteContent, BlockDescription, - SequenceIndexChunksLeftToWriteCounters, + Context.SequenceIndexChunksLeftToWriteCounters, Work, WritePool, CompositeBuffer(std::move(BlockBuffer)), - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats)) + Context.RemoteLookup, + Context.RemoteChunkIndexNeedsCopyFromSourceFlags, + Stats.DiskStats)) { std::error_code DummyEc; RemoveFile(BlockChunkPath, DummyEc); @@ -7444,9 +7418,9 @@ namespace { TryRemoveFile(BlockChunkPath); } - WritePartsComplete++; + Context.WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) + if (Context.WritePartsComplete == Context.TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); } @@ -7464,32 +7438,34 @@ namespace { Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); - uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + - DownloadStats.DownloadedBlockByteCount.load() + - +DownloadStats.DownloadedPartialBlockByteCount.load(); - FilteredWrittenBytesPerSecond.Update(DiskStats.WriteByteCount.load()); + uint64_t DownloadedBytes = Stats.DownloadStats.DownloadedChunkByteCount.load() + + Stats.DownloadStats.DownloadedBlockByteCount.load() + + Stats.DownloadStats.DownloadedPartialBlockByteCount.load(); + FilteredWrittenBytesPerSecond.Update(Stats.DiskStats.WriteByteCount.load()); FilteredDownloadedBytesPerSecond.Update(DownloadedBytes); std::string DownloadRateString = - (DownloadStats.RequestsCompleteCount == TotalRequestCount) + (Stats.DownloadStats.RequestsCompleteCount == Context.TotalRequestCount) ? "" : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8)); - std::string WriteDetails = PrimeCacheOnly ? "" - : fmt::format(" {}/{} ({}B/s) written.", - NiceBytes(DiskStats.WriteByteCount.load()), - NiceBytes(BytesToWrite), - NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); + std::string WriteDetails = InContext.Base.PrimeCacheOnly + ? "" + : fmt::format(" {}/{} ({}B/s) written.", + NiceBytes(Stats.DiskStats.WriteByteCount.load()), + NiceBytes(BytesToWrite), + NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}", - DownloadStats.RequestsCompleteCount.load(), - TotalRequestCount, + Stats.DownloadStats.RequestsCompleteCount.load(), + Context.TotalRequestCount, NiceBytes(DownloadedBytes), DownloadRateString, WriteDetails); WriteProgressBar.UpdateState( - {.Task = PrimeCacheOnly ? "Downloading " : "Writing chunks ", + {.Task = InContext.Base.PrimeCacheOnly ? "Downloading " : "Writing chunks ", .Details = Details, - .TotalCount = PrimeCacheOnly ? TotalRequestCount : BytesToWrite, - .RemainingCount = PrimeCacheOnly ? (TotalRequestCount - DownloadStats.RequestsCompleteCount.load()) - : (BytesToWrite - DiskStats.WriteByteCount.load())}, + .TotalCount = InContext.Base.PrimeCacheOnly ? Context.TotalRequestCount : BytesToWrite, + .RemainingCount = InContext.Base.PrimeCacheOnly + ? (Context.TotalRequestCount - Stats.DownloadStats.RequestsCompleteCount.load()) + : (BytesToWrite - Stats.DiskStats.WriteByteCount.load())}, false); }); } @@ -7504,19 +7480,19 @@ namespace { WriteProgressBar.Finish(); - if (!PrimeCacheOnly) + if (!InContext.Base.PrimeCacheOnly) { uint32_t RawSequencesMissingWriteCount = 0; - for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++) + for (uint32_t SequenceIndex = 0; SequenceIndex < Context.SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++) { - const auto& SequenceIndexChunksLeftToWriteCounter = SequenceIndexChunksLeftToWriteCounters[SequenceIndex]; + const auto& SequenceIndexChunksLeftToWriteCounter = Context.SequenceIndexChunksLeftToWriteCounters[SequenceIndex]; if (SequenceIndexChunksLeftToWriteCounter.load() != 0) { RawSequencesMissingWriteCount++; - const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; - const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex]; + const uint32_t PathIndex = Context.RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + const std::filesystem::path& IncompletePath = InContext.RemoteContent.Paths[PathIndex]; ZEN_ASSERT(!IncompletePath.empty()); - const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; + const uint32_t ExpectedSequenceCount = InContext.RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; ZEN_CONSOLE("{}: Max count {}, Current count {}", IncompletePath, ExpectedSequenceCount, @@ -7527,29 +7503,30 @@ namespace { ZEN_ASSERT(RawSequencesMissingWriteCount == 0); } - const uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() + - +DownloadStats.DownloadedPartialBlockByteCount.load(); + const uint64_t DownloadedBytes = Stats.DownloadStats.DownloadedChunkByteCount.load() + + Stats.DownloadStats.DownloadedBlockByteCount.load() + + Stats.DownloadStats.DownloadedPartialBlockByteCount.load(); ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}", NiceBytes(DownloadedBytes), NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)), NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000), - NiceBytes(DiskStats.WriteByteCount.load()), - NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), DiskStats.WriteByteCount.load())), + NiceBytes(Stats.DiskStats.WriteByteCount.load()), + NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), Stats.DiskStats.WriteByteCount.load())), NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000), NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs())); - WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs(); - WriteChunkStats.DownloadTimeUs = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(); - WriteChunkStats.WriteTimeUs = FilteredWrittenBytesPerSecond.GetElapsedTimeUS(); + Stats.WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs(); + Stats.WriteChunkStats.DownloadTimeUs = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(); + Stats.WriteChunkStats.WriteTimeUs = FilteredWrittenBytesPerSecond.GetElapsedTimeUS(); } - if (PrimeCacheOnly) + if (InContext.Base.PrimeCacheOnly) { return; } tsl::robin_map<uint32_t, uint32_t> RemotePathIndexToLocalPathIndex; - RemotePathIndexToLocalPathIndex.reserve(RemoteContent.Paths.size()); + RemotePathIndexToLocalPathIndex.reserve(InContext.RemoteContent.Paths.size()); tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> SequenceHashToLocalPathIndex; std::vector<uint32_t> RemoveLocalPathIndexes; @@ -7564,10 +7541,10 @@ namespace { tsl::robin_set<IoHash, IoHash::Hasher> CachedRemoteSequences; tsl::robin_map<std::string, uint32_t> RemotePathToRemoteIndex; - RemotePathToRemoteIndex.reserve(RemoteContent.Paths.size()); - for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) + RemotePathToRemoteIndex.reserve(InContext.RemoteContent.Paths.size()); + for (uint32_t RemotePathIndex = 0; RemotePathIndex < InContext.RemoteContent.Paths.size(); RemotePathIndex++) { - RemotePathToRemoteIndex.insert({RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex}); + RemotePathToRemoteIndex.insert({InContext.RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex}); } std::vector<uint32_t> FilesToCache; @@ -7579,24 +7556,24 @@ namespace { std::atomic<uint64_t> CachedByteCount = 0; uint64_t SkippedCount = 0; uint64_t DeleteCount = 0; - for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) + for (uint32_t LocalPathIndex = 0; LocalPathIndex < InContext.LocalContent.Paths.size(); LocalPathIndex++) { if (AbortFlag) { break; } - const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; - const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; + const IoHash& RawHash = InContext.LocalContent.RawHashes[LocalPathIndex]; + const std::filesystem::path& LocalPath = InContext.LocalContent.Paths[LocalPathIndex]; - ZEN_ASSERT_SLOW(IsFile((Path / LocalContent.Paths[LocalPathIndex]).make_preferred())); + ZEN_ASSERT_SLOW(IsFile((InContext.Base.Path / InContext.LocalContent.Paths[LocalPathIndex]).make_preferred())); - if (!WipeTargetFolder) + if (!InContext.Base.WipeTargetFolder) { if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string()); RemotePathIt != RemotePathToRemoteIndex.end()) { const uint32_t RemotePathIndex = RemotePathIt->second; - if (RemoteContent.RawHashes[RemotePathIndex] == RawHash) + if (InContext.RemoteContent.RawHashes[RemotePathIndex] == RawHash) { // It is already in it's desired place RemotePathIndexToLocalPathIndex[RemotePathIndex] = LocalPathIndex; @@ -7614,7 +7591,7 @@ namespace { PathMismatchCount++; } } - if (RemoteLookup.RawHashToSequenceIndex.contains(RawHash)) + if (Context.RemoteLookup.RawHashToSequenceIndex.contains(RawHash)) { if (!CachedRemoteSequences.contains(RawHash)) { @@ -7629,7 +7606,7 @@ namespace { SkippedCount++; } } - else if (!WipeTargetFolder) + else if (!InContext.Base.WipeTargetFolder) { // We don't need it RemoveLocalPathIndexes.push_back(LocalPathIndex); @@ -7658,22 +7635,23 @@ namespace { { break; } - Work.ScheduleWork( - WritePool, - [&Path, &LocalContent, &CacheFolderPath, &CachedCount, &CachedByteCount, LocalPathIndex](std::atomic<bool>&) { - ZEN_TRACE_CPU("UpdateFolder_AsyncCopyToCache"); - if (!AbortFlag) - { - const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; - const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath)); - const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); - RenameFileWithRetry(LocalFilePath, CacheFilePath); - CachedCount++; - CachedByteCount += LocalContent.RawSizes[LocalPathIndex]; - } - }); + Work.ScheduleWork(WritePool, + [&InContext, &Context, &CachedCount, &CachedByteCount, LocalPathIndex](std::atomic<bool>&) { + ZEN_TRACE_CPU("UpdateFolder_AsyncCopyToCache"); + if (!AbortFlag) + { + const IoHash& RawHash = InContext.LocalContent.RawHashes[LocalPathIndex]; + const std::filesystem::path& LocalPath = InContext.LocalContent.Paths[LocalPathIndex]; + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(Context.CacheFolderPath, RawHash); + ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath)); + const std::filesystem::path LocalFilePath = + (InContext.Base.Path / LocalPath).make_preferred(); + RenameFileWithRetry(LocalFilePath, CacheFilePath); + CachedCount++; + CachedByteCount += InContext.LocalContent.RawSizes[LocalPathIndex]; + } + }); } { @@ -7712,17 +7690,17 @@ namespace { } } - if (WipeTargetFolder) + if (InContext.Base.WipeTargetFolder) { ZEN_TRACE_CPU("UpdateFolder_WipeTarget"); Stopwatch Timer; // Clean target folder - if (!CleanDirectory(Path, DefaultExcludeFolders)) + if (!CleanDirectory(InContext.Base.Path, DefaultExcludeFolders)) { - ZEN_WARN("Some files in {} could not be removed", Path); + ZEN_WARN("Some files in {} could not be removed", InContext.Base.Path); } - RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs(); + Stats.RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs(); } if (AbortFlag) @@ -7739,10 +7717,10 @@ namespace { ProgressBar RebuildProgressBar(ProgressMode, "Rebuild State"); ParallelWork Work(AbortFlag); - OutLocalFolderState.Paths.resize(RemoteContent.Paths.size()); - OutLocalFolderState.RawSizes.resize(RemoteContent.Paths.size()); - OutLocalFolderState.Attributes.resize(RemoteContent.Paths.size()); - OutLocalFolderState.ModificationTicks.resize(RemoteContent.Paths.size()); + OutLocalFolderState.Paths.resize(InContext.RemoteContent.Paths.size()); + OutLocalFolderState.RawSizes.resize(InContext.RemoteContent.Paths.size()); + OutLocalFolderState.Attributes.resize(InContext.RemoteContent.Paths.size()); + OutLocalFolderState.ModificationTicks.resize(InContext.RemoteContent.Paths.size()); std::atomic<uint64_t> DeletedCount = 0; @@ -7752,10 +7730,11 @@ namespace { { break; } - Work.ScheduleWork(WritePool, [&Path, &LocalContent, &DeletedCount, LocalPathIndex](std::atomic<bool>&) { + Work.ScheduleWork(WritePool, [&InContext, &DeletedCount, LocalPathIndex](std::atomic<bool>&) { if (!AbortFlag) { - const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + const std::filesystem::path LocalFilePath = + (InContext.Base.Path / InContext.LocalContent.Paths[LocalPathIndex]).make_preferred(); SetFileReadOnlyWithRetry(LocalFilePath, false); RemoveFileWithRetry(LocalFilePath); DeletedCount++; @@ -7772,10 +7751,11 @@ namespace { }; std::vector<FinalizeTarget> Targets; - Targets.reserve(RemoteContent.Paths.size()); - for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) + Targets.reserve(InContext.RemoteContent.Paths.size()); + for (uint32_t RemotePathIndex = 0; RemotePathIndex < InContext.RemoteContent.Paths.size(); RemotePathIndex++) { - Targets.push_back(FinalizeTarget{.RawHash = RemoteContent.RawHashes[RemotePathIndex], .RemotePathIndex = RemotePathIndex}); + Targets.push_back( + FinalizeTarget{.RawHash = InContext.RemoteContent.RawHashes[RemotePathIndex], .RemotePathIndex = RemotePathIndex}); } std::sort(Targets.begin(), Targets.end(), [](const FinalizeTarget& Lhs, const FinalizeTarget& Rhs) { if (Lhs.RawHash < Rhs.RawHash) @@ -7805,15 +7785,12 @@ namespace { Work.ScheduleWork( WritePool, - [&Path, - &LocalContent, + [&InContext, + &Context, &SequenceHashToLocalPathIndex, - &RemoteContent, - &RemoteLookup, - &CacheFolderPath, &Targets, &RemotePathIndexToLocalPathIndex, - &RebuildFolderStateStats, + &Stats, &OutLocalFolderState, BaseTargetOffset = TargetOffset, TargetCount, @@ -7832,8 +7809,8 @@ namespace { { const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); - const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; - std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); + const std::filesystem::path& TargetPath = InContext.RemoteContent.Paths[RemotePathIndex]; + std::filesystem::path TargetFilePath = (InContext.Base.Path / TargetPath).make_preferred(); if (!RemotePathIndexToLocalPathIndex[RemotePathIndex]) { if (IsFileWithRetry(TargetFilePath)) @@ -7848,14 +7825,14 @@ namespace { OutputFile.Open(TargetFilePath, BasicFile::Mode::kTruncate); } OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; - OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; + OutLocalFolderState.RawSizes[RemotePathIndex] = InContext.RemoteContent.RawSizes[RemotePathIndex]; OutLocalFolderState.Attributes[RemotePathIndex] = - RemoteContent.Attributes.empty() + InContext.RemoteContent.Attributes.empty() ? GetNativeFileAttributes(TargetFilePath) : SetNativeFileAttributes(TargetFilePath, - RemoteContent.Platform, - RemoteContent.Attributes[RemotePathIndex]); + InContext.RemoteContent.Platform, + InContext.RemoteContent.Attributes[RemotePathIndex]); OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); TargetOffset++; @@ -7865,10 +7842,10 @@ namespace { else { ZEN_TRACE_CPU("Files"); - ZEN_ASSERT(RemoteLookup.RawHashToSequenceIndex.contains(RawHash)); + ZEN_ASSERT(Context.RemoteLookup.RawHashToSequenceIndex.contains(RawHash)); const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex; - const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstRemotePathIndex]; - std::filesystem::path FirstTargetFilePath = (Path / FirstTargetPath).make_preferred(); + const std::filesystem::path& FirstTargetPath = InContext.RemoteContent.Paths[FirstRemotePathIndex]; + std::filesystem::path FirstTargetFilePath = (InContext.Base.Path / FirstTargetPath).make_preferred(); if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end()) @@ -7891,39 +7868,39 @@ namespace { { ZEN_TRACE_CPU("Copy"); const uint32_t LocalPathIndex = InplaceIt->second; - const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex]; - std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred(); + const std::filesystem::path& SourcePath = InContext.LocalContent.Paths[LocalPathIndex]; + std::filesystem::path SourceFilePath = (InContext.Base.Path / SourcePath).make_preferred(); ZEN_ASSERT_SLOW(IsFileWithRetry(SourceFilePath)); ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath); - const uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; + const uint64_t RawSize = InContext.LocalContent.RawSizes[LocalPathIndex]; std::atomic<uint64_t> WriteCount; std::atomic<uint64_t> WriteByteCount; CopyFile(SourceFilePath, FirstTargetFilePath, RawSize, WriteCount, WriteByteCount); - RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + Stats.RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; } else { ZEN_TRACE_CPU("Rename"); const std::filesystem::path CacheFilePath = - GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); + GetFinalChunkedSequenceFileName(Context.CacheFolderPath, RawHash); ZEN_ASSERT_SLOW(IsFileWithRetry(CacheFilePath)); RenameFileWithRetry(CacheFilePath, FirstTargetFilePath); - RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; + Stats.RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; } } OutLocalFolderState.Paths[FirstRemotePathIndex] = FirstTargetPath; - OutLocalFolderState.RawSizes[FirstRemotePathIndex] = RemoteContent.RawSizes[FirstRemotePathIndex]; + OutLocalFolderState.RawSizes[FirstRemotePathIndex] = InContext.RemoteContent.RawSizes[FirstRemotePathIndex]; OutLocalFolderState.Attributes[FirstRemotePathIndex] = - RemoteContent.Attributes.empty() + InContext.RemoteContent.Attributes.empty() ? GetNativeFileAttributes(FirstTargetFilePath) : SetNativeFileAttributes(FirstTargetFilePath, - RemoteContent.Platform, - RemoteContent.Attributes[FirstRemotePathIndex]); + InContext.RemoteContent.Platform, + InContext.RemoteContent.Attributes[FirstRemotePathIndex]); OutLocalFolderState.ModificationTicks[FirstRemotePathIndex] = GetModificationTickFromPath(FirstTargetFilePath); @@ -7934,8 +7911,8 @@ namespace { { const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); - const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; - std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); + const std::filesystem::path& TargetPath = InContext.RemoteContent.Paths[RemotePathIndex]; + std::filesystem::path TargetFilePath = (InContext.Base.Path / TargetPath).make_preferred(); if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end()) @@ -7956,22 +7933,22 @@ namespace { ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath); - const uint64_t RawSize = RemoteContent.RawSizes[RemotePathIndex]; + const uint64_t RawSize = InContext.RemoteContent.RawSizes[RemotePathIndex]; std::atomic<uint64_t> WriteCount; std::atomic<uint64_t> WriteByteCount; CopyFile(FirstTargetFilePath, TargetFilePath, RawSize, WriteCount, WriteByteCount); - RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + Stats.RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; } OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; - OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; + OutLocalFolderState.RawSizes[RemotePathIndex] = InContext.RemoteContent.RawSizes[RemotePathIndex]; OutLocalFolderState.Attributes[RemotePathIndex] = - RemoteContent.Attributes.empty() + InContext.RemoteContent.Attributes.empty() ? GetNativeFileAttributes(TargetFilePath) : SetNativeFileAttributes(TargetFilePath, - RemoteContent.Platform, - RemoteContent.Attributes[RemotePathIndex]); + InContext.RemoteContent.Platform, + InContext.RemoteContent.Attributes[RemotePathIndex]); OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); TargetOffset++; @@ -8000,7 +7977,7 @@ namespace { }); } - RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs = Timer.GetElapsedTimeUs(); + Stats.RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs = Timer.GetElapsedTimeUs(); if (AbortFlag) { @@ -8114,7 +8091,7 @@ namespace { std::vector<std::pair<Oid, std::string>> ResolveBuildPartNames(BuildStorage& Storage, const Oid& BuildId, - const std::vector<Oid>& BuildPartIds, + std::span<const Oid> BuildPartIds, std::span<const std::string> BuildPartNames, std::uint64_t& OutPreferredMultipartChunkSize) { @@ -8742,19 +8719,7 @@ namespace { return Result; }; - void DownloadFolder(StorageInstance& Storage, - const Oid& BuildId, - const std::vector<Oid>& BuildPartIds, - std::span<const std::string> BuildPartNames, - const std::filesystem::path& Path, - const std::filesystem::path& ZenFolderPath, - const std::filesystem::path SystemRootDir, - bool AllowMultiparts, - bool AllowPartialBlockRequests, - bool WipeTargetFolder, - bool PostDownloadVerify, - bool PrimeCacheOnly, - bool EnableScavenging) + void DownloadFolder(StorageInstance& Storage, const DownloadFolderContext& Context) { ZEN_TRACE_CPU("DownloadFolder"); @@ -8773,23 +8738,26 @@ namespace { auto EndProgress = MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); - ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests))); + ZEN_ASSERT((!Context.PrimeCacheOnly) || (Context.PrimeCacheOnly && (!Context.AllowPartialBlockRequests))); Stopwatch DownloadTimer; ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount); - const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath); + const std::filesystem::path ZenTempFolder = ZenTempFolderPath(Context.ZenFolderPath); CreateDirectories(ZenTempFolder); - CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath)); - CreateDirectories(ZenTempCacheFolderPath(ZenFolderPath)); - CreateDirectories(ZenTempDownloadFolderPath(ZenFolderPath)); + CreateDirectories(ZenTempBlockFolderPath(Context.ZenFolderPath)); + CreateDirectories(ZenTempCacheFolderPath(Context.ZenFolderPath)); + CreateDirectories(ZenTempDownloadFolderPath(Context.ZenFolderPath)); std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - std::vector<std::pair<Oid, std::string>> AllBuildParts = - ResolveBuildPartNames(*Storage.BuildStorage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); + std::vector<std::pair<Oid, std::string>> AllBuildParts = ResolveBuildPartNames(*Storage.BuildStorage, + Context.BuildId, + Context.BuildPartIds, + Context.BuildPartNames, + PreferredMultipartChunkSize); std::vector<ChunkedFolderContent> PartContents; @@ -8801,18 +8769,18 @@ namespace { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount); ChunkedFolderContent RemoteContent = - GetRemoteContent(Storage, BuildId, AllBuildParts, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes); + GetRemoteContent(Storage, Context.BuildId, AllBuildParts, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes); - const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; + const std::uint64_t LargeAttachmentSize = Context.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; GetFolderContentStatistics LocalFolderScanStats; ChunkingStatistics ChunkingStats; ChunkedFolderContent LocalContent; FolderContent LocalFolderContent; - if (!PrimeCacheOnly) + if (!Context.PrimeCacheOnly) { - if (IsDir(Path)) + if (IsDir(Context.Path)) { - if (!WipeTargetFolder) + if (!Context.WipeTargetFolder) { if (!ChunkController) { @@ -8822,8 +8790,8 @@ namespace { LocalContent = GetLocalContent(LocalFolderScanStats, ChunkingStats, - Path, - ZenStateFilePath(ZenFolderPath), + Context.Path, + ZenStateFilePath(Context.ZenFolderPath), *ChunkController, RemoteContent.Paths, LocalFolderContent); @@ -8831,7 +8799,7 @@ namespace { } else { - CreateDirectories(Path); + CreateDirectories(Context.Path); } } if (AbortFlag) @@ -8887,12 +8855,12 @@ namespace { NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); Stopwatch WriteStateTimer; - CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent, Path); - CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); - TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); + CbObject StateObject = CreateStateObject(Context.BuildId, AllBuildParts, PartContents, LocalFolderContent, Context.Path); + CreateDirectories(ZenStateFilePath(Context.ZenFolderPath).parent_path()); + TemporaryFile::SafeWriteFile(ZenStateFilePath(Context.ZenFolderPath), StateObject.GetView()); ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); - AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path); + AddDownloadedPath(Context.SystemRootDir, Context.BuildId, AllBuildParts, ZenStateFilePath(Context.ZenFolderPath), Context.Path); } else { @@ -8901,7 +8869,7 @@ namespace { { BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first)); } - ZEN_CONSOLE("Downloading build {}, parts:{} to '{}'", BuildId, BuildPartString.ToView(), Path); + ZEN_CONSOLE("Downloading build {}, parts:{} to '{}'", Context.BuildId, BuildPartString.ToView(), Context.Path); FolderContent LocalFolderState; DiskStatistics DiskStats; @@ -8913,49 +8881,60 @@ namespace { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount); - UpdateFolder(SystemRootDir, - Storage, - BuildId, - Path, - ZenFolderPath, - LargeAttachmentSize, - PreferredMultipartChunkSize, - LocalContent, - RemoteContent, - BlockDescriptions, - LooseChunkHashes, - AllowPartialBlockRequests, - WipeTargetFolder, - PrimeCacheOnly, - EnableScavenging, - LocalFolderState, - DiskStats, - CacheMappingStats, - DownloadStats, - WriteChunkStats, - RebuildFolderStateStats); + UpdateFolderContext UpdateCxt = {.Base = Context, + + .LargeAttachmentSize = LargeAttachmentSize, + .PreferredMultipartChunkSize = PreferredMultipartChunkSize, + + .LocalContent = LocalContent, + .RemoteContent = RemoteContent, + + .BlockDescriptions = BlockDescriptions, + .LooseChunkHashes = LooseChunkHashes}; + + UpdateFolderStats UpdateStats{.DiskStats = DiskStats, + .CacheMappingStats = CacheMappingStats, + .DownloadStats = DownloadStats, + .WriteChunkStats = WriteChunkStats, + .RebuildFolderStateStats = RebuildFolderStateStats}; + + UpdateFolder(Storage, UpdateCxt, LocalFolderState, UpdateStats); if (!AbortFlag) { - if (!PrimeCacheOnly) + if (!Context.PrimeCacheOnly) { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount); - VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats); + struct VerifyFolderContext + { + const std::filesystem::path Path; + + const bool PostDownloadVerify; + + const ChunkedFolderContent& RemoteContent; + VerifyFolderStatistics VerifyFolderStats; + }; + + VerifyFolder(RemoteContent, Context.Path, Context.PostDownloadVerify, VerifyFolderStats); Stopwatch WriteStateTimer; - CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState, Path); + CbObject StateObject = CreateStateObject(Context.BuildId, AllBuildParts, PartContents, LocalFolderState, Context.Path); - CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); - TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); + CreateDirectories(ZenStateFilePath(Context.ZenFolderPath).parent_path()); + TemporaryFile::SafeWriteFile(ZenStateFilePath(Context.ZenFolderPath), StateObject.GetView()); ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); - AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path); + AddDownloadedPath(Context.SystemRootDir, + Context.BuildId, + AllBuildParts, + ZenStateFilePath(Context.ZenFolderPath), + Context.Path); #if 0 ExtendableStringBuilder<1024> SB; CompactBinaryToJson(StateObject, SB); - WriteFile(ZenStateFileJsonPath(ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); + WriteFile(ZenStateFileJsonPath(Context.ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); #endif // 0 } const uint64_t DownloadCount = DownloadStats.DownloadedChunkCount.load() + DownloadStats.DownloadedBlockCount.load() + @@ -8972,7 +8951,7 @@ namespace { " Clean: {}\n" " Finalize: {}\n" " Verify: {}", - BuildId, + Context.BuildId, BuildPartString.ToView(), NiceTimeSpanMs(DownloadTimeMs), @@ -8991,7 +8970,7 @@ namespace { NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); } } - if (PrimeCacheOnly) + if (Context.PrimeCacheOnly) { if (Storage.BuildCacheStorage) { @@ -9680,13 +9659,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } - std::filesystem::path SystemRootDir; - auto ParseSystemOptions = [&]() { - if (m_SystemRootDir.empty()) - { - m_SystemRootDir = PickDefaultSystemRootDirectory(); - } - MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); + auto ParseSystemOptions = [&]() { + if (m_SystemRootDir.empty()) + { + m_SystemRootDir = PickDefaultSystemRootDirectory(); + } + MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); }; ParseSystemOptions(); @@ -10488,19 +10466,23 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); - DownloadFolder(Storage, - BuildId, - BuildPartIds, - m_BuildPartNames, - m_Path, - m_ZenFolderPath, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests && !m_PrimeCacheOnly, - m_Clean, - m_PostDownloadVerify, - m_PrimeCacheOnly, - m_EnableScavenging); + DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir, + + .BuildId = BuildId, + .BuildPartIds = BuildPartIds, + .BuildPartNames = m_BuildPartNames, + + .Path = m_Path, + .ZenFolderPath = m_ZenFolderPath, + + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests && !m_PrimeCacheOnly, + .WipeTargetFolder = m_Clean, + .PrimeCacheOnly = m_PrimeCacheOnly, + .EnableScavenging = m_EnableScavenging, + .PostDownloadVerify = m_PostDownloadVerify}; + + DownloadFolder(Storage, DownloadFolderCxt); return AbortFlag ? 11 : 0; } @@ -10622,19 +10604,24 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, m_DownloadOptions.help())); } - DownloadFolder(Storage, - BuildId, - {}, - {}, - m_Path, - m_ZenFolderPath, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - BuildIdString == m_BuildIds.front(), - true, - false, - m_EnableScavenging); + + DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir, + + .BuildId = BuildId, + .BuildPartIds = {}, + .BuildPartNames = {}, + + .Path = m_Path, + .ZenFolderPath = m_ZenFolderPath, + + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .WipeTargetFolder = BuildIdString == m_BuildIds.front(), + .PrimeCacheOnly = false, + .EnableScavenging = m_EnableScavenging, + .PostDownloadVerify = true}; + + DownloadFolder(Storage, DownloadFolderCxt); if (AbortFlag) { ZEN_CONSOLE("Download cancelled"); @@ -10741,23 +10728,31 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, - BuildId, - {BuildPartId}, - {}, - DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - true, - true, - false, - m_EnableScavenging); - if (AbortFlag) + + const std::vector<Oid> BuildPartIds = {BuildPartId}; { - ZEN_CONSOLE("Download failed."); - return 11; + DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir, + + .BuildId = BuildId, + .BuildPartIds = BuildPartIds, + .BuildPartNames = {}, + + .Path = DownloadPath, + .ZenFolderPath = DownloadPath / ZenFolderName, + + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .WipeTargetFolder = true, + .PrimeCacheOnly = false, + .EnableScavenging = m_EnableScavenging, + .PostDownloadVerify = true}; + + DownloadFolder(Storage, DownloadFolderCxt); + if (AbortFlag) + { + ZEN_CONSOLE("Download failed."); + return 11; + } } ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", @@ -10765,23 +10760,29 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, - BuildId, - {BuildPartId}, - {}, - DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging); - if (AbortFlag) { - ZEN_CONSOLE("Re-download failed. (identical target)"); - return 11; + DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir, + + .BuildId = BuildId, + .BuildPartIds = BuildPartIds, + .BuildPartNames = {}, + + .Path = DownloadPath, + .ZenFolderPath = DownloadPath / ZenFolderName, + + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .WipeTargetFolder = false, + .PrimeCacheOnly = false, + .EnableScavenging = m_EnableScavenging, + .PostDownloadVerify = true}; + + DownloadFolder(Storage, DownloadFolderCxt); + if (AbortFlag) + { + ZEN_CONSOLE("Re-download failed. (identical target)"); + return 11; + } } auto ScrambleDir = [](const std::filesystem::path& Path) { @@ -10883,23 +10884,29 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, - BuildId, - {BuildPartId}, - {}, - DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging); - if (AbortFlag) { - ZEN_CONSOLE("Re-download failed. (scrambled target)"); - return 11; + DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir, + + .BuildId = BuildId, + .BuildPartIds = BuildPartIds, + .BuildPartNames = {}, + + .Path = DownloadPath, + .ZenFolderPath = DownloadPath / ZenFolderName, + + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .WipeTargetFolder = false, + .PrimeCacheOnly = false, + .EnableScavenging = m_EnableScavenging, + .PostDownloadVerify = true}; + + DownloadFolder(Storage, DownloadFolderCxt); + if (AbortFlag) + { + ZEN_CONSOLE("Re-download failed. (scrambled target)"); + return 11; + } } ScrambleDir(DownloadPath); @@ -10935,83 +10942,94 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, - BuildId, - {BuildPartId}, - {}, - DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging); - if (AbortFlag) { - ZEN_CONSOLE("Re-download failed."); - return 11; + DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir, + + .BuildId = BuildId, + .BuildPartIds = BuildPartIds, + .BuildPartNames = {}, + + .Path = DownloadPath, + .ZenFolderPath = DownloadPath / ZenFolderName, + + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .WipeTargetFolder = false, + .PrimeCacheOnly = false, + .EnableScavenging = m_EnableScavenging, + .PostDownloadVerify = true}; + + DownloadFolder(Storage, DownloadFolderCxt); + if (AbortFlag) + { + ZEN_CONSOLE("Re-download failed."); + return 11; + } } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, - BuildId2, - {BuildPartId2}, - {}, - DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging); - if (AbortFlag) + const std::vector<Oid> BuildPartIds2 = {BuildPartId2}; { - ZEN_CONSOLE("Re-download failed."); - return 11; - } + DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir, - ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, - BuildId2, - {BuildPartId2}, - {}, - DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging); - if (AbortFlag) - { - ZEN_CONSOLE("Re-download failed."); - return 11; + .BuildId = BuildId2, + .BuildPartIds = BuildPartIds2, + .BuildPartNames = {}, + + .Path = DownloadPath, + .ZenFolderPath = DownloadPath / ZenFolderName, + + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .WipeTargetFolder = false, + .PrimeCacheOnly = false, + .EnableScavenging = m_EnableScavenging, + .PostDownloadVerify = true}; + + DownloadFolder(Storage, DownloadFolderCxt); + if (AbortFlag) + { + ZEN_CONSOLE("Re-download failed."); + return 11; + } + + ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", + BuildId2, + BuildPartId2, + m_BuildPartName, + DownloadPath); + DownloadFolder(Storage, DownloadFolderCxt); + if (AbortFlag) + { + ZEN_CONSOLE("Re-download failed."); + return 11; + } } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2); - DownloadFolder(Storage, - BuildId, - {BuildPartId}, - {}, - DownloadPath2, - DownloadPath2 / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging); - if (AbortFlag) { - ZEN_CONSOLE("Re-download failed."); - return 11; + DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir, + + .BuildId = BuildId, + .BuildPartIds = BuildPartIds, + .BuildPartNames = {}, + + .Path = DownloadPath2, + .ZenFolderPath = DownloadPath2 / ZenFolderName, + + .AllowMultiparts = m_AllowMultiparts, + .AllowPartialBlockRequests = m_AllowPartialBlockRequests, + .WipeTargetFolder = false, + .PrimeCacheOnly = false, + .EnableScavenging = m_EnableScavenging, + .PostDownloadVerify = true}; + + DownloadFolder(Storage, DownloadFolderCxt); + if (AbortFlag) + { + ZEN_CONSOLE("Re-download failed."); + return 11; + } } return 0; |