aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2025-05-23 14:21:58 +0200
committerDan Engelbrecht <[email protected]>2025-05-23 14:21:58 +0200
commit658732f04f6c3c71a63ef940906643d38eb9dbfd (patch)
tree2b3d2a9dc775f537f6a314f5f90f71a3d8bb45ac
parent5.6.8 (diff)
downloadzen-de/builds-cmd-refactor.tar.xz
zen-de/builds-cmd-refactor.zip
create context structures to reduce individually captured variablesde/builds-cmd-refactor
Diffstat
-rw-r--r--src/zen/cmds/builds_cmd.cpp1698
1 files changed, 858 insertions, 840 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index d998ed3e8..a8ec3be52 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -838,6 +838,8 @@ namespace {
uint64_t LocalChunkMatchingRemoteCount = 0;
uint64_t LocalChunkMatchingRemoteByteCount = 0;
+ uint64_t LocalPathIndexesMatchingSequenceIndexCount = 0;
+
uint64_t LocalScanElapsedWallTimeUs = 0;
uint32_t ScavengedPathsMatchingSequencesCount = 0;
@@ -5435,58 +5437,101 @@ namespace {
return Result;
}
- void UpdateFolder(const std::filesystem::path& SystemRootDir,
- StorageInstance& Storage,
- const Oid& BuildId,
- const std::filesystem::path& Path,
- const std::filesystem::path& ZenFolderPath,
- const std::uint64_t LargeAttachmentSize,
- const std::uint64_t PreferredMultipartChunkSize,
- const ChunkedFolderContent& LocalContent,
- const ChunkedFolderContent& RemoteContent,
- const std::vector<ChunkBlockDescription>& BlockDescriptions,
- const std::vector<IoHash>& LooseChunkHashes,
- bool AllowPartialBlockRequests,
- bool WipeTargetFolder,
- bool PrimeCacheOnly,
- bool EnableScavenging,
- FolderContent& OutLocalFolderState,
- DiskStatistics& DiskStats,
- CacheMappingStatistics& CacheMappingStats,
- DownloadStatistics& DownloadStats,
- WriteChunkStatistics& WriteChunkStats,
- RebuildFolderStateStatistics& RebuildFolderStateStats)
+ struct DownloadFolderContext
+ {
+ const std::filesystem::path SystemRootDir;
+
+ const Oid BuildId;
+ std::span<const Oid> BuildPartIds;
+ std::span<const std::string> BuildPartNames;
+
+ const std::filesystem::path Path;
+ const std::filesystem::path ZenFolderPath;
+
+ const bool AllowMultiparts;
+ const bool AllowPartialBlockRequests;
+ const bool WipeTargetFolder;
+ const bool PrimeCacheOnly;
+ const bool EnableScavenging;
+ const bool PostDownloadVerify;
+ };
+
+ struct UpdateFolderContext
+ {
+ const DownloadFolderContext& Base;
+
+ const uint64_t LargeAttachmentSize;
+ const uint64_t PreferredMultipartChunkSize;
+
+ const ChunkedFolderContent& LocalContent;
+ const ChunkedFolderContent& RemoteContent;
+
+ std::span<const ChunkBlockDescription> BlockDescriptions;
+ std::span<const IoHash> LooseChunkHashes;
+ };
+
+ struct UpdateFolderStats
+ {
+ DiskStatistics& DiskStats;
+ CacheMappingStatistics& CacheMappingStats;
+ DownloadStatistics& DownloadStats;
+ WriteChunkStatistics& WriteChunkStats;
+ RebuildFolderStateStatistics& RebuildFolderStateStats;
+ };
+
+ void UpdateFolder(StorageInstance& Storage,
+ const UpdateFolderContext& InContext,
+ FolderContent& OutLocalFolderState,
+ UpdateFolderStats& Stats)
{
ZEN_TRACE_CPU("UpdateFolder");
- ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests)));
+ ZEN_ASSERT((!InContext.Base.PrimeCacheOnly) || (InContext.Base.PrimeCacheOnly && (!InContext.Base.AllowPartialBlockRequests)));
Stopwatch IndexTimer;
- const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent);
+ struct Context
+ {
+ const ChunkedContentLookup LocalLookup;
+ const ChunkedContentLookup RemoteLookup;
+ const std::filesystem::path CacheFolderPath;
- const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent);
+ std::vector<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters;
+ std::vector<bool> RemoteChunkIndexNeedsCopyFromLocalFileFlags;
+ std::vector<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags;
- ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
+ uint64_t TotalRequestCount;
+ uint64_t TotalPartWriteCount;
+ std::atomic<uint64_t> WritePartsComplete;
+ };
- const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(ZenFolderPath);
+ Context Context = {
+ .LocalLookup = BuildChunkedContentLookup(InContext.LocalContent),
+ .RemoteLookup = BuildChunkedContentLookup(InContext.RemoteContent),
+ .CacheFolderPath = ZenTempCacheFolderPath(InContext.Base.ZenFolderPath),
+ .SequenceIndexChunksLeftToWriteCounters =
+ std::vector<std::atomic<uint32_t>>(InContext.RemoteContent.ChunkedContent.SequenceRawHashes.size()),
+ .RemoteChunkIndexNeedsCopyFromLocalFileFlags = std::vector<bool>(InContext.RemoteContent.ChunkedContent.ChunkHashes.size()),
+ .RemoteChunkIndexNeedsCopyFromSourceFlags =
+ std::vector<std::atomic<bool>>(InContext.RemoteContent.ChunkedContent.ChunkHashes.size()),
+ .TotalRequestCount = 0,
+ .TotalPartWriteCount = 0,
+ .WritePartsComplete = 0};
- Stopwatch CacheMappingTimer;
+ ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
- std::vector<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters(RemoteContent.ChunkedContent.SequenceRawHashes.size());
- std::vector<bool> RemoteChunkIndexNeedsCopyFromLocalFileFlags(RemoteContent.ChunkedContent.ChunkHashes.size());
- std::vector<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags(RemoteContent.ChunkedContent.ChunkHashes.size());
+ Stopwatch CacheMappingTimer;
tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedChunkHashesFound;
tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedSequenceHashesFound;
- if (!PrimeCacheOnly)
+ if (!InContext.Base.PrimeCacheOnly)
{
ZEN_TRACE_CPU("UpdateFolder_CheckChunkCache");
Stopwatch CacheTimer;
DirectoryContent CacheDirContent;
- GetDirectoryContent(CacheFolderPath,
+ GetDirectoryContent(Context.CacheFolderPath,
DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes,
CacheDirContent);
for (size_t Index = 0; Index < CacheDirContent.Files.size(); Index++)
@@ -5494,34 +5539,34 @@ namespace {
IoHash FileHash;
if (IoHash::TryParse(CacheDirContent.Files[Index].filename().string(), FileHash))
{
- if (auto ChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(FileHash);
- ChunkIt != RemoteLookup.ChunkHashToChunkIndex.end())
+ if (auto ChunkIt = Context.RemoteLookup.ChunkHashToChunkIndex.find(FileHash);
+ ChunkIt != Context.RemoteLookup.ChunkHashToChunkIndex.end())
{
const uint32_t ChunkIndex = ChunkIt->second;
- const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ const uint64_t ChunkSize = InContext.RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
if (ChunkSize == CacheDirContent.FileSizes[Index])
{
CachedChunkHashesFound.insert({FileHash, ChunkIndex});
- CacheMappingStats.CacheChunkCount++;
- CacheMappingStats.CacheChunkByteCount += ChunkSize;
+ Stats.CacheMappingStats.CacheChunkCount++;
+ Stats.CacheMappingStats.CacheChunkByteCount += ChunkSize;
continue;
}
}
- else if (auto SequenceIt = RemoteLookup.RawHashToSequenceIndex.find(FileHash);
- SequenceIt != RemoteLookup.RawHashToSequenceIndex.end())
+ else if (auto SequenceIt = Context.RemoteLookup.RawHashToSequenceIndex.find(FileHash);
+ SequenceIt != Context.RemoteLookup.RawHashToSequenceIndex.end())
{
const uint32_t SequenceIndex = SequenceIt->second;
- const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
- const uint64_t SequenceSize = RemoteContent.RawSizes[PathIndex];
+ const uint32_t PathIndex = Context.RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
+ const uint64_t SequenceSize = InContext.RemoteContent.RawSizes[PathIndex];
if (SequenceSize == CacheDirContent.FileSizes[Index])
{
CachedSequenceHashesFound.insert({FileHash, SequenceIndex});
- CacheMappingStats.CacheSequenceHashesCount++;
- CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize;
+ Stats.CacheMappingStats.CacheSequenceHashesCount++;
+ Stats.CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize;
const std::filesystem::path CacheFilePath =
- GetFinalChunkedSequenceFileName(CacheFolderPath,
- RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
+ GetFinalChunkedSequenceFileName(Context.CacheFolderPath,
+ InContext.RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
continue;
@@ -5530,26 +5575,26 @@ namespace {
}
TryRemoveFile(CacheDirContent.Files[Index]);
}
- CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs();
+ Stats.CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs();
}
tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedBlocksFound;
- if (!PrimeCacheOnly)
+ if (!InContext.Base.PrimeCacheOnly)
{
ZEN_TRACE_CPU("UpdateFolder_CheckBlockCache");
Stopwatch CacheTimer;
tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> AllBlockSizes;
- AllBlockSizes.reserve(BlockDescriptions.size());
- for (uint32_t BlockIndex = 0; BlockIndex < BlockDescriptions.size(); BlockIndex++)
+ AllBlockSizes.reserve(InContext.BlockDescriptions.size());
+ for (uint32_t BlockIndex = 0; BlockIndex < InContext.BlockDescriptions.size(); BlockIndex++)
{
- const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+ const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex];
AllBlockSizes.insert({BlockDescription.BlockHash, BlockIndex});
}
DirectoryContent BlockDirContent;
- GetDirectoryContent(ZenTempBlockFolderPath(ZenFolderPath),
+ GetDirectoryContent(ZenTempBlockFolderPath(InContext.Base.ZenFolderPath),
DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes,
BlockDirContent);
CachedBlocksFound.reserve(BlockDirContent.Files.size());
@@ -5561,7 +5606,7 @@ namespace {
if (auto BlockIt = AllBlockSizes.find(FileHash); BlockIt != AllBlockSizes.end())
{
const uint32_t BlockIndex = BlockIt->second;
- const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+ const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex];
uint64_t BlockSize = CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize;
for (uint64_t ChunkSize : BlockDescription.ChunkCompressedLengths)
{
@@ -5571,8 +5616,8 @@ namespace {
if (BlockSize == BlockDirContent.FileSizes[Index])
{
CachedBlocksFound.insert({FileHash, BlockIndex});
- CacheMappingStats.CacheBlockCount++;
- CacheMappingStats.CacheBlocksByteCount += BlockSize;
+ Stats.CacheMappingStats.CacheBlockCount++;
+ Stats.CacheMappingStats.CacheBlocksByteCount += BlockSize;
continue;
}
}
@@ -5580,69 +5625,71 @@ namespace {
TryRemoveFile(BlockDirContent.Files[Index]);
}
- CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs();
+ Stats.CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs();
}
- std::vector<uint32_t> LocalPathIndexesMatchingSequenceIndexes;
tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> SequenceIndexesLeftToFindToRemoteIndex;
- if (!PrimeCacheOnly)
+ if (!InContext.Base.PrimeCacheOnly)
{
// Pick up all whole files we can use from current local state
ZEN_TRACE_CPU("UpdateFolder_GetLocalSequences");
Stopwatch LocalTimer;
- for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size();
+ for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < InContext.RemoteContent.ChunkedContent.SequenceRawHashes.size();
RemoteSequenceIndex++)
{
- const IoHash& RemoteSequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
- const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex);
- const uint64_t RemoteRawSize = RemoteContent.RawSizes[RemotePathIndex];
+ const IoHash& RemoteSequenceRawHash = InContext.RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
+ const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(Context.RemoteLookup, RemoteSequenceIndex);
+ const uint64_t RemoteRawSize = InContext.RemoteContent.RawSizes[RemotePathIndex];
if (auto CacheSequenceIt = CachedSequenceHashesFound.find(RemoteSequenceRawHash);
CacheSequenceIt != CachedSequenceHashesFound.end())
{
- const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash);
+ const std::filesystem::path CacheFilePath =
+ GetFinalChunkedSequenceFileName(Context.CacheFolderPath, RemoteSequenceRawHash);
ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
ZEN_CONSOLE_VERBOSE("Found sequence {} at {} ({})", RemoteSequenceRawHash, CacheFilePath, NiceBytes(RemoteRawSize));
}
else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash);
CacheChunkIt != CachedChunkHashesFound.end())
{
- const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash);
+ const std::filesystem::path CacheFilePath =
+ GetFinalChunkedSequenceFileName(Context.CacheFolderPath, RemoteSequenceRawHash);
ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
ZEN_CONSOLE_VERBOSE("Found chunk {} at {} ({})", RemoteSequenceRawHash, CacheFilePath, NiceBytes(RemoteRawSize));
}
- else if (auto It = LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash);
- It != LocalLookup.RawHashToSequenceIndex.end())
+ else if (auto It = Context.LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash);
+ It != Context.LocalLookup.RawHashToSequenceIndex.end())
{
- const uint32_t LocalSequenceIndex = It->second;
- const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(LocalLookup, LocalSequenceIndex);
- const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred();
+ const uint32_t LocalSequenceIndex = It->second;
+ const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(Context.LocalLookup, LocalSequenceIndex);
+ const std::filesystem::path LocalFilePath =
+ (InContext.Base.Path / InContext.LocalContent.Paths[LocalPathIndex]).make_preferred();
ZEN_ASSERT_SLOW(IsFile(LocalFilePath));
- LocalPathIndexesMatchingSequenceIndexes.push_back(LocalPathIndex);
- CacheMappingStats.LocalPathsMatchingSequencesCount++;
- CacheMappingStats.LocalPathsMatchingSequencesByteCount += RemoteRawSize;
+ Stats.CacheMappingStats.LocalPathIndexesMatchingSequenceIndexCount++;
+ Stats.CacheMappingStats.LocalPathsMatchingSequencesCount++;
+ Stats.CacheMappingStats.LocalPathsMatchingSequencesByteCount += RemoteRawSize;
ZEN_CONSOLE_VERBOSE("Found sequence {} at {} ({})", RemoteSequenceRawHash, LocalFilePath, NiceBytes(RemoteRawSize));
}
else
{
// We must write the sequence
- const uint32_t ChunkCount = RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex];
- SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount;
+ const uint32_t ChunkCount = InContext.RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex];
+ Context.SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount;
SequenceIndexesLeftToFindToRemoteIndex.insert({RemoteSequenceRawHash, RemoteSequenceIndex});
}
}
- CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs();
+ Stats.CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs();
}
else
{
- for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size();
+ for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < InContext.RemoteContent.ChunkedContent.SequenceRawHashes.size();
RemoteSequenceIndex++)
{
- const uint32_t ChunkCount = RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex];
- SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount;
+ const uint32_t ChunkCount = InContext.RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex];
+ Context.SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount;
}
}
@@ -5661,7 +5708,7 @@ namespace {
std::vector<ScavengeCopyOperation> ScavengeCopyOperations;
uint64_t ScavengedPathsCount = 0;
- if (!PrimeCacheOnly && EnableScavenging)
+ if (!InContext.Base.PrimeCacheOnly && InContext.Base.EnableScavenging)
{
ZEN_TRACE_CPU("UpdateFolder_GetScavengedSequences");
@@ -5669,10 +5716,10 @@ namespace {
if (!SequenceIndexesLeftToFindToRemoteIndex.empty())
{
- std::vector<ScavengeSource> ScavengeSources = GetDownloadedStatePaths(SystemRootDir);
- auto EraseIt = std::remove_if(ScavengeSources.begin(), ScavengeSources.end(), [&Path](const ScavengeSource& Source) {
- return Source.Path == Path;
- });
+ std::vector<ScavengeSource> ScavengeSources = GetDownloadedStatePaths(InContext.Base.SystemRootDir);
+ auto EraseIt = std::remove_if(ScavengeSources.begin(),
+ ScavengeSources.end(),
+ [Path = InContext.Base.Path](const ScavengeSource& Source) { return Source.Path == Path; });
ScavengeSources.erase(EraseIt, ScavengeSources.end());
const size_t ScavengePathCount = ScavengeSources.size();
@@ -5736,7 +5783,7 @@ namespace {
{
const uint32_t RemoteSequenceIndex = It->second;
const uint64_t RawSize =
- RemoteContent.RawSizes[RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]];
+ InContext.RemoteContent.RawSizes[Context.RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]];
ZEN_ASSERT(RawSize > 0);
const uint32_t ScavengedPathIndex = ScavengedLookup.SequenceIndexFirstPathIndex[ScavengedSequenceIndex];
@@ -5748,23 +5795,25 @@ namespace {
.RawSize = RawSize});
SequenceIndexesLeftToFindToRemoteIndex.erase(SequenceRawHash);
- SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = 0;
+ Context.SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = 0;
- CacheMappingStats.ScavengedPathsMatchingSequencesCount++;
- CacheMappingStats.ScavengedPathsMatchingSequencesByteCount += RawSize;
+ Stats.CacheMappingStats.ScavengedPathsMatchingSequencesCount++;
+ Stats.CacheMappingStats.ScavengedPathsMatchingSequencesByteCount += RawSize;
}
}
ScavengedPathsCount++;
}
}
}
- CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs();
+ Stats.CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs();
}
uint32_t RemainingChunkCount = 0;
- for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++)
+ for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < InContext.RemoteContent.ChunkedContent.ChunkHashes.size();
+ RemoteChunkIndex++)
{
- uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex);
+ uint64_t ChunkWriteCount =
+ GetChunkWriteCount(Context.SequenceIndexChunksLeftToWriteCounters, Context.RemoteLookup, RemoteChunkIndex);
if (ChunkWriteCount > 0)
{
RemainingChunkCount++;
@@ -5790,36 +5839,39 @@ namespace {
tsl::robin_map<IoHash, size_t, IoHash::Hasher> RawHashToCacheCopyDataIndex;
std::vector<CacheCopyData> CacheCopyDatas;
- if (!PrimeCacheOnly)
+ if (!InContext.Base.PrimeCacheOnly)
{
ZEN_TRACE_CPU("UpdateFolder_GetLocalChunks");
Stopwatch LocalTimer;
for (uint32_t LocalSequenceIndex = 0;
- LocalSequenceIndex < LocalContent.ChunkedContent.SequenceRawHashes.size() && (RemainingChunkCount > 0);
+ LocalSequenceIndex < InContext.LocalContent.ChunkedContent.SequenceRawHashes.size() && (RemainingChunkCount > 0);
LocalSequenceIndex++)
{
- const IoHash& LocalSequenceRawHash = LocalContent.ChunkedContent.SequenceRawHashes[LocalSequenceIndex];
- const uint32_t LocalOrderOffset = LocalLookup.SequenceIndexChunkOrderOffset[LocalSequenceIndex];
+ const IoHash& LocalSequenceRawHash = InContext.LocalContent.ChunkedContent.SequenceRawHashes[LocalSequenceIndex];
+ const uint32_t LocalOrderOffset = Context.LocalLookup.SequenceIndexChunkOrderOffset[LocalSequenceIndex];
{
uint64_t SourceOffset = 0;
- const uint32_t LocalChunkCount = LocalContent.ChunkedContent.ChunkCounts[LocalSequenceIndex];
+ const uint32_t LocalChunkCount = InContext.LocalContent.ChunkedContent.ChunkCounts[LocalSequenceIndex];
for (uint32_t LocalOrderIndex = 0; LocalOrderIndex < LocalChunkCount; LocalOrderIndex++)
{
- const uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[LocalOrderOffset + LocalOrderIndex];
- const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
- const uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex];
+ const uint32_t LocalChunkIndex =
+ InContext.LocalContent.ChunkedContent.ChunkOrders[LocalOrderOffset + LocalOrderIndex];
+ const IoHash& LocalChunkHash = InContext.LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
+ const uint64_t LocalChunkRawSize = InContext.LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex];
- if (auto RemoteChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(LocalChunkHash);
- RemoteChunkIt != RemoteLookup.ChunkHashToChunkIndex.end())
+ if (auto RemoteChunkIt = Context.RemoteLookup.ChunkHashToChunkIndex.find(LocalChunkHash);
+ RemoteChunkIt != Context.RemoteLookup.ChunkHashToChunkIndex.end())
{
const uint32_t RemoteChunkIndex = RemoteChunkIt->second;
- if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
+ if (!Context.RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
{
std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs =
- GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex);
+ GetRemainingChunkTargets(Context.SequenceIndexChunksLeftToWriteCounters,
+ Context.RemoteLookup,
+ RemoteChunkIndex);
if (!ChunkTargetPtrs.empty())
{
@@ -5857,9 +5909,9 @@ namespace {
.TargetChunkLocationPtrs = ChunkTargetPtrs,
.ChunkTargets = std::vector<CacheCopyData::ChunkTarget>{Target}});
}
- CacheMappingStats.LocalChunkMatchingRemoteCount++;
- CacheMappingStats.LocalChunkMatchingRemoteByteCount += LocalChunkRawSize;
- RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true;
+ Stats.CacheMappingStats.LocalChunkMatchingRemoteCount++;
+ Stats.CacheMappingStats.LocalChunkMatchingRemoteByteCount += LocalChunkRawSize;
+ Context.RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true;
RemainingChunkCount--;
}
}
@@ -5868,10 +5920,10 @@ namespace {
}
}
}
- CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs();
+ Stats.CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs();
}
- if (!PrimeCacheOnly)
+ if (!InContext.Base.PrimeCacheOnly)
{
ZEN_TRACE_CPU("UpdateFolder_GetScavengeChunks");
@@ -5901,14 +5953,16 @@ namespace {
const IoHash& ScavengedChunkHash = ScavengedContent.ChunkedContent.ChunkHashes[ScavengedChunkIndex];
const uint64_t ScavengedChunkRawSize = ScavengedContent.ChunkedContent.ChunkRawSizes[ScavengedChunkIndex];
- if (auto RemoteChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(ScavengedChunkHash);
- RemoteChunkIt != RemoteLookup.ChunkHashToChunkIndex.end())
+ if (auto RemoteChunkIt = Context.RemoteLookup.ChunkHashToChunkIndex.find(ScavengedChunkHash);
+ RemoteChunkIt != Context.RemoteLookup.ChunkHashToChunkIndex.end())
{
const uint32_t RemoteChunkIndex = RemoteChunkIt->second;
- if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
+ if (!Context.RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
{
std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs =
- GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex);
+ GetRemainingChunkTargets(Context.SequenceIndexChunksLeftToWriteCounters,
+ Context.RemoteLookup,
+ RemoteChunkIndex);
if (!ChunkTargetPtrs.empty())
{
@@ -5947,9 +6001,9 @@ namespace {
.TargetChunkLocationPtrs = ChunkTargetPtrs,
.ChunkTargets = std::vector<CacheCopyData::ChunkTarget>{Target}});
}
- CacheMappingStats.ScavengedChunkMatchingRemoteCount++;
- CacheMappingStats.ScavengedChunkMatchingRemoteByteCount += ScavengedChunkRawSize;
- RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true;
+ Stats.CacheMappingStats.ScavengedChunkMatchingRemoteCount++;
+ Stats.CacheMappingStats.ScavengedChunkMatchingRemoteByteCount += ScavengedChunkRawSize;
+ Context.RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true;
RemainingChunkCount--;
}
}
@@ -5959,52 +6013,56 @@ namespace {
}
}
}
- CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs();
+ Stats.CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs();
}
if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty())
{
ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}",
CachedSequenceHashesFound.size(),
- NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount),
+ NiceBytes(Stats.CacheMappingStats.CacheSequenceHashesByteCount),
CachedChunkHashesFound.size(),
- NiceBytes(CacheMappingStats.CacheChunkByteCount),
+ NiceBytes(Stats.CacheMappingStats.CacheChunkByteCount),
CachedBlocksFound.size(),
- NiceBytes(CacheMappingStats.CacheBlocksByteCount),
- NiceTimeSpanMs(CacheMappingStats.CacheScanElapsedWallTimeUs / 1000));
+ NiceBytes(Stats.CacheMappingStats.CacheBlocksByteCount),
+ NiceTimeSpanMs(Stats.CacheMappingStats.CacheScanElapsedWallTimeUs / 1000));
}
- if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0)
+ if (Stats.CacheMappingStats.LocalPathIndexesMatchingSequenceIndexCount > 0 ||
+ Stats.CacheMappingStats.LocalChunkMatchingRemoteCount > 0)
{
ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}",
- LocalPathIndexesMatchingSequenceIndexes.size(),
- NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount),
- CacheMappingStats.LocalChunkMatchingRemoteCount,
- NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount),
- NiceTimeSpanMs(CacheMappingStats.LocalScanElapsedWallTimeUs / 1000));
+ Stats.CacheMappingStats.LocalPathIndexesMatchingSequenceIndexCount,
+ NiceBytes(Stats.CacheMappingStats.LocalPathsMatchingSequencesByteCount),
+ Stats.CacheMappingStats.LocalChunkMatchingRemoteCount,
+ NiceBytes(Stats.CacheMappingStats.LocalChunkMatchingRemoteByteCount),
+ NiceTimeSpanMs(Stats.CacheMappingStats.LocalScanElapsedWallTimeUs / 1000));
}
- if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0)
+ if (Stats.CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 ||
+ Stats.CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0)
{
ZEN_CONSOLE("Scavenge of {} paths found {} ({}) chunk sequences, {} ({}) chunks in {}",
ScavengedPathsCount,
- CacheMappingStats.ScavengedPathsMatchingSequencesCount,
- NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount),
- CacheMappingStats.ScavengedChunkMatchingRemoteCount,
- NiceBytes(CacheMappingStats.ScavengedChunkMatchingRemoteByteCount),
- NiceTimeSpanMs(CacheMappingStats.ScavengeElapsedWallTimeUs / 1000));
+ Stats.CacheMappingStats.ScavengedPathsMatchingSequencesCount,
+ NiceBytes(Stats.CacheMappingStats.ScavengedPathsMatchingSequencesByteCount),
+ Stats.CacheMappingStats.ScavengedChunkMatchingRemoteCount,
+ NiceBytes(Stats.CacheMappingStats.ScavengedChunkMatchingRemoteByteCount),
+ NiceTimeSpanMs(Stats.CacheMappingStats.ScavengeElapsedWallTimeUs / 1000));
}
uint64_t BytesToWrite = 0;
- for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++)
+ for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < InContext.RemoteContent.ChunkedContent.ChunkHashes.size();
+ RemoteChunkIndex++)
{
- uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex);
+ uint64_t ChunkWriteCount =
+ GetChunkWriteCount(Context.SequenceIndexChunksLeftToWriteCounters, Context.RemoteLookup, RemoteChunkIndex);
if (ChunkWriteCount > 0)
{
- BytesToWrite += RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] * ChunkWriteCount;
- if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
+ BytesToWrite += InContext.RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] * ChunkWriteCount;
+ if (!Context.RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
{
- RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex] = true;
+ Context.RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex] = true;
}
}
}
@@ -6014,10 +6072,6 @@ namespace {
BytesToWrite += ScavengeCopyOp.RawSize;
}
- uint64_t TotalRequestCount = 0;
- uint64_t TotalPartWriteCount = 0;
- std::atomic<uint64_t> WritePartsComplete = 0;
-
{
ZEN_TRACE_CPU("WriteChunks");
@@ -6029,7 +6083,7 @@ namespace {
WorkerThreadPool& NetworkPool = GetNetworkPool();
WorkerThreadPool& WritePool = GetIOWorkerPool();
- ProgressBar WriteProgressBar(ProgressMode, PrimeCacheOnly ? "Downloading" : "Writing");
+ ProgressBar WriteProgressBar(ProgressMode, InContext.Base.PrimeCacheOnly ? "Downloading" : "Writing");
ParallelWork Work(AbortFlag);
struct LooseChunkHashWorkData
@@ -6039,24 +6093,24 @@ namespace {
};
std::vector<LooseChunkHashWorkData> LooseChunkHashWorks;
- TotalPartWriteCount += CacheCopyDatas.size();
- TotalPartWriteCount += ScavengeCopyOperations.size();
+ Context.TotalPartWriteCount += CacheCopyDatas.size();
+ Context.TotalPartWriteCount += ScavengeCopyOperations.size();
- for (const IoHash ChunkHash : LooseChunkHashes)
+ for (const IoHash ChunkHash : InContext.LooseChunkHashes)
{
- auto RemoteChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash);
- ZEN_ASSERT(RemoteChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end());
+ auto RemoteChunkIndexIt = Context.RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash);
+ ZEN_ASSERT(RemoteChunkIndexIt != Context.RemoteLookup.ChunkHashToChunkIndex.end());
const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second;
- if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
+ if (Context.RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
{
ZEN_CONSOLE_VERBOSE("Skipping chunk {} due to cache reuse", ChunkHash);
continue;
}
bool NeedsCopy = true;
- if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false))
+ if (Context.RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false))
{
std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs =
- GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex);
+ GetRemainingChunkTargets(Context.SequenceIndexChunksLeftToWriteCounters, Context.RemoteLookup, RemoteChunkIndex);
if (ChunkTargetPtrs.empty())
{
@@ -6064,41 +6118,40 @@ namespace {
}
else
{
- TotalRequestCount++;
- TotalPartWriteCount++;
+ Context.TotalRequestCount++;
+ Context.TotalPartWriteCount++;
LooseChunkHashWorks.push_back(
LooseChunkHashWorkData{.ChunkTargetPtrs = ChunkTargetPtrs, .RemoteChunkIndex = RemoteChunkIndex});
}
}
}
- uint32_t BlockCount = gsl::narrow<uint32_t>(BlockDescriptions.size());
-
- std::vector<bool> ChunkIsPickedUpByBlock(RemoteContent.ChunkedContent.ChunkHashes.size(), false);
- auto GetNeededChunkBlockIndexes = [&RemoteContent,
- &RemoteLookup,
- &RemoteChunkIndexNeedsCopyFromSourceFlags,
- &ChunkIsPickedUpByBlock](const ChunkBlockDescription& BlockDescription) {
- ZEN_TRACE_CPU("UpdateFolder_GetNeededChunkBlockIndexes");
- std::vector<uint32_t> NeededBlockChunkIndexes;
- for (uint32_t ChunkBlockIndex = 0; ChunkBlockIndex < BlockDescription.ChunkRawHashes.size(); ChunkBlockIndex++)
- {
- const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex];
- if (auto It = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != RemoteLookup.ChunkHashToChunkIndex.end())
- {
- const uint32_t RemoteChunkIndex = It->second;
- if (!ChunkIsPickedUpByBlock[RemoteChunkIndex])
- {
- if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex])
- {
- ChunkIsPickedUpByBlock[RemoteChunkIndex] = true;
- NeededBlockChunkIndexes.push_back(ChunkBlockIndex);
- }
- }
- }
- }
- return NeededBlockChunkIndexes;
- };
+ uint32_t BlockCount = gsl::narrow<uint32_t>(InContext.BlockDescriptions.size());
+
+ std::vector<bool> ChunkIsPickedUpByBlock(InContext.RemoteContent.ChunkedContent.ChunkHashes.size(), false);
+ auto GetNeededChunkBlockIndexes =
+ [&InContext, &Context, &ChunkIsPickedUpByBlock](const ChunkBlockDescription& BlockDescription) {
+ ZEN_TRACE_CPU("UpdateFolder_GetNeededChunkBlockIndexes");
+ std::vector<uint32_t> NeededBlockChunkIndexes;
+ for (uint32_t ChunkBlockIndex = 0; ChunkBlockIndex < BlockDescription.ChunkRawHashes.size(); ChunkBlockIndex++)
+ {
+ const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex];
+ if (auto It = Context.RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash);
+ It != Context.RemoteLookup.ChunkHashToChunkIndex.end())
+ {
+ const uint32_t RemoteChunkIndex = It->second;
+ if (!ChunkIsPickedUpByBlock[RemoteChunkIndex])
+ {
+ if (Context.RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex])
+ {
+ ChunkIsPickedUpByBlock[RemoteChunkIndex] = true;
+ NeededBlockChunkIndexes.push_back(ChunkBlockIndex);
+ }
+ }
+ }
+ }
+ return NeededBlockChunkIndexes;
+ };
std::vector<uint32_t> CachedChunkBlockIndexes;
@@ -6116,14 +6169,14 @@ namespace {
for (uint32_t BlockIndex = 0; BlockIndex < BlockCount; BlockIndex++)
{
- const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+ const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex];
const std::vector<uint32_t> BlockChunkIndexNeeded = GetNeededChunkBlockIndexes(BlockDescription);
if (!BlockChunkIndexNeeded.empty())
{
- if (PrimeCacheOnly)
+ if (InContext.Base.PrimeCacheOnly)
{
- TotalRequestCount++;
- TotalPartWriteCount++;
+ Context.TotalRequestCount++;
+ Context.TotalPartWriteCount++;
FullBlockWorks.push_back(BlockIndex);
}
@@ -6134,10 +6187,10 @@ namespace {
{
ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_CacheGet");
- TotalPartWriteCount++;
+ Context.TotalPartWriteCount++;
std::filesystem::path BlockPath =
- ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
+ ZenTempBlockFolderPath(InContext.Base.ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
if (IsFile(BlockPath))
{
CachedChunkBlockIndexes.push_back(BlockIndex);
@@ -6151,7 +6204,7 @@ namespace {
bool CanDoPartialBlockDownload =
(BlockDescription.HeaderSize > 0) &&
(BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size());
- if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload)
+ if (InContext.Base.AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload)
{
std::vector<BlockRangeDescriptor> BlockRanges;
@@ -6239,8 +6292,8 @@ namespace {
NiceBytes(WantedSize),
BlockDescription.BlockHash,
NiceBytes(TotalBlockSize));
- TotalRequestCount++;
- TotalPartWriteCount++;
+ Context.TotalRequestCount++;
+ Context.TotalPartWriteCount++;
FullBlockWorks.push_back(BlockIndex);
}
@@ -6252,8 +6305,8 @@ namespace {
BlockDescription.BlockHash,
NiceBytes(TotalBlockSize),
CollapsedBlockRanges.size());
- TotalRequestCount++;
- TotalPartWriteCount++;
+ Context.TotalRequestCount++;
+ Context.TotalPartWriteCount++;
FullBlockWorks.push_back(BlockIndex);
}
@@ -6265,8 +6318,8 @@ namespace {
BlockDescription.BlockHash,
NiceBytes(TotalBlockSize),
CollapsedBlockRanges.size());
- TotalRequestCount++;
- TotalPartWriteCount++;
+ Context.TotalRequestCount++;
+ Context.TotalPartWriteCount++;
FullBlockWorks.push_back(BlockIndex);
}
@@ -6278,8 +6331,8 @@ namespace {
BlockDescription.BlockHash,
NiceBytes(TotalBlockSize),
CollapsedBlockRanges.size());
- TotalRequestCount++;
- TotalPartWriteCount++;
+ Context.TotalRequestCount++;
+ Context.TotalPartWriteCount++;
FullBlockWorks.push_back(BlockIndex);
}
@@ -6291,8 +6344,8 @@ namespace {
BlockDescription.BlockHash,
NiceBytes(TotalBlockSize),
CollapsedBlockRanges.size());
- TotalRequestCount++;
- TotalPartWriteCount++;
+ Context.TotalRequestCount++;
+ Context.TotalPartWriteCount++;
FullBlockWorks.push_back(BlockIndex);
}
@@ -6307,16 +6360,16 @@ namespace {
NiceBytes(TotalBlockSize),
CollapsedBlockRanges.size());
}
- TotalRequestCount += CollapsedBlockRanges.size();
- TotalPartWriteCount += CollapsedBlockRanges.size();
+ Context.TotalRequestCount += CollapsedBlockRanges.size();
+ Context.TotalPartWriteCount += CollapsedBlockRanges.size();
BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end());
}
}
else
{
- TotalRequestCount++;
- TotalPartWriteCount++;
+ Context.TotalRequestCount++;
+ Context.TotalPartWriteCount++;
FullBlockWorks.push_back(BlockIndex);
}
@@ -6325,7 +6378,7 @@ namespace {
}
else
{
- ZEN_CONSOLE_VERBOSE("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash);
+ ZEN_CONSOLE_VERBOSE("Skipping block {} due to cache reuse", InContext.BlockDescriptions[BlockIndex].BlockHash);
}
}
@@ -6347,17 +6400,17 @@ namespace {
BlobHashesSet.reserve(LooseChunkHashWorks.size() + FullBlockWorks.size());
for (LooseChunkHashWorkData& LooseChunkHashWork : LooseChunkHashWorks)
{
- BlobHashesSet.insert(RemoteContent.ChunkedContent.ChunkHashes[LooseChunkHashWork.RemoteChunkIndex]);
+ BlobHashesSet.insert(InContext.RemoteContent.ChunkedContent.ChunkHashes[LooseChunkHashWork.RemoteChunkIndex]);
}
for (const BlockRangeDescriptor& BlockRange : BlockRangeWorks)
{
const uint32_t BlockIndex = BlockRange.BlockIndex;
- const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+ const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex];
BlobHashesSet.insert(BlockDescription.BlockHash);
}
for (uint32_t BlockIndex : FullBlockWorks)
{
- const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+ const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex];
BlobHashesSet.insert(BlockDescription.BlockHash);
}
@@ -6365,7 +6418,7 @@ namespace {
{
const std::vector<IoHash> BlobHashes(BlobHashesSet.begin(), BlobHashesSet.end());
const std::vector<BuildStorageCache::BlobExistsResult> CacheExistsResult =
- Storage.BuildCacheStorage->BlobsExists(BuildId, BlobHashes);
+ Storage.BuildCacheStorage->BlobsExists(InContext.Base.BuildId, BlobHashes);
if (CacheExistsResult.size() == BlobHashes.size())
{
@@ -6395,20 +6448,18 @@ namespace {
{
break;
}
- if (!PrimeCacheOnly)
+ if (!InContext.Base.PrimeCacheOnly)
{
Work.ScheduleWork(
WritePool,
- [&RemoteContent,
- &CacheFolderPath,
+ [&InContext,
+ &Context,
&ScavengedPaths,
&ScavengeCopyOperations,
&ScavengedContents,
&FilteredWrittenBytesPerSecond,
ScavengeOpIndex,
- &WritePartsComplete,
- TotalPartWriteCount,
- &DiskStats](std::atomic<bool>&) mutable {
+ &Stats](std::atomic<bool>&) mutable {
if (!AbortFlag)
{
ZEN_TRACE_CPU("UpdateFolder_WriteScavanged");
@@ -6424,19 +6475,23 @@ namespace {
ZEN_ASSERT_SLOW(FileSizeFromPath(ScavengedFilePath) == ScavengeOp.RawSize);
const IoHash& RemoteSequenceRawHash =
- RemoteContent.ChunkedContent.SequenceRawHashes[ScavengeOp.RemoteSequenceIndex];
+ InContext.RemoteContent.ChunkedContent.SequenceRawHashes[ScavengeOp.RemoteSequenceIndex];
const std::filesystem::path TempFilePath =
- GetTempChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash);
+ GetTempChunkedSequenceFileName(Context.CacheFolderPath, RemoteSequenceRawHash);
const uint64_t RawSize = ScavengedContent.RawSizes[ScavengeOp.ScavengedContentIndex];
- CopyFile(ScavengedFilePath, TempFilePath, RawSize, DiskStats.WriteCount, DiskStats.WriteByteCount);
+ CopyFile(ScavengedFilePath,
+ TempFilePath,
+ RawSize,
+ Stats.DiskStats.WriteCount,
+ Stats.DiskStats.WriteByteCount);
const std::filesystem::path CacheFilePath =
- GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash);
+ GetFinalChunkedSequenceFileName(Context.CacheFolderPath, RemoteSequenceRawHash);
RenameFile(TempFilePath, CacheFilePath);
- WritePartsComplete++;
- if (WritePartsComplete == TotalPartWriteCount)
+ Context.WritePartsComplete++;
+ if (Context.WritePartsComplete == Context.TotalPartWriteCount)
{
FilteredWrittenBytesPerSecond.Stop();
}
@@ -6458,48 +6513,36 @@ namespace {
std::move(LooseChunkHashWork.ChunkTargetPtrs);
const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex;
- if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]))
+ if (InContext.Base.PrimeCacheOnly &&
+ ExistsResult.ExistingBlobs.contains(InContext.RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]))
{
- DownloadStats.RequestsCompleteCount++;
+ Stats.DownloadStats.RequestsCompleteCount++;
continue;
}
Work.ScheduleWork(
WritePool,
[&Storage,
- &Path,
- &ZenFolderPath,
- &RemoteContent,
- &RemoteLookup,
- &CacheFolderPath,
- &SequenceIndexChunksLeftToWriteCounters,
+ &InContext,
+ &Context,
&Work,
&WritePool,
&NetworkPool,
- PrimeCacheOnly,
&ExistsResult,
- &DiskStats,
- &DownloadStats,
- &WriteChunkStats,
- &WritePartsComplete,
+ &Stats,
RemoteChunkIndex,
ChunkTargetPtrs,
- BuildId = Oid(BuildId),
- LargeAttachmentSize,
- PreferredMultipartChunkSize,
- TotalRequestCount,
- TotalPartWriteCount,
&FilteredDownloadedBytesPerSecond,
&FilteredWrittenBytesPerSecond](std::atomic<bool>&) mutable {
if (!AbortFlag)
{
ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded");
std::filesystem::path ExistingCompressedChunkPath;
- if (!PrimeCacheOnly)
+ if (!InContext.Base.PrimeCacheOnly)
{
- const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
+ const IoHash& ChunkHash = InContext.RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
std::filesystem::path CompressedChunkPath =
- ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString();
+ ZenTempDownloadFolderPath(InContext.Base.ZenFolderPath) / ChunkHash.ToHexString();
if (IsFile(CompressedChunkPath))
{
IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath);
@@ -6509,8 +6552,8 @@ namespace {
uint64_t RawSize;
if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart, RawHash, RawSize))
{
- DownloadStats.RequestsCompleteCount++;
- if (DownloadStats.RequestsCompleteCount == TotalRequestCount)
+ Stats.DownloadStats.RequestsCompleteCount++;
+ if (Stats.DownloadStats.RequestsCompleteCount == Context.TotalRequestCount)
{
FilteredDownloadedBytesPerSecond.Stop();
}
@@ -6531,18 +6574,11 @@ namespace {
{
Work.ScheduleWork(
WritePool,
- [&Path,
- &ZenFolderPath,
- &RemoteContent,
- &RemoteLookup,
- &CacheFolderPath,
- &SequenceIndexChunksLeftToWriteCounters,
+ [&InContext,
+ &Context,
&Work,
&WritePool,
- &DiskStats,
- &WriteChunkStats,
- &WritePartsComplete,
- TotalPartWriteCount,
+ &Stats,
&FilteredWrittenBytesPerSecond,
RemoteChunkIndex,
ChunkTargetPtrs,
@@ -6553,7 +6589,8 @@ namespace {
FilteredWrittenBytesPerSecond.Start();
- const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
+ const IoHash& ChunkHash =
+ InContext.RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath);
if (!CompressedPart)
@@ -6564,19 +6601,18 @@ namespace {
CompressedChunkPath));
}
- std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath);
- bool NeedHashVerify = WriteCompressedChunk(TargetFolder,
- RemoteContent,
- RemoteLookup,
- ChunkHash,
- ChunkTargetPtrs,
- std::move(CompressedPart),
- DiskStats);
- WritePartsComplete++;
+ bool NeedHashVerify = WriteCompressedChunk(Context.CacheFolderPath,
+ InContext.RemoteContent,
+ Context.RemoteLookup,
+ ChunkHash,
+ ChunkTargetPtrs,
+ std::move(CompressedPart),
+ Stats.DiskStats);
+ Context.WritePartsComplete++;
if (!AbortFlag)
{
- if (WritePartsComplete == TotalPartWriteCount)
+ if (Context.WritePartsComplete == Context.TotalPartWriteCount)
{
FilteredWrittenBytesPerSecond.Stop();
}
@@ -6584,19 +6620,22 @@ namespace {
TryRemoveFile(CompressedChunkPath);
std::vector<uint32_t> CompletedSequences =
- CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters);
+ CompleteChunkTargets(ChunkTargetPtrs,
+ Context.SequenceIndexChunksLeftToWriteCounters);
if (NeedHashVerify)
{
- VerifyAndCompleteChunkSequencesAsync(TargetFolder,
- RemoteContent,
- RemoteLookup,
+ VerifyAndCompleteChunkSequencesAsync(Context.CacheFolderPath,
+ InContext.RemoteContent,
+ Context.RemoteLookup,
CompletedSequences,
Work,
WritePool);
}
else
{
- FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences);
+ FinalizeChunkSequences(Context.CacheFolderPath,
+ InContext.RemoteContent,
+ CompletedSequences);
}
}
}
@@ -6606,140 +6645,123 @@ namespace {
{
Work.ScheduleWork(
NetworkPool,
- [&Path,
- &ZenFolderPath,
- &Storage,
- BuildId = Oid(BuildId),
- &PrimeCacheOnly,
- &RemoteContent,
- &RemoteLookup,
+ [&Storage,
+ &InContext,
+ &Context,
&ExistsResult,
- &SequenceIndexChunksLeftToWriteCounters,
&Work,
&WritePool,
&NetworkPool,
- &DiskStats,
- &WriteChunkStats,
- &WritePartsComplete,
- TotalPartWriteCount,
- TotalRequestCount,
+ &Stats,
&FilteredDownloadedBytesPerSecond,
&FilteredWrittenBytesPerSecond,
- LargeAttachmentSize,
- PreferredMultipartChunkSize,
RemoteChunkIndex,
- ChunkTargetPtrs,
- &DownloadStats](std::atomic<bool>&) mutable {
+ ChunkTargetPtrs](std::atomic<bool>&) mutable {
if (!AbortFlag)
{
- const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
+ const IoHash& ChunkHash =
+ InContext.RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
FilteredDownloadedBytesPerSecond.Start();
IoBuffer BuildBlob;
const bool ExistsInCache =
Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash);
if (ExistsInCache)
{
- BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash);
+ BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(InContext.Base.BuildId, ChunkHash);
}
if (BuildBlob)
{
uint64_t BlobSize = BuildBlob.GetSize();
- DownloadStats.DownloadedChunkCount++;
- DownloadStats.DownloadedChunkByteCount += BlobSize;
- DownloadStats.RequestsCompleteCount++;
- if (DownloadStats.RequestsCompleteCount == TotalRequestCount)
+ Stats.DownloadStats.DownloadedChunkCount++;
+ Stats.DownloadStats.DownloadedChunkByteCount += BlobSize;
+ Stats.DownloadStats.RequestsCompleteCount++;
+ if (Stats.DownloadStats.RequestsCompleteCount == Context.TotalRequestCount)
{
FilteredDownloadedBytesPerSecond.Stop();
}
- AsyncWriteDownloadedChunk(ZenFolderPath,
- RemoteContent,
- RemoteLookup,
+ AsyncWriteDownloadedChunk(InContext.Base.ZenFolderPath,
+ InContext.RemoteContent,
+ Context.RemoteLookup,
RemoteChunkIndex,
std::move(ChunkTargetPtrs),
Work,
WritePool,
std::move(BuildBlob),
- SequenceIndexChunksLeftToWriteCounters,
- WritePartsComplete,
- TotalPartWriteCount,
+ Context.SequenceIndexChunksLeftToWriteCounters,
+ Context.WritePartsComplete,
+ Context.TotalPartWriteCount,
FilteredWrittenBytesPerSecond,
- DiskStats);
+ Stats.DiskStats);
}
else
{
- if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize)
+ if (InContext.RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >=
+ InContext.LargeAttachmentSize)
{
ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk");
- DownloadLargeBlob(*Storage.BuildStorage,
- ZenTempDownloadFolderPath(ZenFolderPath),
- BuildId,
- ChunkHash,
- PreferredMultipartChunkSize,
- Work,
- NetworkPool,
- DownloadStats,
- [&Storage,
- &ZenFolderPath,
- &RemoteContent,
- &RemoteLookup,
- BuildId,
- PrimeCacheOnly,
- &SequenceIndexChunksLeftToWriteCounters,
- &Work,
- &WritePool,
- ChunkHash,
- TotalPartWriteCount,
- TotalRequestCount,
- &WritePartsComplete,
- &FilteredWrittenBytesPerSecond,
- &FilteredDownloadedBytesPerSecond,
- &DownloadStats,
- &DiskStats,
- RemoteChunkIndex,
- ChunkTargetPtrs](IoBuffer&& Payload) mutable {
- DownloadStats.RequestsCompleteCount++;
- if (DownloadStats.RequestsCompleteCount == TotalRequestCount)
- {
- FilteredDownloadedBytesPerSecond.Stop();
- }
- if (Payload && Storage.BuildCacheStorage)
- {
- Storage.BuildCacheStorage->PutBuildBlob(
- BuildId,
- ChunkHash,
- ZenContentType::kCompressedBinary,
- CompositeBuffer(SharedBuffer(Payload)));
- }
- if (!PrimeCacheOnly)
- {
- if (!AbortFlag)
- {
- AsyncWriteDownloadedChunk(
- ZenFolderPath,
- RemoteContent,
- RemoteLookup,
- RemoteChunkIndex,
- std::move(ChunkTargetPtrs),
- Work,
- WritePool,
- std::move(Payload),
- SequenceIndexChunksLeftToWriteCounters,
- WritePartsComplete,
- TotalPartWriteCount,
- FilteredWrittenBytesPerSecond,
- DiskStats);
- }
- }
- });
+ DownloadLargeBlob(
+ *Storage.BuildStorage,
+ ZenTempDownloadFolderPath(InContext.Base.ZenFolderPath),
+ InContext.Base.BuildId,
+ ChunkHash,
+ InContext.PreferredMultipartChunkSize,
+ Work,
+ NetworkPool,
+ Stats.DownloadStats,
+ [&Storage,
+ &InContext,
+ &Context,
+ &Work,
+ &WritePool,
+ ChunkHash,
+ &FilteredWrittenBytesPerSecond,
+ &FilteredDownloadedBytesPerSecond,
+ &Stats,
+ RemoteChunkIndex,
+ ChunkTargetPtrs](IoBuffer&& Payload) mutable {
+ Stats.DownloadStats.RequestsCompleteCount++;
+ if (Stats.DownloadStats.RequestsCompleteCount == Context.TotalRequestCount)
+ {
+ FilteredDownloadedBytesPerSecond.Stop();
+ }
+ if (Payload && Storage.BuildCacheStorage)
+ {
+ Storage.BuildCacheStorage->PutBuildBlob(
+ InContext.Base.BuildId,
+ ChunkHash,
+ ZenContentType::kCompressedBinary,
+ CompositeBuffer(SharedBuffer(Payload)));
+ }
+ if (!InContext.Base.PrimeCacheOnly)
+ {
+ if (!AbortFlag)
+ {
+ AsyncWriteDownloadedChunk(
+ InContext.Base.ZenFolderPath,
+ InContext.RemoteContent,
+ Context.RemoteLookup,
+ RemoteChunkIndex,
+ std::move(ChunkTargetPtrs),
+ Work,
+ WritePool,
+ std::move(Payload),
+ Context.SequenceIndexChunksLeftToWriteCounters,
+ Context.WritePartsComplete,
+ Context.TotalPartWriteCount,
+ FilteredWrittenBytesPerSecond,
+ Stats.DiskStats);
+ }
+ }
+ });
}
else
{
ZEN_TRACE_CPU("UpdateFolder_GetChunk");
- BuildBlob = Storage.BuildStorage->GetBuildBlob(BuildId, ChunkHash);
+ BuildBlob = Storage.BuildStorage->GetBuildBlob(InContext.Base.BuildId, ChunkHash);
if (BuildBlob && Storage.BuildCacheStorage)
{
Storage.BuildCacheStorage->PutBuildBlob(
- BuildId,
+ InContext.Base.BuildId,
ChunkHash,
BuildBlob.GetContentType(),
CompositeBuffer(SharedBuffer(BuildBlob)));
@@ -6748,31 +6770,31 @@ namespace {
{
throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash));
}
- if (!PrimeCacheOnly)
+ if (!InContext.Base.PrimeCacheOnly)
{
if (!AbortFlag)
{
uint64_t BlobSize = BuildBlob.GetSize();
- DownloadStats.DownloadedChunkCount++;
- DownloadStats.DownloadedChunkByteCount += BlobSize;
- DownloadStats.RequestsCompleteCount++;
- if (DownloadStats.RequestsCompleteCount == TotalRequestCount)
+ Stats.DownloadStats.DownloadedChunkCount++;
+ Stats.DownloadStats.DownloadedChunkByteCount += BlobSize;
+ Stats.DownloadStats.RequestsCompleteCount++;
+ if (Stats.DownloadStats.RequestsCompleteCount == Context.TotalRequestCount)
{
FilteredDownloadedBytesPerSecond.Stop();
}
- AsyncWriteDownloadedChunk(ZenFolderPath,
- RemoteContent,
- RemoteLookup,
+ AsyncWriteDownloadedChunk(InContext.Base.ZenFolderPath,
+ InContext.RemoteContent,
+ Context.RemoteLookup,
RemoteChunkIndex,
std::move(ChunkTargetPtrs),
Work,
WritePool,
std::move(BuildBlob),
- SequenceIndexChunksLeftToWriteCounters,
- WritePartsComplete,
- TotalPartWriteCount,
+ Context.SequenceIndexChunksLeftToWriteCounters,
+ Context.WritePartsComplete,
+ Context.TotalPartWriteCount,
FilteredWrittenBytesPerSecond,
- DiskStats);
+ Stats.DiskStats);
}
}
}
@@ -6787,7 +6809,7 @@ namespace {
for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++)
{
- ZEN_ASSERT(!PrimeCacheOnly);
+ ZEN_ASSERT(!InContext.Base.PrimeCacheOnly);
if (AbortFlag)
{
break;
@@ -6795,13 +6817,8 @@ namespace {
Work.ScheduleWork(
WritePool,
- [&Path,
- &LocalContent,
- &RemoteContent,
- &RemoteLookup,
- &CacheFolderPath,
- &LocalLookup,
- &SequenceIndexChunksLeftToWriteCounters,
+ [&InContext,
+ &Context,
&Work,
&WritePool,
&FilteredWrittenBytesPerSecond,
@@ -6809,9 +6826,7 @@ namespace {
&ScavengedContents,
&ScavengedLookups,
&ScavengedPaths,
- &WritePartsComplete,
- TotalPartWriteCount,
- &DiskStats,
+ &Stats,
CopyDataIndex](std::atomic<bool>&) {
if (!AbortFlag)
{
@@ -6824,8 +6839,9 @@ namespace {
if (CopyData.ScavengeSourceIndex == (uint32_t)-1)
{
- const uint32_t LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex];
- SourceFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred();
+ const uint32_t LocalPathIndex =
+ Context.LocalLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex];
+ SourceFilePath = (InContext.Base.Path / InContext.LocalContent.Paths[LocalPathIndex]).make_preferred();
}
else
{
@@ -6894,8 +6910,8 @@ namespace {
tsl::robin_set<uint32_t> ChunkIndexesWritten;
- BufferedOpenFile SourceFile(SourceFilePath, DiskStats);
- WriteFileCache OpenFileCache(DiskStats);
+ BufferedOpenFile SourceFile(SourceFilePath, Stats.DiskStats);
+ WriteFileCache OpenFileCache(Stats.DiskStats);
for (size_t WriteOpIndex = 0; WriteOpIndex < WriteOps.size();)
{
if (AbortFlag)
@@ -6905,11 +6921,11 @@ namespace {
const WriteOp& Op = WriteOps[WriteOpIndex];
const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex;
- ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() <=
- RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]);
- ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0);
- const uint32_t RemotePathIndex = RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex];
- const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex];
+ ZEN_ASSERT(Context.SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() <=
+ InContext.RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]);
+ ZEN_ASSERT(Context.SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0);
+ const uint32_t RemotePathIndex = Context.RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex];
+ const uint64_t ChunkSize = InContext.RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex];
uint64_t ReadLength = ChunkSize;
size_t WriteCount = 1;
@@ -6930,7 +6946,8 @@ namespace {
{
break;
}
- const uint64_t NextChunkLength = RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex];
+ const uint64_t NextChunkLength =
+ InContext.RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex];
if (ReadLength + NextChunkLength > 512u * 1024u)
{
break;
@@ -6942,18 +6959,19 @@ namespace {
}
CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ReadLength);
- ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]);
+ ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <=
+ InContext.RemoteContent.RawSizes[RemotePathIndex]);
OpenFileCache.WriteToFile<CompositeBuffer>(
RemoteSequenceIndex,
- [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) {
+ [&InContext, &Context](uint32_t SequenceIndex) {
return GetTempChunkedSequenceFileName(
- CacheFolderPath,
- RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
+ Context.CacheFolderPath,
+ InContext.RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
},
ChunkSource,
Op.Target->Offset,
- RemoteContent.RawSizes[RemotePathIndex]);
+ InContext.RemoteContent.RawSizes[RemotePathIndex]);
CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes?
@@ -6967,21 +6985,21 @@ namespace {
for (const WriteOp& Op : WriteOps)
{
const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex;
- if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters))
+ if (CompleteSequenceChunk(RemoteSequenceIndex, Context.SequenceIndexChunksLeftToWriteCounters))
{
CompletedChunkSequences.push_back(RemoteSequenceIndex);
}
}
- VerifyAndCompleteChunkSequencesAsync(CacheFolderPath,
- RemoteContent,
- RemoteLookup,
+ VerifyAndCompleteChunkSequencesAsync(Context.CacheFolderPath,
+ InContext.RemoteContent,
+ Context.RemoteLookup,
CompletedChunkSequences,
Work,
WritePool);
ZEN_CONSOLE_VERBOSE("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), SourceFilePath);
}
- WritePartsComplete++;
- if (WritePartsComplete == TotalPartWriteCount)
+ Context.WritePartsComplete++;
+ if (Context.WritePartsComplete == Context.TotalPartWriteCount)
{
FilteredWrittenBytesPerSecond.Stop();
}
@@ -6991,7 +7009,7 @@ namespace {
for (uint32_t BlockIndex : CachedChunkBlockIndexes)
{
- ZEN_ASSERT(!PrimeCacheOnly);
+ ZEN_ASSERT(!InContext.Base.PrimeCacheOnly);
if (AbortFlag)
{
break;
@@ -6999,29 +7017,17 @@ namespace {
Work.ScheduleWork(
WritePool,
- [&ZenFolderPath,
- &CacheFolderPath,
- &RemoteContent,
- &RemoteLookup,
- &RemoteChunkIndexNeedsCopyFromSourceFlags,
- &SequenceIndexChunksLeftToWriteCounters,
- &Work,
- &WritePool,
- &BlockDescriptions,
- &FilteredWrittenBytesPerSecond,
- &DiskStats,
- &WritePartsComplete,
- TotalPartWriteCount,
- BlockIndex](std::atomic<bool>&) mutable {
+ [&InContext, &Context, &Work, &WritePool, &FilteredWrittenBytesPerSecond, &Stats, BlockIndex](
+ std::atomic<bool>&) mutable {
if (!AbortFlag)
{
ZEN_TRACE_CPU("UpdateFolder_WriteCachedBlock");
- const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+ const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex];
FilteredWrittenBytesPerSecond.Start();
std::filesystem::path BlockChunkPath =
- ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
+ ZenTempBlockFolderPath(InContext.Base.ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath);
if (!BlockBuffer)
{
@@ -7031,16 +7037,16 @@ namespace {
if (!AbortFlag)
{
- if (!WriteBlockToDisk(CacheFolderPath,
- RemoteContent,
+ if (!WriteBlockToDisk(Context.CacheFolderPath,
+ InContext.RemoteContent,
BlockDescription,
- SequenceIndexChunksLeftToWriteCounters,
+ Context.SequenceIndexChunksLeftToWriteCounters,
Work,
WritePool,
CompositeBuffer(std::move(BlockBuffer)),
- RemoteLookup,
- RemoteChunkIndexNeedsCopyFromSourceFlags,
- DiskStats))
+ Context.RemoteLookup,
+ Context.RemoteChunkIndexNeedsCopyFromSourceFlags,
+ Stats.DiskStats))
{
std::error_code DummyEc;
RemoveFile(BlockChunkPath, DummyEc);
@@ -7049,9 +7055,9 @@ namespace {
TryRemoveFile(BlockChunkPath);
- WritePartsComplete++;
+ Context.WritePartsComplete++;
- if (WritePartsComplete == TotalPartWriteCount)
+ if (Context.WritePartsComplete == Context.TotalPartWriteCount)
{
FilteredWrittenBytesPerSecond.Stop();
}
@@ -7062,7 +7068,7 @@ namespace {
for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++)
{
- ZEN_ASSERT(!PrimeCacheOnly);
+ ZEN_ASSERT(!InContext.Base.PrimeCacheOnly);
if (AbortFlag)
{
break;
@@ -7073,22 +7079,12 @@ namespace {
Work.ScheduleWork(
NetworkPool,
[&Storage,
- &ZenFolderPath,
- BuildId,
- &RemoteLookup,
- &BlockDescriptions,
- &RemoteChunkIndexNeedsCopyFromSourceFlags,
- &CacheFolderPath,
- &RemoteContent,
- &SequenceIndexChunksLeftToWriteCounters,
+ &InContext,
+ &Context,
&ExistsResult,
&FilteredDownloadedBytesPerSecond,
- TotalRequestCount,
- &WritePartsComplete,
- TotalPartWriteCount,
&FilteredWrittenBytesPerSecond,
- &DiskStats,
- &DownloadStats,
+ &Stats,
&Work,
&WritePool,
BlockIndex,
@@ -7097,20 +7093,20 @@ namespace {
{
ZEN_TRACE_CPU("UpdateFolder_GetPartialBlock");
- const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+ const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex];
FilteredDownloadedBytesPerSecond.Start();
IoBuffer BlockBuffer;
if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash))
{
- BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId,
+ BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(InContext.Base.BuildId,
BlockDescription.BlockHash,
BlockRange.RangeStart,
BlockRange.RangeLength);
}
if (!BlockBuffer)
{
- BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId,
+ BlockBuffer = Storage.BuildStorage->GetBuildBlob(InContext.Base.BuildId,
BlockDescription.BlockHash,
BlockRange.RangeStart,
BlockRange.RangeLength);
@@ -7122,10 +7118,10 @@ namespace {
if (!AbortFlag)
{
uint64_t BlockSize = BlockBuffer.GetSize();
- DownloadStats.DownloadedBlockCount++;
- DownloadStats.DownloadedBlockByteCount += BlockSize;
- DownloadStats.RequestsCompleteCount++;
- if (DownloadStats.RequestsCompleteCount == TotalRequestCount)
+ Stats.DownloadStats.DownloadedBlockCount++;
+ Stats.DownloadStats.DownloadedBlockByteCount += BlockSize;
+ Stats.DownloadStats.RequestsCompleteCount++;
+ if (Stats.DownloadStats.RequestsCompleteCount == Context.TotalRequestCount)
{
FilteredDownloadedBytesPerSecond.Stop();
}
@@ -7146,10 +7142,11 @@ namespace {
{
BlockBuffer.SetDeleteOnClose(false);
BlockBuffer = {};
- BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}",
- BlockDescription.BlockHash,
- BlockRange.RangeStart,
- BlockRange.RangeLength);
+ BlockChunkPath = ZenTempBlockFolderPath(InContext.Base.ZenFolderPath) /
+ fmt::format("{}_{:x}_{:x}",
+ BlockDescription.BlockHash,
+ BlockRange.RangeStart,
+ BlockRange.RangeLength);
RenameFile(TempBlobPath, BlockChunkPath, Ec);
if (Ec)
{
@@ -7168,10 +7165,11 @@ namespace {
{
ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock");
// Could not be moved and rather large, lets store it on disk
- BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}",
- BlockDescription.BlockHash,
- BlockRange.RangeStart,
- BlockRange.RangeLength);
+ BlockChunkPath =
+ ZenTempBlockFolderPath(InContext.Base.ZenFolderPath) / fmt::format("{}_{:x}_{:x}",
+ BlockDescription.BlockHash,
+ BlockRange.RangeStart,
+ BlockRange.RangeLength);
TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer);
BlockBuffer = {};
}
@@ -7180,17 +7178,11 @@ namespace {
{
Work.ScheduleWork(
WritePool,
- [&CacheFolderPath,
- &RemoteContent,
- &RemoteLookup,
- &BlockDescriptions,
- &RemoteChunkIndexNeedsCopyFromSourceFlags,
- &SequenceIndexChunksLeftToWriteCounters,
- &WritePartsComplete,
+ [&InContext,
+ &Context,
&Work,
- TotalPartWriteCount,
&WritePool,
- &DiskStats,
+ &Stats,
&FilteredWrittenBytesPerSecond,
BlockIndex,
BlockRange,
@@ -7200,7 +7192,7 @@ namespace {
{
ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock");
- const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+ const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex];
if (BlockChunkPath.empty())
{
@@ -7221,18 +7213,18 @@ namespace {
FilteredWrittenBytesPerSecond.Start();
if (!WritePartialBlockToDisk(
- CacheFolderPath,
- RemoteContent,
+ Context.CacheFolderPath,
+ InContext.RemoteContent,
BlockDescription,
- SequenceIndexChunksLeftToWriteCounters,
+ Context.SequenceIndexChunksLeftToWriteCounters,
Work,
WritePool,
CompositeBuffer(std::move(BlockPartialBuffer)),
BlockRange.ChunkBlockIndexStart,
BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1,
- RemoteLookup,
- RemoteChunkIndexNeedsCopyFromSourceFlags,
- DiskStats))
+ Context.RemoteLookup,
+ Context.RemoteChunkIndexNeedsCopyFromSourceFlags,
+ Stats.DiskStats))
{
std::error_code DummyEc;
RemoveFile(BlockChunkPath, DummyEc);
@@ -7245,8 +7237,8 @@ namespace {
TryRemoveFile(BlockChunkPath);
}
- WritePartsComplete++;
- if (WritePartsComplete == TotalPartWriteCount)
+ Context.WritePartsComplete++;
+ if (Context.WritePartsComplete == Context.TotalPartWriteCount)
{
FilteredWrittenBytesPerSecond.Stop();
}
@@ -7265,41 +7257,29 @@ namespace {
break;
}
- if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(BlockDescriptions[BlockIndex].BlockHash))
+ if (InContext.Base.PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(InContext.BlockDescriptions[BlockIndex].BlockHash))
{
- DownloadStats.RequestsCompleteCount++;
+ Stats.DownloadStats.RequestsCompleteCount++;
continue;
}
Work.ScheduleWork(
NetworkPool,
[&Storage,
- &ZenFolderPath,
- BuildId,
- PrimeCacheOnly,
- &BlockDescriptions,
- &WritePartsComplete,
- TotalPartWriteCount,
+ &InContext,
+ &Context,
&FilteredWrittenBytesPerSecond,
&ExistsResult,
&Work,
&WritePool,
- &RemoteContent,
- &RemoteLookup,
- &CacheFolderPath,
- &RemoteChunkIndexNeedsCopyFromSourceFlags,
- &SequenceIndexChunksLeftToWriteCounters,
&FilteredDownloadedBytesPerSecond,
- &WriteChunkStats,
- &DiskStats,
- &DownloadStats,
- TotalRequestCount,
+ &Stats,
BlockIndex](std::atomic<bool>&) {
if (!AbortFlag)
{
ZEN_TRACE_CPU("UpdateFolder_GetFullBlock");
- const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+ const ChunkBlockDescription& BlockDescription = InContext.BlockDescriptions[BlockIndex];
FilteredDownloadedBytesPerSecond.Start();
@@ -7308,14 +7288,14 @@ namespace {
Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash);
if (ExistsInCache)
{
- BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash);
+ BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(InContext.Base.BuildId, BlockDescription.BlockHash);
}
if (!BlockBuffer)
{
- BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash);
+ BlockBuffer = Storage.BuildStorage->GetBuildBlob(InContext.Base.BuildId, BlockDescription.BlockHash);
if (BlockBuffer && Storage.BuildCacheStorage)
{
- Storage.BuildCacheStorage->PutBuildBlob(BuildId,
+ Storage.BuildCacheStorage->PutBuildBlob(InContext.Base.BuildId,
BlockDescription.BlockHash,
BlockBuffer.GetContentType(),
CompositeBuffer(SharedBuffer(BlockBuffer)));
@@ -7328,15 +7308,15 @@ namespace {
if (!AbortFlag)
{
uint64_t BlockSize = BlockBuffer.GetSize();
- DownloadStats.DownloadedBlockCount++;
- DownloadStats.DownloadedBlockByteCount += BlockSize;
- DownloadStats.RequestsCompleteCount++;
- if (DownloadStats.RequestsCompleteCount == TotalRequestCount)
+ Stats.DownloadStats.DownloadedBlockCount++;
+ Stats.DownloadStats.DownloadedBlockByteCount += BlockSize;
+ Stats.DownloadStats.RequestsCompleteCount++;
+ if (Stats.DownloadStats.RequestsCompleteCount == Context.TotalRequestCount)
{
FilteredDownloadedBytesPerSecond.Stop();
}
- if (!PrimeCacheOnly)
+ if (!InContext.Base.PrimeCacheOnly)
{
std::filesystem::path BlockChunkPath;
@@ -7352,9 +7332,9 @@ namespace {
if (!Ec)
{
BlockBuffer.SetDeleteOnClose(false);
- BlockBuffer = {};
- BlockChunkPath =
- ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
+ BlockBuffer = {};
+ BlockChunkPath = ZenTempBlockFolderPath(InContext.Base.ZenFolderPath) /
+ BlockDescription.BlockHash.ToHexString();
RenameFile(TempBlobPath, BlockChunkPath, Ec);
if (Ec)
{
@@ -7373,7 +7353,8 @@ namespace {
{
ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock");
// Could not be moved and rather large, lets store it on disk
- BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
+ BlockChunkPath =
+ ZenTempBlockFolderPath(InContext.Base.ZenFolderPath) / BlockDescription.BlockHash.ToHexString();
TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer);
BlockBuffer = {};
}
@@ -7383,17 +7364,10 @@ namespace {
Work.ScheduleWork(WritePool,
[&Work,
&WritePool,
- &RemoteContent,
- &RemoteLookup,
- CacheFolderPath,
- &RemoteChunkIndexNeedsCopyFromSourceFlags,
- &SequenceIndexChunksLeftToWriteCounters,
+ &InContext,
+ &Context,
BlockIndex,
- &BlockDescriptions,
- &WriteChunkStats,
- &DiskStats,
- &WritePartsComplete,
- TotalPartWriteCount,
+ &Stats,
&FilteredWrittenBytesPerSecond,
BlockChunkPath,
BlockBuffer = std::move(BlockBuffer)](std::atomic<bool>&) mutable {
@@ -7402,7 +7376,7 @@ namespace {
ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock");
const ChunkBlockDescription& BlockDescription =
- BlockDescriptions[BlockIndex];
+ InContext.BlockDescriptions[BlockIndex];
if (BlockChunkPath.empty())
{
@@ -7422,16 +7396,16 @@ namespace {
}
FilteredWrittenBytesPerSecond.Start();
- if (!WriteBlockToDisk(CacheFolderPath,
- RemoteContent,
+ if (!WriteBlockToDisk(Context.CacheFolderPath,
+ InContext.RemoteContent,
BlockDescription,
- SequenceIndexChunksLeftToWriteCounters,
+ Context.SequenceIndexChunksLeftToWriteCounters,
Work,
WritePool,
CompositeBuffer(std::move(BlockBuffer)),
- RemoteLookup,
- RemoteChunkIndexNeedsCopyFromSourceFlags,
- DiskStats))
+ Context.RemoteLookup,
+ Context.RemoteChunkIndexNeedsCopyFromSourceFlags,
+ Stats.DiskStats))
{
std::error_code DummyEc;
RemoveFile(BlockChunkPath, DummyEc);
@@ -7444,9 +7418,9 @@ namespace {
TryRemoveFile(BlockChunkPath);
}
- WritePartsComplete++;
+ Context.WritePartsComplete++;
- if (WritePartsComplete == TotalPartWriteCount)
+ if (Context.WritePartsComplete == Context.TotalPartWriteCount)
{
FilteredWrittenBytesPerSecond.Stop();
}
@@ -7464,32 +7438,34 @@ namespace {
Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) {
ZEN_UNUSED(IsAborted, PendingWork);
- uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() +
- DownloadStats.DownloadedBlockByteCount.load() +
- +DownloadStats.DownloadedPartialBlockByteCount.load();
- FilteredWrittenBytesPerSecond.Update(DiskStats.WriteByteCount.load());
+ uint64_t DownloadedBytes = Stats.DownloadStats.DownloadedChunkByteCount.load() +
+ Stats.DownloadStats.DownloadedBlockByteCount.load() +
+ Stats.DownloadStats.DownloadedPartialBlockByteCount.load();
+ FilteredWrittenBytesPerSecond.Update(Stats.DiskStats.WriteByteCount.load());
FilteredDownloadedBytesPerSecond.Update(DownloadedBytes);
std::string DownloadRateString =
- (DownloadStats.RequestsCompleteCount == TotalRequestCount)
+ (Stats.DownloadStats.RequestsCompleteCount == Context.TotalRequestCount)
? ""
: fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8));
- std::string WriteDetails = PrimeCacheOnly ? ""
- : fmt::format(" {}/{} ({}B/s) written.",
- NiceBytes(DiskStats.WriteByteCount.load()),
- NiceBytes(BytesToWrite),
- NiceNum(FilteredWrittenBytesPerSecond.GetCurrent()));
+ std::string WriteDetails = InContext.Base.PrimeCacheOnly
+ ? ""
+ : fmt::format(" {}/{} ({}B/s) written.",
+ NiceBytes(Stats.DiskStats.WriteByteCount.load()),
+ NiceBytes(BytesToWrite),
+ NiceNum(FilteredWrittenBytesPerSecond.GetCurrent()));
std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}",
- DownloadStats.RequestsCompleteCount.load(),
- TotalRequestCount,
+ Stats.DownloadStats.RequestsCompleteCount.load(),
+ Context.TotalRequestCount,
NiceBytes(DownloadedBytes),
DownloadRateString,
WriteDetails);
WriteProgressBar.UpdateState(
- {.Task = PrimeCacheOnly ? "Downloading " : "Writing chunks ",
+ {.Task = InContext.Base.PrimeCacheOnly ? "Downloading " : "Writing chunks ",
.Details = Details,
- .TotalCount = PrimeCacheOnly ? TotalRequestCount : BytesToWrite,
- .RemainingCount = PrimeCacheOnly ? (TotalRequestCount - DownloadStats.RequestsCompleteCount.load())
- : (BytesToWrite - DiskStats.WriteByteCount.load())},
+ .TotalCount = InContext.Base.PrimeCacheOnly ? Context.TotalRequestCount : BytesToWrite,
+ .RemainingCount = InContext.Base.PrimeCacheOnly
+ ? (Context.TotalRequestCount - Stats.DownloadStats.RequestsCompleteCount.load())
+ : (BytesToWrite - Stats.DiskStats.WriteByteCount.load())},
false);
});
}
@@ -7504,19 +7480,19 @@ namespace {
WriteProgressBar.Finish();
- if (!PrimeCacheOnly)
+ if (!InContext.Base.PrimeCacheOnly)
{
uint32_t RawSequencesMissingWriteCount = 0;
- for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++)
+ for (uint32_t SequenceIndex = 0; SequenceIndex < Context.SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++)
{
- const auto& SequenceIndexChunksLeftToWriteCounter = SequenceIndexChunksLeftToWriteCounters[SequenceIndex];
+ const auto& SequenceIndexChunksLeftToWriteCounter = Context.SequenceIndexChunksLeftToWriteCounters[SequenceIndex];
if (SequenceIndexChunksLeftToWriteCounter.load() != 0)
{
RawSequencesMissingWriteCount++;
- const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
- const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex];
+ const uint32_t PathIndex = Context.RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
+ const std::filesystem::path& IncompletePath = InContext.RemoteContent.Paths[PathIndex];
ZEN_ASSERT(!IncompletePath.empty());
- const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex];
+ const uint32_t ExpectedSequenceCount = InContext.RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex];
ZEN_CONSOLE("{}: Max count {}, Current count {}",
IncompletePath,
ExpectedSequenceCount,
@@ -7527,29 +7503,30 @@ namespace {
ZEN_ASSERT(RawSequencesMissingWriteCount == 0);
}
- const uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() +
- +DownloadStats.DownloadedPartialBlockByteCount.load();
+ const uint64_t DownloadedBytes = Stats.DownloadStats.DownloadedChunkByteCount.load() +
+ Stats.DownloadStats.DownloadedBlockByteCount.load() +
+ Stats.DownloadStats.DownloadedPartialBlockByteCount.load();
ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}",
NiceBytes(DownloadedBytes),
NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)),
NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000),
- NiceBytes(DiskStats.WriteByteCount.load()),
- NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), DiskStats.WriteByteCount.load())),
+ NiceBytes(Stats.DiskStats.WriteByteCount.load()),
+ NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), Stats.DiskStats.WriteByteCount.load())),
NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000),
NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs()));
- WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs();
- WriteChunkStats.DownloadTimeUs = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS();
- WriteChunkStats.WriteTimeUs = FilteredWrittenBytesPerSecond.GetElapsedTimeUS();
+ Stats.WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs();
+ Stats.WriteChunkStats.DownloadTimeUs = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS();
+ Stats.WriteChunkStats.WriteTimeUs = FilteredWrittenBytesPerSecond.GetElapsedTimeUS();
}
- if (PrimeCacheOnly)
+ if (InContext.Base.PrimeCacheOnly)
{
return;
}
tsl::robin_map<uint32_t, uint32_t> RemotePathIndexToLocalPathIndex;
- RemotePathIndexToLocalPathIndex.reserve(RemoteContent.Paths.size());
+ RemotePathIndexToLocalPathIndex.reserve(InContext.RemoteContent.Paths.size());
tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> SequenceHashToLocalPathIndex;
std::vector<uint32_t> RemoveLocalPathIndexes;
@@ -7564,10 +7541,10 @@ namespace {
tsl::robin_set<IoHash, IoHash::Hasher> CachedRemoteSequences;
tsl::robin_map<std::string, uint32_t> RemotePathToRemoteIndex;
- RemotePathToRemoteIndex.reserve(RemoteContent.Paths.size());
- for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++)
+ RemotePathToRemoteIndex.reserve(InContext.RemoteContent.Paths.size());
+ for (uint32_t RemotePathIndex = 0; RemotePathIndex < InContext.RemoteContent.Paths.size(); RemotePathIndex++)
{
- RemotePathToRemoteIndex.insert({RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex});
+ RemotePathToRemoteIndex.insert({InContext.RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex});
}
std::vector<uint32_t> FilesToCache;
@@ -7579,24 +7556,24 @@ namespace {
std::atomic<uint64_t> CachedByteCount = 0;
uint64_t SkippedCount = 0;
uint64_t DeleteCount = 0;
- for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++)
+ for (uint32_t LocalPathIndex = 0; LocalPathIndex < InContext.LocalContent.Paths.size(); LocalPathIndex++)
{
if (AbortFlag)
{
break;
}
- const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex];
- const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex];
+ const IoHash& RawHash = InContext.LocalContent.RawHashes[LocalPathIndex];
+ const std::filesystem::path& LocalPath = InContext.LocalContent.Paths[LocalPathIndex];
- ZEN_ASSERT_SLOW(IsFile((Path / LocalContent.Paths[LocalPathIndex]).make_preferred()));
+ ZEN_ASSERT_SLOW(IsFile((InContext.Base.Path / InContext.LocalContent.Paths[LocalPathIndex]).make_preferred()));
- if (!WipeTargetFolder)
+ if (!InContext.Base.WipeTargetFolder)
{
if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string());
RemotePathIt != RemotePathToRemoteIndex.end())
{
const uint32_t RemotePathIndex = RemotePathIt->second;
- if (RemoteContent.RawHashes[RemotePathIndex] == RawHash)
+ if (InContext.RemoteContent.RawHashes[RemotePathIndex] == RawHash)
{
// It is already in it's desired place
RemotePathIndexToLocalPathIndex[RemotePathIndex] = LocalPathIndex;
@@ -7614,7 +7591,7 @@ namespace {
PathMismatchCount++;
}
}
- if (RemoteLookup.RawHashToSequenceIndex.contains(RawHash))
+ if (Context.RemoteLookup.RawHashToSequenceIndex.contains(RawHash))
{
if (!CachedRemoteSequences.contains(RawHash))
{
@@ -7629,7 +7606,7 @@ namespace {
SkippedCount++;
}
}
- else if (!WipeTargetFolder)
+ else if (!InContext.Base.WipeTargetFolder)
{
// We don't need it
RemoveLocalPathIndexes.push_back(LocalPathIndex);
@@ -7658,22 +7635,23 @@ namespace {
{
break;
}
- Work.ScheduleWork(
- WritePool,
- [&Path, &LocalContent, &CacheFolderPath, &CachedCount, &CachedByteCount, LocalPathIndex](std::atomic<bool>&) {
- ZEN_TRACE_CPU("UpdateFolder_AsyncCopyToCache");
- if (!AbortFlag)
- {
- const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex];
- const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex];
- const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash);
- ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath));
- const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred();
- RenameFileWithRetry(LocalFilePath, CacheFilePath);
- CachedCount++;
- CachedByteCount += LocalContent.RawSizes[LocalPathIndex];
- }
- });
+ Work.ScheduleWork(WritePool,
+ [&InContext, &Context, &CachedCount, &CachedByteCount, LocalPathIndex](std::atomic<bool>&) {
+ ZEN_TRACE_CPU("UpdateFolder_AsyncCopyToCache");
+ if (!AbortFlag)
+ {
+ const IoHash& RawHash = InContext.LocalContent.RawHashes[LocalPathIndex];
+ const std::filesystem::path& LocalPath = InContext.LocalContent.Paths[LocalPathIndex];
+ const std::filesystem::path CacheFilePath =
+ GetFinalChunkedSequenceFileName(Context.CacheFolderPath, RawHash);
+ ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath));
+ const std::filesystem::path LocalFilePath =
+ (InContext.Base.Path / LocalPath).make_preferred();
+ RenameFileWithRetry(LocalFilePath, CacheFilePath);
+ CachedCount++;
+ CachedByteCount += InContext.LocalContent.RawSizes[LocalPathIndex];
+ }
+ });
}
{
@@ -7712,17 +7690,17 @@ namespace {
}
}
- if (WipeTargetFolder)
+ if (InContext.Base.WipeTargetFolder)
{
ZEN_TRACE_CPU("UpdateFolder_WipeTarget");
Stopwatch Timer;
// Clean target folder
- if (!CleanDirectory(Path, DefaultExcludeFolders))
+ if (!CleanDirectory(InContext.Base.Path, DefaultExcludeFolders))
{
- ZEN_WARN("Some files in {} could not be removed", Path);
+ ZEN_WARN("Some files in {} could not be removed", InContext.Base.Path);
}
- RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs();
+ Stats.RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs();
}
if (AbortFlag)
@@ -7739,10 +7717,10 @@ namespace {
ProgressBar RebuildProgressBar(ProgressMode, "Rebuild State");
ParallelWork Work(AbortFlag);
- OutLocalFolderState.Paths.resize(RemoteContent.Paths.size());
- OutLocalFolderState.RawSizes.resize(RemoteContent.Paths.size());
- OutLocalFolderState.Attributes.resize(RemoteContent.Paths.size());
- OutLocalFolderState.ModificationTicks.resize(RemoteContent.Paths.size());
+ OutLocalFolderState.Paths.resize(InContext.RemoteContent.Paths.size());
+ OutLocalFolderState.RawSizes.resize(InContext.RemoteContent.Paths.size());
+ OutLocalFolderState.Attributes.resize(InContext.RemoteContent.Paths.size());
+ OutLocalFolderState.ModificationTicks.resize(InContext.RemoteContent.Paths.size());
std::atomic<uint64_t> DeletedCount = 0;
@@ -7752,10 +7730,11 @@ namespace {
{
break;
}
- Work.ScheduleWork(WritePool, [&Path, &LocalContent, &DeletedCount, LocalPathIndex](std::atomic<bool>&) {
+ Work.ScheduleWork(WritePool, [&InContext, &DeletedCount, LocalPathIndex](std::atomic<bool>&) {
if (!AbortFlag)
{
- const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred();
+ const std::filesystem::path LocalFilePath =
+ (InContext.Base.Path / InContext.LocalContent.Paths[LocalPathIndex]).make_preferred();
SetFileReadOnlyWithRetry(LocalFilePath, false);
RemoveFileWithRetry(LocalFilePath);
DeletedCount++;
@@ -7772,10 +7751,11 @@ namespace {
};
std::vector<FinalizeTarget> Targets;
- Targets.reserve(RemoteContent.Paths.size());
- for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++)
+ Targets.reserve(InContext.RemoteContent.Paths.size());
+ for (uint32_t RemotePathIndex = 0; RemotePathIndex < InContext.RemoteContent.Paths.size(); RemotePathIndex++)
{
- Targets.push_back(FinalizeTarget{.RawHash = RemoteContent.RawHashes[RemotePathIndex], .RemotePathIndex = RemotePathIndex});
+ Targets.push_back(
+ FinalizeTarget{.RawHash = InContext.RemoteContent.RawHashes[RemotePathIndex], .RemotePathIndex = RemotePathIndex});
}
std::sort(Targets.begin(), Targets.end(), [](const FinalizeTarget& Lhs, const FinalizeTarget& Rhs) {
if (Lhs.RawHash < Rhs.RawHash)
@@ -7805,15 +7785,12 @@ namespace {
Work.ScheduleWork(
WritePool,
- [&Path,
- &LocalContent,
+ [&InContext,
+ &Context,
&SequenceHashToLocalPathIndex,
- &RemoteContent,
- &RemoteLookup,
- &CacheFolderPath,
&Targets,
&RemotePathIndexToLocalPathIndex,
- &RebuildFolderStateStats,
+ &Stats,
&OutLocalFolderState,
BaseTargetOffset = TargetOffset,
TargetCount,
@@ -7832,8 +7809,8 @@ namespace {
{
const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex;
ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash);
- const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex];
- std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred();
+ const std::filesystem::path& TargetPath = InContext.RemoteContent.Paths[RemotePathIndex];
+ std::filesystem::path TargetFilePath = (InContext.Base.Path / TargetPath).make_preferred();
if (!RemotePathIndexToLocalPathIndex[RemotePathIndex])
{
if (IsFileWithRetry(TargetFilePath))
@@ -7848,14 +7825,14 @@ namespace {
OutputFile.Open(TargetFilePath, BasicFile::Mode::kTruncate);
}
OutLocalFolderState.Paths[RemotePathIndex] = TargetPath;
- OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex];
+ OutLocalFolderState.RawSizes[RemotePathIndex] = InContext.RemoteContent.RawSizes[RemotePathIndex];
OutLocalFolderState.Attributes[RemotePathIndex] =
- RemoteContent.Attributes.empty()
+ InContext.RemoteContent.Attributes.empty()
? GetNativeFileAttributes(TargetFilePath)
: SetNativeFileAttributes(TargetFilePath,
- RemoteContent.Platform,
- RemoteContent.Attributes[RemotePathIndex]);
+ InContext.RemoteContent.Platform,
+ InContext.RemoteContent.Attributes[RemotePathIndex]);
OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath);
TargetOffset++;
@@ -7865,10 +7842,10 @@ namespace {
else
{
ZEN_TRACE_CPU("Files");
- ZEN_ASSERT(RemoteLookup.RawHashToSequenceIndex.contains(RawHash));
+ ZEN_ASSERT(Context.RemoteLookup.RawHashToSequenceIndex.contains(RawHash));
const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex;
- const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstRemotePathIndex];
- std::filesystem::path FirstTargetFilePath = (Path / FirstTargetPath).make_preferred();
+ const std::filesystem::path& FirstTargetPath = InContext.RemoteContent.Paths[FirstRemotePathIndex];
+ std::filesystem::path FirstTargetFilePath = (InContext.Base.Path / FirstTargetPath).make_preferred();
if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex);
InPlaceIt != RemotePathIndexToLocalPathIndex.end())
@@ -7891,39 +7868,39 @@ namespace {
{
ZEN_TRACE_CPU("Copy");
const uint32_t LocalPathIndex = InplaceIt->second;
- const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex];
- std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred();
+ const std::filesystem::path& SourcePath = InContext.LocalContent.Paths[LocalPathIndex];
+ std::filesystem::path SourceFilePath = (InContext.Base.Path / SourcePath).make_preferred();
ZEN_ASSERT_SLOW(IsFileWithRetry(SourceFilePath));
ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath);
- const uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex];
+ const uint64_t RawSize = InContext.LocalContent.RawSizes[LocalPathIndex];
std::atomic<uint64_t> WriteCount;
std::atomic<uint64_t> WriteByteCount;
CopyFile(SourceFilePath, FirstTargetFilePath, RawSize, WriteCount, WriteByteCount);
- RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++;
+ Stats.RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++;
}
else
{
ZEN_TRACE_CPU("Rename");
const std::filesystem::path CacheFilePath =
- GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash);
+ GetFinalChunkedSequenceFileName(Context.CacheFolderPath, RawHash);
ZEN_ASSERT_SLOW(IsFileWithRetry(CacheFilePath));
RenameFileWithRetry(CacheFilePath, FirstTargetFilePath);
- RebuildFolderStateStats.FinalizeTreeFilesMovedCount++;
+ Stats.RebuildFolderStateStats.FinalizeTreeFilesMovedCount++;
}
}
OutLocalFolderState.Paths[FirstRemotePathIndex] = FirstTargetPath;
- OutLocalFolderState.RawSizes[FirstRemotePathIndex] = RemoteContent.RawSizes[FirstRemotePathIndex];
+ OutLocalFolderState.RawSizes[FirstRemotePathIndex] = InContext.RemoteContent.RawSizes[FirstRemotePathIndex];
OutLocalFolderState.Attributes[FirstRemotePathIndex] =
- RemoteContent.Attributes.empty()
+ InContext.RemoteContent.Attributes.empty()
? GetNativeFileAttributes(FirstTargetFilePath)
: SetNativeFileAttributes(FirstTargetFilePath,
- RemoteContent.Platform,
- RemoteContent.Attributes[FirstRemotePathIndex]);
+ InContext.RemoteContent.Platform,
+ InContext.RemoteContent.Attributes[FirstRemotePathIndex]);
OutLocalFolderState.ModificationTicks[FirstRemotePathIndex] =
GetModificationTickFromPath(FirstTargetFilePath);
@@ -7934,8 +7911,8 @@ namespace {
{
const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex;
ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash);
- const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex];
- std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred();
+ const std::filesystem::path& TargetPath = InContext.RemoteContent.Paths[RemotePathIndex];
+ std::filesystem::path TargetFilePath = (InContext.Base.Path / TargetPath).make_preferred();
if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex);
InPlaceIt != RemotePathIndexToLocalPathIndex.end())
@@ -7956,22 +7933,22 @@ namespace {
ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath));
ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath);
- const uint64_t RawSize = RemoteContent.RawSizes[RemotePathIndex];
+ const uint64_t RawSize = InContext.RemoteContent.RawSizes[RemotePathIndex];
std::atomic<uint64_t> WriteCount;
std::atomic<uint64_t> WriteByteCount;
CopyFile(FirstTargetFilePath, TargetFilePath, RawSize, WriteCount, WriteByteCount);
- RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++;
+ Stats.RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++;
}
OutLocalFolderState.Paths[RemotePathIndex] = TargetPath;
- OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex];
+ OutLocalFolderState.RawSizes[RemotePathIndex] = InContext.RemoteContent.RawSizes[RemotePathIndex];
OutLocalFolderState.Attributes[RemotePathIndex] =
- RemoteContent.Attributes.empty()
+ InContext.RemoteContent.Attributes.empty()
? GetNativeFileAttributes(TargetFilePath)
: SetNativeFileAttributes(TargetFilePath,
- RemoteContent.Platform,
- RemoteContent.Attributes[RemotePathIndex]);
+ InContext.RemoteContent.Platform,
+ InContext.RemoteContent.Attributes[RemotePathIndex]);
OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath);
TargetOffset++;
@@ -8000,7 +7977,7 @@ namespace {
});
}
- RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs = Timer.GetElapsedTimeUs();
+ Stats.RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs = Timer.GetElapsedTimeUs();
if (AbortFlag)
{
@@ -8114,7 +8091,7 @@ namespace {
std::vector<std::pair<Oid, std::string>> ResolveBuildPartNames(BuildStorage& Storage,
const Oid& BuildId,
- const std::vector<Oid>& BuildPartIds,
+ std::span<const Oid> BuildPartIds,
std::span<const std::string> BuildPartNames,
std::uint64_t& OutPreferredMultipartChunkSize)
{
@@ -8742,19 +8719,7 @@ namespace {
return Result;
};
- void DownloadFolder(StorageInstance& Storage,
- const Oid& BuildId,
- const std::vector<Oid>& BuildPartIds,
- std::span<const std::string> BuildPartNames,
- const std::filesystem::path& Path,
- const std::filesystem::path& ZenFolderPath,
- const std::filesystem::path SystemRootDir,
- bool AllowMultiparts,
- bool AllowPartialBlockRequests,
- bool WipeTargetFolder,
- bool PostDownloadVerify,
- bool PrimeCacheOnly,
- bool EnableScavenging)
+ void DownloadFolder(StorageInstance& Storage, const DownloadFolderContext& Context)
{
ZEN_TRACE_CPU("DownloadFolder");
@@ -8773,23 +8738,26 @@ namespace {
auto EndProgress =
MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); });
- ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests)));
+ ZEN_ASSERT((!Context.PrimeCacheOnly) || (Context.PrimeCacheOnly && (!Context.AllowPartialBlockRequests)));
Stopwatch DownloadTimer;
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount);
- const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath);
+ const std::filesystem::path ZenTempFolder = ZenTempFolderPath(Context.ZenFolderPath);
CreateDirectories(ZenTempFolder);
- CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath));
- CreateDirectories(ZenTempCacheFolderPath(ZenFolderPath));
- CreateDirectories(ZenTempDownloadFolderPath(ZenFolderPath));
+ CreateDirectories(ZenTempBlockFolderPath(Context.ZenFolderPath));
+ CreateDirectories(ZenTempCacheFolderPath(Context.ZenFolderPath));
+ CreateDirectories(ZenTempDownloadFolderPath(Context.ZenFolderPath));
std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
- std::vector<std::pair<Oid, std::string>> AllBuildParts =
- ResolveBuildPartNames(*Storage.BuildStorage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
+ std::vector<std::pair<Oid, std::string>> AllBuildParts = ResolveBuildPartNames(*Storage.BuildStorage,
+ Context.BuildId,
+ Context.BuildPartIds,
+ Context.BuildPartNames,
+ PreferredMultipartChunkSize);
std::vector<ChunkedFolderContent> PartContents;
@@ -8801,18 +8769,18 @@ namespace {
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount);
ChunkedFolderContent RemoteContent =
- GetRemoteContent(Storage, BuildId, AllBuildParts, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes);
+ GetRemoteContent(Storage, Context.BuildId, AllBuildParts, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes);
- const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
+ const std::uint64_t LargeAttachmentSize = Context.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
GetFolderContentStatistics LocalFolderScanStats;
ChunkingStatistics ChunkingStats;
ChunkedFolderContent LocalContent;
FolderContent LocalFolderContent;
- if (!PrimeCacheOnly)
+ if (!Context.PrimeCacheOnly)
{
- if (IsDir(Path))
+ if (IsDir(Context.Path))
{
- if (!WipeTargetFolder)
+ if (!Context.WipeTargetFolder)
{
if (!ChunkController)
{
@@ -8822,8 +8790,8 @@ namespace {
LocalContent = GetLocalContent(LocalFolderScanStats,
ChunkingStats,
- Path,
- ZenStateFilePath(ZenFolderPath),
+ Context.Path,
+ ZenStateFilePath(Context.ZenFolderPath),
*ChunkController,
RemoteContent.Paths,
LocalFolderContent);
@@ -8831,7 +8799,7 @@ namespace {
}
else
{
- CreateDirectories(Path);
+ CreateDirectories(Context.Path);
}
}
if (AbortFlag)
@@ -8887,12 +8855,12 @@ namespace {
NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs()));
Stopwatch WriteStateTimer;
- CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent, Path);
- CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path());
- TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView());
+ CbObject StateObject = CreateStateObject(Context.BuildId, AllBuildParts, PartContents, LocalFolderContent, Context.Path);
+ CreateDirectories(ZenStateFilePath(Context.ZenFolderPath).parent_path());
+ TemporaryFile::SafeWriteFile(ZenStateFilePath(Context.ZenFolderPath), StateObject.GetView());
ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
- AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path);
+ AddDownloadedPath(Context.SystemRootDir, Context.BuildId, AllBuildParts, ZenStateFilePath(Context.ZenFolderPath), Context.Path);
}
else
{
@@ -8901,7 +8869,7 @@ namespace {
{
BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first));
}
- ZEN_CONSOLE("Downloading build {}, parts:{} to '{}'", BuildId, BuildPartString.ToView(), Path);
+ ZEN_CONSOLE("Downloading build {}, parts:{} to '{}'", Context.BuildId, BuildPartString.ToView(), Context.Path);
FolderContent LocalFolderState;
DiskStatistics DiskStats;
@@ -8913,49 +8881,60 @@ namespace {
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount);
- UpdateFolder(SystemRootDir,
- Storage,
- BuildId,
- Path,
- ZenFolderPath,
- LargeAttachmentSize,
- PreferredMultipartChunkSize,
- LocalContent,
- RemoteContent,
- BlockDescriptions,
- LooseChunkHashes,
- AllowPartialBlockRequests,
- WipeTargetFolder,
- PrimeCacheOnly,
- EnableScavenging,
- LocalFolderState,
- DiskStats,
- CacheMappingStats,
- DownloadStats,
- WriteChunkStats,
- RebuildFolderStateStats);
+ UpdateFolderContext UpdateCxt = {.Base = Context,
+
+ .LargeAttachmentSize = LargeAttachmentSize,
+ .PreferredMultipartChunkSize = PreferredMultipartChunkSize,
+
+ .LocalContent = LocalContent,
+ .RemoteContent = RemoteContent,
+
+ .BlockDescriptions = BlockDescriptions,
+ .LooseChunkHashes = LooseChunkHashes};
+
+ UpdateFolderStats UpdateStats{.DiskStats = DiskStats,
+ .CacheMappingStats = CacheMappingStats,
+ .DownloadStats = DownloadStats,
+ .WriteChunkStats = WriteChunkStats,
+ .RebuildFolderStateStats = RebuildFolderStateStats};
+
+ UpdateFolder(Storage, UpdateCxt, LocalFolderState, UpdateStats);
if (!AbortFlag)
{
- if (!PrimeCacheOnly)
+ if (!Context.PrimeCacheOnly)
{
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount);
- VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats);
+ struct VerifyFolderContext
+ {
+ const std::filesystem::path Path;
+
+ const bool PostDownloadVerify;
+
+ const ChunkedFolderContent& RemoteContent;
+ VerifyFolderStatistics VerifyFolderStats;
+ };
+
+ VerifyFolder(RemoteContent, Context.Path, Context.PostDownloadVerify, VerifyFolderStats);
Stopwatch WriteStateTimer;
- CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState, Path);
+ CbObject StateObject = CreateStateObject(Context.BuildId, AllBuildParts, PartContents, LocalFolderState, Context.Path);
- CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path());
- TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView());
+ CreateDirectories(ZenStateFilePath(Context.ZenFolderPath).parent_path());
+ TemporaryFile::SafeWriteFile(ZenStateFilePath(Context.ZenFolderPath), StateObject.GetView());
ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
- AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path);
+ AddDownloadedPath(Context.SystemRootDir,
+ Context.BuildId,
+ AllBuildParts,
+ ZenStateFilePath(Context.ZenFolderPath),
+ Context.Path);
#if 0
ExtendableStringBuilder<1024> SB;
CompactBinaryToJson(StateObject, SB);
- WriteFile(ZenStateFileJsonPath(ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
+ WriteFile(ZenStateFileJsonPath(Context.ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
#endif // 0
}
const uint64_t DownloadCount = DownloadStats.DownloadedChunkCount.load() + DownloadStats.DownloadedBlockCount.load() +
@@ -8972,7 +8951,7 @@ namespace {
" Clean: {}\n"
" Finalize: {}\n"
" Verify: {}",
- BuildId,
+ Context.BuildId,
BuildPartString.ToView(),
NiceTimeSpanMs(DownloadTimeMs),
@@ -8991,7 +8970,7 @@ namespace {
NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000));
}
}
- if (PrimeCacheOnly)
+ if (Context.PrimeCacheOnly)
{
if (Storage.BuildCacheStorage)
{
@@ -9680,13 +9659,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return 0;
}
- std::filesystem::path SystemRootDir;
- auto ParseSystemOptions = [&]() {
- if (m_SystemRootDir.empty())
- {
- m_SystemRootDir = PickDefaultSystemRootDirectory();
- }
- MakeSafeAbsolutePathÍnPlace(m_SystemRootDir);
+ auto ParseSystemOptions = [&]() {
+ if (m_SystemRootDir.empty())
+ {
+ m_SystemRootDir = PickDefaultSystemRootDirectory();
+ }
+ MakeSafeAbsolutePathÍnPlace(m_SystemRootDir);
};
ParseSystemOptions();
@@ -10488,19 +10466,23 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath));
- DownloadFolder(Storage,
- BuildId,
- BuildPartIds,
- m_BuildPartNames,
- m_Path,
- m_ZenFolderPath,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests && !m_PrimeCacheOnly,
- m_Clean,
- m_PostDownloadVerify,
- m_PrimeCacheOnly,
- m_EnableScavenging);
+ DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir,
+
+ .BuildId = BuildId,
+ .BuildPartIds = BuildPartIds,
+ .BuildPartNames = m_BuildPartNames,
+
+ .Path = m_Path,
+ .ZenFolderPath = m_ZenFolderPath,
+
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests && !m_PrimeCacheOnly,
+ .WipeTargetFolder = m_Clean,
+ .PrimeCacheOnly = m_PrimeCacheOnly,
+ .EnableScavenging = m_EnableScavenging,
+ .PostDownloadVerify = m_PostDownloadVerify};
+
+ DownloadFolder(Storage, DownloadFolderCxt);
return AbortFlag ? 11 : 0;
}
@@ -10622,19 +10604,24 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, m_DownloadOptions.help()));
}
- DownloadFolder(Storage,
- BuildId,
- {},
- {},
- m_Path,
- m_ZenFolderPath,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- BuildIdString == m_BuildIds.front(),
- true,
- false,
- m_EnableScavenging);
+
+ DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir,
+
+ .BuildId = BuildId,
+ .BuildPartIds = {},
+ .BuildPartNames = {},
+
+ .Path = m_Path,
+ .ZenFolderPath = m_ZenFolderPath,
+
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .WipeTargetFolder = BuildIdString == m_BuildIds.front(),
+ .PrimeCacheOnly = false,
+ .EnableScavenging = m_EnableScavenging,
+ .PostDownloadVerify = true};
+
+ DownloadFolder(Storage, DownloadFolderCxt);
if (AbortFlag)
{
ZEN_CONSOLE("Download cancelled");
@@ -10741,23 +10728,31 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(Storage,
- BuildId,
- {BuildPartId},
- {},
- DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- true,
- true,
- false,
- m_EnableScavenging);
- if (AbortFlag)
+
+ const std::vector<Oid> BuildPartIds = {BuildPartId};
{
- ZEN_CONSOLE("Download failed.");
- return 11;
+ DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir,
+
+ .BuildId = BuildId,
+ .BuildPartIds = BuildPartIds,
+ .BuildPartNames = {},
+
+ .Path = DownloadPath,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .WipeTargetFolder = true,
+ .PrimeCacheOnly = false,
+ .EnableScavenging = m_EnableScavenging,
+ .PostDownloadVerify = true};
+
+ DownloadFolder(Storage, DownloadFolderCxt);
+ if (AbortFlag)
+ {
+ ZEN_CONSOLE("Download failed.");
+ return 11;
+ }
}
ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)",
@@ -10765,23 +10760,29 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
BuildPartId,
m_BuildPartName,
DownloadPath);
- DownloadFolder(Storage,
- BuildId,
- {BuildPartId},
- {},
- DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging);
- if (AbortFlag)
{
- ZEN_CONSOLE("Re-download failed. (identical target)");
- return 11;
+ DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir,
+
+ .BuildId = BuildId,
+ .BuildPartIds = BuildPartIds,
+ .BuildPartNames = {},
+
+ .Path = DownloadPath,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .WipeTargetFolder = false,
+ .PrimeCacheOnly = false,
+ .EnableScavenging = m_EnableScavenging,
+ .PostDownloadVerify = true};
+
+ DownloadFolder(Storage, DownloadFolderCxt);
+ if (AbortFlag)
+ {
+ ZEN_CONSOLE("Re-download failed. (identical target)");
+ return 11;
+ }
}
auto ScrambleDir = [](const std::filesystem::path& Path) {
@@ -10883,23 +10884,29 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
BuildPartId,
m_BuildPartName,
DownloadPath);
- DownloadFolder(Storage,
- BuildId,
- {BuildPartId},
- {},
- DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging);
- if (AbortFlag)
{
- ZEN_CONSOLE("Re-download failed. (scrambled target)");
- return 11;
+ DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir,
+
+ .BuildId = BuildId,
+ .BuildPartIds = BuildPartIds,
+ .BuildPartNames = {},
+
+ .Path = DownloadPath,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .WipeTargetFolder = false,
+ .PrimeCacheOnly = false,
+ .EnableScavenging = m_EnableScavenging,
+ .PostDownloadVerify = true};
+
+ DownloadFolder(Storage, DownloadFolderCxt);
+ if (AbortFlag)
+ {
+ ZEN_CONSOLE("Re-download failed. (scrambled target)");
+ return 11;
+ }
}
ScrambleDir(DownloadPath);
@@ -10935,83 +10942,94 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(Storage,
- BuildId,
- {BuildPartId},
- {},
- DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging);
- if (AbortFlag)
{
- ZEN_CONSOLE("Re-download failed.");
- return 11;
+ DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir,
+
+ .BuildId = BuildId,
+ .BuildPartIds = BuildPartIds,
+ .BuildPartNames = {},
+
+ .Path = DownloadPath,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .WipeTargetFolder = false,
+ .PrimeCacheOnly = false,
+ .EnableScavenging = m_EnableScavenging,
+ .PostDownloadVerify = true};
+
+ DownloadFolder(Storage, DownloadFolderCxt);
+ if (AbortFlag)
+ {
+ ZEN_CONSOLE("Re-download failed.");
+ return 11;
+ }
}
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
- DownloadFolder(Storage,
- BuildId2,
- {BuildPartId2},
- {},
- DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging);
- if (AbortFlag)
+ const std::vector<Oid> BuildPartIds2 = {BuildPartId2};
{
- ZEN_CONSOLE("Re-download failed.");
- return 11;
- }
+ DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir,
- ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
- DownloadFolder(Storage,
- BuildId2,
- {BuildPartId2},
- {},
- DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging);
- if (AbortFlag)
- {
- ZEN_CONSOLE("Re-download failed.");
- return 11;
+ .BuildId = BuildId2,
+ .BuildPartIds = BuildPartIds2,
+ .BuildPartNames = {},
+
+ .Path = DownloadPath,
+ .ZenFolderPath = DownloadPath / ZenFolderName,
+
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .WipeTargetFolder = false,
+ .PrimeCacheOnly = false,
+ .EnableScavenging = m_EnableScavenging,
+ .PostDownloadVerify = true};
+
+ DownloadFolder(Storage, DownloadFolderCxt);
+ if (AbortFlag)
+ {
+ ZEN_CONSOLE("Re-download failed.");
+ return 11;
+ }
+
+ ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)",
+ BuildId2,
+ BuildPartId2,
+ m_BuildPartName,
+ DownloadPath);
+ DownloadFolder(Storage, DownloadFolderCxt);
+ if (AbortFlag)
+ {
+ ZEN_CONSOLE("Re-download failed.");
+ return 11;
+ }
}
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2);
- DownloadFolder(Storage,
- BuildId,
- {BuildPartId},
- {},
- DownloadPath2,
- DownloadPath2 / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging);
- if (AbortFlag)
{
- ZEN_CONSOLE("Re-download failed.");
- return 11;
+ DownloadFolderContext DownloadFolderCxt = {.SystemRootDir = m_SystemRootDir,
+
+ .BuildId = BuildId,
+ .BuildPartIds = BuildPartIds,
+ .BuildPartNames = {},
+
+ .Path = DownloadPath2,
+ .ZenFolderPath = DownloadPath2 / ZenFolderName,
+
+ .AllowMultiparts = m_AllowMultiparts,
+ .AllowPartialBlockRequests = m_AllowPartialBlockRequests,
+ .WipeTargetFolder = false,
+ .PrimeCacheOnly = false,
+ .EnableScavenging = m_EnableScavenging,
+ .PostDownloadVerify = true};
+
+ DownloadFolder(Storage, DownloadFolderCxt);
+ if (AbortFlag)
+ {
+ ZEN_CONSOLE("Re-download failed.");
+ return 11;
+ }
}
return 0;
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-11 03:43:28 +0000