aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2026-01-20 16:31:50 +0100
committerGitHub Enterprise <[email protected]>2026-01-20 16:31:50 +0100
commit16c8562384ed5ef704b417ba52ec2acbe2a03430 (patch)
treefb34be2facd5b51d32d2ed785ec915da3b1f1d4a
parentzenserver API changes, some other minor changes (#720) (diff)
downloadzen-16c8562384ed5ef704b417ba52ec2acbe2a03430.tar.xz
zen-16c8562384ed5ef704b417ba52ec2acbe2a03430.zip
builds multipart upload (#722)
- Feature: `zen builds upload` now support structure manifest input for `--manifest-path` when the path has a `.json` extension - The structured manifest supports splitting a build into multiple parts { "parts": { "default" : { "partId": "f939f3939939fff3f3202", # optional - used to control the id of each part "files": [ "foo/bar", "baz.exe" ] }, "symbols": { "files": [ "baz.pdb" ] } } }
Diffstat
-rw-r--r--CHANGELOG.md21
-rw-r--r--src/zen/cmds/builds_cmd.cpp896
-rw-r--r--src/zenremotestore/builds/buildstorageoperations.cpp2606
-rw-r--r--src/zenremotestore/chunking/chunkblock.cpp5
-rw-r--r--src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h114
-rw-r--r--src/zenremotestore/include/zenremotestore/chunking/chunkblock.h13
-rw-r--r--src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h19
-rw-r--r--src/zenremotestore/zenremotestore.cpp3
8 files changed, 2240 insertions, 1437 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 47ef6f3e1..0e87d1557 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,25 @@
##
+- Feature: `zen builds upload` now support structure manifest input for `--manifest-path` when the path has a `.json` extension
+ - The structured manifest supports splitting a build into multiple parts
+
+ {
+ "parts": {
+ "default" : {
+ "partId": "f939f3939939fff3f3202", # optional - used to control the id of each part
+ "files": [
+ "foo/bar",
+ "baz.exe"
+ ]
+ },
+ "symbols": {
+ "files": [
+ "baz.pdb"
+ ]
+ }
+ }
+ }
+
+## 5.7.16
- Feature: Added `--exclude-folders` to `zen upload`, `zen download` and `zen diff` to extend the default exclude folders. Each folder name is separated with ';' or ','
- The default folders that are always excluded are:
- `.unsync`
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index d7980cc24..c4a5a998a 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -289,13 +289,6 @@ namespace {
return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u;
}
- bool IncludePath(std::span<const std::string> IncludeWildcards,
- std::span<const std::string> ExcludeWildcards,
- const std::filesystem::path& Path)
- {
- return zen::IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(Path.generic_string()), /*CaseSensitive*/ true);
- }
-
class FilteredRate
{
public:
@@ -427,253 +420,249 @@ namespace {
NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000));
}
- void UploadFolder(OperationLogOutput& Output,
- TransferThreadWorkers& Workers,
- StorageInstance& Storage,
- const Oid& BuildId,
- const Oid& BuildPartId,
- const std::string_view BuildPartName,
- const std::filesystem::path& Path,
- const std::filesystem::path& TempDir,
- const std::filesystem::path& ManifestPath,
- const uint64_t FindBlockMaxCount,
- const uint8_t BlockReuseMinPercentLimit,
- bool AllowMultiparts,
- const CbObject& MetaData,
- bool CreateBuild,
- bool IgnoreExistingBlocks,
- bool UploadToZenCache,
- const std::vector<std::string>& ExcludeFolders,
- const std::vector<std::string>& ExcludeExtensions)
+ std::vector<std::pair<Oid, std::string>> UploadFolder(OperationLogOutput& Output,
+ TransferThreadWorkers& Workers,
+ StorageInstance& Storage,
+ const Oid& BuildId,
+ const Oid& BuildPartId,
+ const std::string_view BuildPartName,
+ const std::filesystem::path& Path,
+ const std::filesystem::path& TempDir,
+ const std::filesystem::path& ManifestPath,
+ const uint64_t FindBlockMaxCount,
+ const uint8_t BlockReuseMinPercentLimit,
+ bool AllowMultiparts,
+ const CbObject& MetaData,
+ bool CreateBuild,
+ bool IgnoreExistingBlocks,
+ bool UploadToZenCache,
+ const std::vector<std::string>& ExcludeFolders,
+ const std::vector<std::string>& ExcludeExtensions)
{
ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder");
+
+ Stopwatch UploadTimer;
+
+ BuildsOperationUploadFolder UploadOp(Output,
+ Storage,
+ AbortFlag,
+ PauseFlag,
+ Workers.GetIOWorkerPool(),
+ Workers.GetNetworkPool(),
+ BuildId,
+ Path,
+ CreateBuild,
+ std::move(MetaData),
+ BuildsOperationUploadFolder::Options{.IsQuiet = IsQuiet,
+ .IsVerbose = IsVerbose,
+ .DoExtraContentValidation = DoExtraContentVerify,
+ .FindBlockMaxCount = FindBlockMaxCount,
+ .BlockReuseMinPercentLimit = BlockReuseMinPercentLimit,
+ .AllowMultiparts = AllowMultiparts,
+ .IgnoreExistingBlocks = IgnoreExistingBlocks,
+ .TempDir = TempDir,
+ .ExcludeFolders = ExcludeFolders,
+ .ExcludeExtensions = ExcludeExtensions,
+ .ZenExcludeManifestName = ZenExcludeManifestName,
+ .NonCompressableExtensions = DefaultSplitOnlyExtensions,
+ .PopulateCache = UploadToZenCache});
+ std::vector<std::pair<Oid, std::string>> UploadedParts = UploadOp.Execute(BuildPartId, BuildPartName, ManifestPath);
+ if (AbortFlag)
{
- Stopwatch UploadTimer;
+ return {};
+ }
- BuildsOperationUploadFolder UploadOp(
- Output,
- Storage,
- AbortFlag,
- PauseFlag,
- Workers.GetIOWorkerPool(),
- Workers.GetNetworkPool(),
- BuildId,
+ ZEN_CONSOLE_VERBOSE(
+ "Folder scanning stats:"
+ "\n FoundFileCount: {}"
+ "\n FoundFileByteCount: {}"
+ "\n AcceptedFileCount: {}"
+ "\n AcceptedFileByteCount: {}"
+ "\n ElapsedWallTimeUS: {}",
+ UploadOp.m_LocalFolderScanStats.FoundFileCount.load(),
+ NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()),
+ UploadOp.m_LocalFolderScanStats.AcceptedFileCount.load(),
+ NiceBytes(UploadOp.m_LocalFolderScanStats.AcceptedFileByteCount.load()),
+ NiceLatencyNs(UploadOp.m_LocalFolderScanStats.ElapsedWallTimeUS * 1000));
+
+ ZEN_CONSOLE_VERBOSE(
+ "Chunking stats:"
+ "\n FilesProcessed: {}"
+ "\n FilesChunked: {}"
+ "\n BytesHashed: {}"
+ "\n UniqueChunksFound: {}"
+ "\n UniqueSequencesFound: {}"
+ "\n UniqueBytesFound: {}"
+ "\n ElapsedWallTimeUS: {}",
+ UploadOp.m_ChunkingStats.FilesProcessed.load(),
+ UploadOp.m_ChunkingStats.FilesChunked.load(),
+ NiceBytes(UploadOp.m_ChunkingStats.BytesHashed.load()),
+ UploadOp.m_ChunkingStats.UniqueChunksFound.load(),
+ UploadOp.m_ChunkingStats.UniqueSequencesFound.load(),
+ NiceBytes(UploadOp.m_ChunkingStats.UniqueBytesFound.load()),
+ NiceLatencyNs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS * 1000));
+
+ ZEN_CONSOLE_VERBOSE(
+ "Find block stats:"
+ "\n FindBlockTimeMS: {}"
+ "\n PotentialChunkCount: {}"
+ "\n PotentialChunkByteCount: {}"
+ "\n FoundBlockCount: {}"
+ "\n FoundBlockChunkCount: {}"
+ "\n FoundBlockByteCount: {}"
+ "\n AcceptedBlockCount: {}"
+ "\n NewBlocksCount: {}"
+ "\n NewBlocksChunkCount: {}"
+ "\n NewBlocksChunkByteCount: {}",
+ NiceTimeSpanMs(UploadOp.m_FindBlocksStats.FindBlockTimeMS),
+ UploadOp.m_FindBlocksStats.PotentialChunkCount,
+ NiceBytes(UploadOp.m_FindBlocksStats.PotentialChunkByteCount),
+ UploadOp.m_FindBlocksStats.FoundBlockCount,
+ UploadOp.m_FindBlocksStats.FoundBlockChunkCount,
+ NiceBytes(UploadOp.m_FindBlocksStats.FoundBlockByteCount),
+ UploadOp.m_FindBlocksStats.AcceptedBlockCount,
+ UploadOp.m_FindBlocksStats.NewBlocksCount,
+ UploadOp.m_FindBlocksStats.NewBlocksChunkCount,
+ NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount));
+
+ ZEN_CONSOLE_VERBOSE(
+ "Reuse block stats:"
+ "\n AcceptedChunkCount: {}"
+ "\n AcceptedByteCount: {}"
+ "\n AcceptedRawByteCount: {}"
+ "\n RejectedBlockCount: {}"
+ "\n RejectedChunkCount: {}"
+ "\n RejectedByteCount: {}"
+ "\n AcceptedReduntantChunkCount: {}"
+ "\n AcceptedReduntantByteCount: {}",
+ UploadOp.m_ReuseBlocksStats.AcceptedChunkCount,
+ NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedByteCount),
+ NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedRawByteCount),
+ UploadOp.m_ReuseBlocksStats.RejectedBlockCount,
+ UploadOp.m_ReuseBlocksStats.RejectedChunkCount,
+ NiceBytes(UploadOp.m_ReuseBlocksStats.RejectedByteCount),
+ UploadOp.m_ReuseBlocksStats.AcceptedReduntantChunkCount,
+ NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedReduntantByteCount));
+
+ ZEN_CONSOLE_VERBOSE(
+ "Generate blocks stats:"
+ "\n GeneratedBlockByteCount: {}"
+ "\n GeneratedBlockCount: {}"
+ "\n GenerateBlocksElapsedWallTimeUS: {}",
+ NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()),
+ UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(),
+ NiceLatencyNs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000));
+
+ ZEN_CONSOLE_VERBOSE(
+ "Generate blocks stats:"
+ "\n ChunkCount: {}"
+ "\n ChunkByteCount: {}"
+ "\n CompressedChunkCount: {}"
+ "\n CompressChunksElapsedWallTimeUS: {}",
+ UploadOp.m_LooseChunksStats.ChunkCount,
+ NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount),
+ UploadOp.m_LooseChunksStats.CompressedChunkCount.load(),
+ NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()),
+ NiceLatencyNs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000));
+
+ ZEN_CONSOLE_VERBOSE(
+ "Disk stats:"
+ "\n OpenReadCount: {}"
+ "\n OpenWriteCount: {}"
+ "\n ReadCount: {}"
+ "\n ReadByteCount: {}"
+ "\n WriteCount: {} ({} cloned)"
+ "\n WriteByteCount: {} ({} cloned)"
+ "\n CurrentOpenFileCount: {}",
+ UploadOp.m_DiskStats.OpenReadCount.load(),
+ UploadOp.m_DiskStats.OpenWriteCount.load(),
+ UploadOp.m_DiskStats.ReadCount.load(),
+ NiceBytes(UploadOp.m_DiskStats.ReadByteCount.load()),
+ UploadOp.m_DiskStats.WriteCount.load(),
+ UploadOp.m_DiskStats.CloneCount.load(),
+ NiceBytes(UploadOp.m_DiskStats.WriteByteCount.load()),
+ NiceBytes(UploadOp.m_DiskStats.CloneByteCount.load()),
+ UploadOp.m_DiskStats.CurrentOpenFileCount.load());
+
+ ZEN_CONSOLE_VERBOSE(
+ "Upload stats:"
+ "\n BlockCount: {}"
+ "\n BlocksBytes: {}"
+ "\n ChunkCount: {}"
+ "\n ChunksBytes: {}"
+ "\n ReadFromDiskBytes: {}"
+ "\n MultipartAttachmentCount: {}"
+ "\n ElapsedWallTimeUS: {}",
+ UploadOp.m_UploadStats.BlockCount.load(),
+ NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()),
+ UploadOp.m_UploadStats.ChunkCount.load(),
+ NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()),
+ NiceBytes(UploadOp.m_UploadStats.ReadFromDiskBytes.load()),
+ UploadOp.m_UploadStats.MultipartAttachmentCount.load(),
+ NiceLatencyNs(UploadOp.m_UploadStats.ElapsedWallTimeUS * 1000));
+
+ const double DeltaByteCountPercent =
+ UploadOp.m_ChunkingStats.BytesHashed > 0
+ ? (100.0 * (UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes)) /
+ (UploadOp.m_ChunkingStats.BytesHashed)
+ : 0.0;
+
+ const std::string MultipartAttachmentStats =
+ AllowMultiparts ? fmt::format(" ({} as multipart)", UploadOp.m_UploadStats.MultipartAttachmentCount.load()) : "";
+
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE(
+ "Uploaded part {} ('{}') to build {}, {}\n"
+ " Scanned files: {:>8} ({}), {}B/sec, {}\n"
+ " New data: {:>8} ({}) {:.1f}%\n"
+ " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n"
+ " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n"
+ " Uploaded: {:>8} ({}), {}bits/sec, {}\n"
+ " Blocks: {:>8} ({})\n"
+ " Chunks: {:>8} ({}){}",
BuildPartId,
BuildPartName,
- Path,
- ManifestPath,
- CreateBuild,
- std::move(MetaData),
- BuildsOperationUploadFolder::Options{.IsQuiet = IsQuiet,
- .IsVerbose = IsVerbose,
- .DoExtraContentValidation = DoExtraContentVerify,
- .FindBlockMaxCount = FindBlockMaxCount,
- .BlockReuseMinPercentLimit = BlockReuseMinPercentLimit,
- .AllowMultiparts = AllowMultiparts,
- .IgnoreExistingBlocks = IgnoreExistingBlocks,
- .TempDir = TempDir,
- .ExcludeFolders = ExcludeFolders,
- .ExcludeExtensions = ExcludeExtensions,
- .ZenExcludeManifestName = ZenExcludeManifestName,
- .NonCompressableExtensions = DefaultSplitOnlyExtensions,
- .PopulateCache = UploadToZenCache});
- UploadOp.Execute();
- if (AbortFlag)
- {
- return;
- }
+ BuildId,
+ NiceTimeSpanMs(UploadTimer.GetElapsedTimeMs()),
- ZEN_CONSOLE_VERBOSE(
- "Folder scanning stats:"
- "\n FoundFileCount: {}"
- "\n FoundFileByteCount: {}"
- "\n AcceptedFileCount: {}"
- "\n AcceptedFileByteCount: {}"
- "\n ElapsedWallTimeUS: {}",
UploadOp.m_LocalFolderScanStats.FoundFileCount.load(),
NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()),
- UploadOp.m_LocalFolderScanStats.AcceptedFileCount.load(),
- NiceBytes(UploadOp.m_LocalFolderScanStats.AcceptedFileByteCount.load()),
- NiceLatencyNs(UploadOp.m_LocalFolderScanStats.ElapsedWallTimeUS * 1000));
-
- ZEN_CONSOLE_VERBOSE(
- "Chunking stats:"
- "\n FilesProcessed: {}"
- "\n FilesChunked: {}"
- "\n BytesHashed: {}"
- "\n UniqueChunksFound: {}"
- "\n UniqueSequencesFound: {}"
- "\n UniqueBytesFound: {}"
- "\n ElapsedWallTimeUS: {}",
- UploadOp.m_ChunkingStats.FilesProcessed.load(),
- UploadOp.m_ChunkingStats.FilesChunked.load(),
- NiceBytes(UploadOp.m_ChunkingStats.BytesHashed.load()),
- UploadOp.m_ChunkingStats.UniqueChunksFound.load(),
- UploadOp.m_ChunkingStats.UniqueSequencesFound.load(),
- NiceBytes(UploadOp.m_ChunkingStats.UniqueBytesFound.load()),
- NiceLatencyNs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS * 1000));
-
- ZEN_CONSOLE_VERBOSE(
- "Find block stats:"
- "\n FindBlockTimeMS: {}"
- "\n PotentialChunkCount: {}"
- "\n PotentialChunkByteCount: {}"
- "\n FoundBlockCount: {}"
- "\n FoundBlockChunkCount: {}"
- "\n FoundBlockByteCount: {}"
- "\n AcceptedBlockCount: {}"
- "\n NewBlocksCount: {}"
- "\n NewBlocksChunkCount: {}"
- "\n NewBlocksChunkByteCount: {}",
- NiceTimeSpanMs(UploadOp.m_FindBlocksStats.FindBlockTimeMS),
- UploadOp.m_FindBlocksStats.PotentialChunkCount,
- NiceBytes(UploadOp.m_FindBlocksStats.PotentialChunkByteCount),
- UploadOp.m_FindBlocksStats.FoundBlockCount,
- UploadOp.m_FindBlocksStats.FoundBlockChunkCount,
- NiceBytes(UploadOp.m_FindBlocksStats.FoundBlockByteCount),
- UploadOp.m_FindBlocksStats.AcceptedBlockCount,
- UploadOp.m_FindBlocksStats.NewBlocksCount,
- UploadOp.m_FindBlocksStats.NewBlocksChunkCount,
- NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount));
-
- ZEN_CONSOLE_VERBOSE(
- "Reuse block stats:"
- "\n AcceptedChunkCount: {}"
- "\n AcceptedByteCount: {}"
- "\n AcceptedRawByteCount: {}"
- "\n RejectedBlockCount: {}"
- "\n RejectedChunkCount: {}"
- "\n RejectedByteCount: {}"
- "\n AcceptedReduntantChunkCount: {}"
- "\n AcceptedReduntantByteCount: {}",
- UploadOp.m_ReuseBlocksStats.AcceptedChunkCount,
- NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedByteCount),
- NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedRawByteCount),
- UploadOp.m_ReuseBlocksStats.RejectedBlockCount,
- UploadOp.m_ReuseBlocksStats.RejectedChunkCount,
- NiceBytes(UploadOp.m_ReuseBlocksStats.RejectedByteCount),
- UploadOp.m_ReuseBlocksStats.AcceptedReduntantChunkCount,
- NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedReduntantByteCount));
-
- ZEN_CONSOLE_VERBOSE(
- "Generate blocks stats:"
- "\n GeneratedBlockByteCount: {}"
- "\n GeneratedBlockCount: {}"
- "\n GenerateBlocksElapsedWallTimeUS: {}",
- NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()),
+ NiceNum(GetBytesPerSecond(UploadOp.m_ChunkingStats.ElapsedWallTimeUS, UploadOp.m_ChunkingStats.BytesHashed)),
+ NiceTimeSpanMs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS / 1000),
+
+ UploadOp.m_FindBlocksStats.NewBlocksChunkCount + UploadOp.m_LooseChunksStats.CompressedChunkCount,
+ NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes),
+ DeltaByteCountPercent,
+
UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(),
- NiceLatencyNs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000));
-
- ZEN_CONSOLE_VERBOSE(
- "Generate blocks stats:"
- "\n ChunkCount: {}"
- "\n ChunkByteCount: {}"
- "\n CompressedChunkCount: {}"
- "\n CompressChunksElapsedWallTimeUS: {}",
- UploadOp.m_LooseChunksStats.ChunkCount,
- NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount),
+ NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount),
+ NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()),
+ NiceNum(GetBytesPerSecond(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
+ UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount)),
+ NiceTimeSpanMs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000),
+
UploadOp.m_LooseChunksStats.CompressedChunkCount.load(),
+ NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount),
NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()),
- NiceLatencyNs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000));
-
- ZEN_CONSOLE_VERBOSE(
- "Disk stats:"
- "\n OpenReadCount: {}"
- "\n OpenWriteCount: {}"
- "\n ReadCount: {}"
- "\n ReadByteCount: {}"
- "\n WriteCount: {} ({} cloned)"
- "\n WriteByteCount: {} ({} cloned)"
- "\n CurrentOpenFileCount: {}",
- UploadOp.m_DiskStats.OpenReadCount.load(),
- UploadOp.m_DiskStats.OpenWriteCount.load(),
- UploadOp.m_DiskStats.ReadCount.load(),
- NiceBytes(UploadOp.m_DiskStats.ReadByteCount.load()),
- UploadOp.m_DiskStats.WriteCount.load(),
- UploadOp.m_DiskStats.CloneCount.load(),
- NiceBytes(UploadOp.m_DiskStats.WriteByteCount.load()),
- NiceBytes(UploadOp.m_DiskStats.CloneByteCount.load()),
- UploadOp.m_DiskStats.CurrentOpenFileCount.load());
-
- ZEN_CONSOLE_VERBOSE(
- "Upload stats:"
- "\n BlockCount: {}"
- "\n BlocksBytes: {}"
- "\n ChunkCount: {}"
- "\n ChunksBytes: {}"
- "\n ReadFromDiskBytes: {}"
- "\n MultipartAttachmentCount: {}"
- "\n ElapsedWallTimeUS: {}",
+ NiceNum(GetBytesPerSecond(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS,
+ UploadOp.m_LooseChunksStats.ChunkByteCount)),
+ NiceTimeSpanMs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000),
+
+ UploadOp.m_UploadStats.BlockCount.load() + UploadOp.m_UploadStats.ChunkCount.load(),
+ NiceBytes(UploadOp.m_UploadStats.BlocksBytes + UploadOp.m_UploadStats.ChunksBytes),
+ NiceNum(GetBytesPerSecond(UploadOp.m_UploadStats.ElapsedWallTimeUS,
+ (UploadOp.m_UploadStats.ChunksBytes + UploadOp.m_UploadStats.BlocksBytes) * 8)),
+ NiceTimeSpanMs(UploadOp.m_UploadStats.ElapsedWallTimeUS / 1000),
+
UploadOp.m_UploadStats.BlockCount.load(),
NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()),
+
UploadOp.m_UploadStats.ChunkCount.load(),
NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()),
- NiceBytes(UploadOp.m_UploadStats.ReadFromDiskBytes.load()),
- UploadOp.m_UploadStats.MultipartAttachmentCount.load(),
- NiceLatencyNs(UploadOp.m_UploadStats.ElapsedWallTimeUS * 1000));
-
- const double DeltaByteCountPercent =
- UploadOp.m_ChunkingStats.BytesHashed > 0
- ? (100.0 * (UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes)) /
- (UploadOp.m_ChunkingStats.BytesHashed)
- : 0.0;
-
- const std::string MultipartAttachmentStats =
- AllowMultiparts ? fmt::format(" ({} as multipart)", UploadOp.m_UploadStats.MultipartAttachmentCount.load()) : "";
-
- if (!IsQuiet)
- {
- ZEN_CONSOLE(
- "Uploaded part {} ('{}') to build {}, {}\n"
- " Scanned files: {:>8} ({}), {}B/sec, {}\n"
- " New data: {:>8} ({}) {:.1f}%\n"
- " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n"
- " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n"
- " Uploaded: {:>8} ({}), {}bits/sec, {}\n"
- " Blocks: {:>8} ({})\n"
- " Chunks: {:>8} ({}){}",
- BuildPartId,
- BuildPartName,
- BuildId,
- NiceTimeSpanMs(UploadTimer.GetElapsedTimeMs()),
-
- UploadOp.m_LocalFolderScanStats.FoundFileCount.load(),
- NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()),
- NiceNum(GetBytesPerSecond(UploadOp.m_ChunkingStats.ElapsedWallTimeUS, UploadOp.m_ChunkingStats.BytesHashed)),
- NiceTimeSpanMs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS / 1000),
-
- UploadOp.m_FindBlocksStats.NewBlocksChunkCount + UploadOp.m_LooseChunksStats.CompressedChunkCount,
- NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes),
- DeltaByteCountPercent,
-
- UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(),
- NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount),
- NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()),
- NiceNum(GetBytesPerSecond(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
- UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount)),
- NiceTimeSpanMs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000),
-
- UploadOp.m_LooseChunksStats.CompressedChunkCount.load(),
- NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount),
- NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()),
- NiceNum(GetBytesPerSecond(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS,
- UploadOp.m_LooseChunksStats.ChunkByteCount)),
- NiceTimeSpanMs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000),
-
- UploadOp.m_UploadStats.BlockCount.load() + UploadOp.m_UploadStats.ChunkCount.load(),
- NiceBytes(UploadOp.m_UploadStats.BlocksBytes + UploadOp.m_UploadStats.ChunksBytes),
- NiceNum(GetBytesPerSecond(UploadOp.m_UploadStats.ElapsedWallTimeUS,
- (UploadOp.m_UploadStats.ChunksBytes + UploadOp.m_UploadStats.BlocksBytes) * 8)),
- NiceTimeSpanMs(UploadOp.m_UploadStats.ElapsedWallTimeUS / 1000),
-
- UploadOp.m_UploadStats.BlockCount.load(),
- NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()),
-
- UploadOp.m_UploadStats.ChunkCount.load(),
- NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()),
- MultipartAttachmentStats);
- }
+ MultipartAttachmentStats);
}
+ return UploadedParts;
}
struct VerifyFolderStatistics
@@ -878,107 +867,6 @@ namespace {
}
}
- std::string GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix)
- {
- ExtendableStringBuilder<512> SB;
- std::vector<std::pair<std::string, std::string>> NameStringValuePairs;
- for (CbFieldView Field : Object)
- {
- std::string_view Name = Field.GetName();
- switch (CbValue Accessor = Field.GetValue(); Accessor.GetType())
- {
- case CbFieldType::String:
- NameStringValuePairs.push_back({std::string(Name), std::string(Accessor.AsString())});
- break;
- case CbFieldType::IntegerPositive:
- NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerPositive())});
- break;
- case CbFieldType::IntegerNegative:
- NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerNegative())});
- break;
- case CbFieldType::Float32:
- {
- const float Value = Accessor.AsFloat32();
- if (std::isfinite(Value))
- {
- NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.9g}", Value)});
- }
- else
- {
- NameStringValuePairs.push_back({std::string(Name), "null"});
- }
- }
- break;
- case CbFieldType::Float64:
- {
- const double Value = Accessor.AsFloat64();
- if (std::isfinite(Value))
- {
- NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.17g}", Value)});
- }
- else
- {
- NameStringValuePairs.push_back({std::string(Name), "null"});
- }
- }
- break;
- case CbFieldType::BoolFalse:
- NameStringValuePairs.push_back({std::string(Name), "false"});
- break;
- case CbFieldType::BoolTrue:
- NameStringValuePairs.push_back({std::string(Name), "true"});
- break;
- case CbFieldType::Hash:
- {
- NameStringValuePairs.push_back({std::string(Name), Accessor.AsHash().ToHexString()});
- }
- break;
- case CbFieldType::Uuid:
- {
- StringBuilder<Oid::StringLength + 1> Builder;
- Accessor.AsUuid().ToString(Builder);
- NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
- }
- break;
- case CbFieldType::DateTime:
- {
- ExtendableStringBuilder<64> Builder;
- Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601();
- NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
- }
- break;
- case CbFieldType::TimeSpan:
- {
- ExtendableStringBuilder<64> Builder;
- const TimeSpan Span(Accessor.AsTimeSpanTicks());
- if (Span.GetDays() == 0)
- {
- Builder << Span.ToString("%h:%m:%s.%n");
- }
- else
- {
- Builder << Span.ToString("%d.%h:%m:%s.%n");
- }
- NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
- break;
- }
- case CbFieldType::ObjectId:
- NameStringValuePairs.push_back({std::string(Name), Accessor.AsObjectId().ToString()});
- break;
- }
- }
- std::string::size_type LongestKey = 0;
- for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs)
- {
- LongestKey = Max(KeyValue.first.length(), LongestKey);
- }
- for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs)
- {
- SB.Append(fmt::format("{}{:<{}}: {}{}", Prefix, KeyValue.first, LongestKey, KeyValue.second, Suffix));
- }
- return SB.ToString();
- }
-
CbObject GetBuild(BuildStorageBase& Storage, const Oid& BuildId)
{
Stopwatch GetBuildTimer;
@@ -995,280 +883,6 @@ namespace {
return BuildObject;
}
- std::vector<std::pair<Oid, std::string>> ResolveBuildPartNames(CbObjectView BuildObject,
- const Oid& BuildId,
- const std::vector<Oid>& BuildPartIds,
- std::span<const std::string> BuildPartNames,
- std::uint64_t& OutPreferredMultipartChunkSize)
- {
- std::vector<std::pair<Oid, std::string>> Result;
- {
- CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView();
- if (!PartsObject)
- {
- throw std::runtime_error("Build object does not have a 'parts' object");
- }
-
- OutPreferredMultipartChunkSize = BuildObject["chunkSize"sv].AsUInt64(OutPreferredMultipartChunkSize);
-
- std::vector<std::pair<Oid, std::string>> AvailableParts;
-
- for (CbFieldView PartView : PartsObject)
- {
- const std::string BuildPartName = std::string(PartView.GetName());
- const Oid BuildPartId = PartView.AsObjectId();
- if (BuildPartId == Oid::Zero)
- {
- ExtendableStringBuilder<128> SB;
- for (CbFieldView ScanPartView : PartsObject)
- {
- SB.Append(fmt::format("\n {}: {}", ScanPartView.GetName(), ScanPartView.AsObjectId()));
- }
- throw std::runtime_error(
- fmt::format("Build object parts does not have a '{}' object id{}", BuildPartName, SB.ToView()));
- }
- AvailableParts.push_back({BuildPartId, BuildPartName});
- }
-
- if (BuildPartIds.empty() && BuildPartNames.empty())
- {
- Result = AvailableParts;
- }
- else
- {
- for (const std::string& BuildPartName : BuildPartNames)
- {
- if (auto It = std::find_if(AvailableParts.begin(),
- AvailableParts.end(),
- [&BuildPartName](const auto& Part) { return Part.second == BuildPartName; });
- It != AvailableParts.end())
- {
- Result.push_back(*It);
- }
- else
- {
- throw std::runtime_error(fmt::format("Build {} object does not have a part named '{}'", BuildId, BuildPartName));
- }
- }
- for (const Oid& BuildPartId : BuildPartIds)
- {
- if (auto It = std::find_if(AvailableParts.begin(),
- AvailableParts.end(),
- [&BuildPartId](const auto& Part) { return Part.first == BuildPartId; });
- It != AvailableParts.end())
- {
- Result.push_back(*It);
- }
- else
- {
- throw std::runtime_error(fmt::format("Build {} object does not have a part with id '{}'", BuildId, BuildPartId));
- }
- }
- }
-
- if (Result.empty())
- {
- throw std::runtime_error(fmt::format("Build object does not have any parts", BuildId));
- }
- }
- return Result;
- }
-
- ChunkedFolderContent GetRemoteContent(OperationLogOutput& Output,
- StorageInstance& Storage,
- const Oid& BuildId,
- const std::vector<std::pair<Oid, std::string>>& BuildParts,
- std::span<const std::string> IncludeWildcards,
- std::span<const std::string> ExcludeWildcards,
- std::unique_ptr<ChunkingController>& OutChunkController,
- std::vector<ChunkedFolderContent>& OutPartContents,
- std::vector<ChunkBlockDescription>& OutBlockDescriptions,
- std::vector<IoHash>& OutLooseChunkHashes)
- {
- ZEN_TRACE_CPU("GetRemoteContent");
-
- Stopwatch GetBuildPartTimer;
- const Oid BuildPartId = BuildParts[0].first;
- const std::string_view BuildPartName = BuildParts[0].second;
- CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
- BuildPartId,
- BuildPartName,
- NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
- NiceBytes(BuildPartManifest.GetSize()));
- ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv));
- }
-
- {
- CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView();
- std::string_view ChunkerName = Chunker["name"sv].AsString();
- CbObjectView Parameters = Chunker["parameters"sv].AsObjectView();
- OutChunkController = CreateChunkingController(ChunkerName, Parameters);
- }
-
- auto ParseBuildPartManifest = [&Output](StorageInstance& Storage,
- const Oid& BuildId,
- const Oid& BuildPartId,
- CbObject BuildPartManifest,
- std::span<const std::string> IncludeWildcards,
- std::span<const std::string> ExcludeWildcards,
- ChunkedFolderContent& OutRemoteContent,
- std::vector<ChunkBlockDescription>& OutBlockDescriptions,
- std::vector<IoHash>& OutLooseChunkHashes) {
- std::vector<uint32_t> AbsoluteChunkOrders;
- std::vector<uint64_t> LooseChunkRawSizes;
- std::vector<IoHash> BlockRawHashes;
-
- ReadBuildContentFromCompactBinary(BuildPartManifest,
- OutRemoteContent.Platform,
- OutRemoteContent.Paths,
- OutRemoteContent.RawHashes,
- OutRemoteContent.RawSizes,
- OutRemoteContent.Attributes,
- OutRemoteContent.ChunkedContent.SequenceRawHashes,
- OutRemoteContent.ChunkedContent.ChunkCounts,
- AbsoluteChunkOrders,
- OutLooseChunkHashes,
- LooseChunkRawSizes,
- BlockRawHashes);
-
- // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them
-
- {
- bool AttemptFallback = false;
- OutBlockDescriptions = GetBlockDescriptions(Output,
- *Storage.BuildStorage,
- Storage.BuildCacheStorage.get(),
- BuildId,
- BuildPartId,
- BlockRawHashes,
- AttemptFallback,
- IsQuiet,
- IsVerbose);
- }
-
- CalculateLocalChunkOrders(AbsoluteChunkOrders,
- OutLooseChunkHashes,
- LooseChunkRawSizes,
- OutBlockDescriptions,
- OutRemoteContent.ChunkedContent.ChunkHashes,
- OutRemoteContent.ChunkedContent.ChunkRawSizes,
- OutRemoteContent.ChunkedContent.ChunkOrders,
- DoExtraContentVerify);
-
- if (!IncludeWildcards.empty() || !ExcludeWildcards.empty())
- {
- std::vector<std::filesystem::path> DeletedPaths;
- for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths)
- {
- if (!IncludePath(IncludeWildcards, ExcludeWildcards, RemotePath))
- {
- DeletedPaths.push_back(RemotePath);
- }
- }
-
- if (!DeletedPaths.empty())
- {
- OutRemoteContent = DeletePathsFromChunkedContent(OutRemoteContent, DeletedPaths);
- InlineRemoveUnusedHashes(OutLooseChunkHashes, OutRemoteContent.ChunkedContent.ChunkHashes);
- }
- }
-
-#if ZEN_BUILD_DEBUG
- ValidateChunkedFolderContent(OutRemoteContent, OutBlockDescriptions, OutLooseChunkHashes, IncludeWildcards, ExcludeWildcards);
-#endif // ZEN_BUILD_DEBUG
- };
-
- OutPartContents.resize(1);
- ParseBuildPartManifest(Storage,
- BuildId,
- BuildPartId,
- BuildPartManifest,
- IncludeWildcards,
- ExcludeWildcards,
- OutPartContents[0],
- OutBlockDescriptions,
- OutLooseChunkHashes);
- ChunkedFolderContent RemoteContent;
- if (BuildParts.size() > 1)
- {
- std::vector<ChunkBlockDescription> OverlayBlockDescriptions;
- std::vector<IoHash> OverlayLooseChunkHashes;
- for (size_t PartIndex = 1; PartIndex < BuildParts.size(); PartIndex++)
- {
- const Oid& OverlayBuildPartId = BuildParts[PartIndex].first;
- const std::string& OverlayBuildPartName = BuildParts[PartIndex].second;
- Stopwatch GetOverlayBuildPartTimer;
- CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId);
- if (!IsQuiet)
- {
- ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}",
- OverlayBuildPartId,
- OverlayBuildPartName,
- NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()),
- NiceBytes(OverlayBuildPartManifest.GetSize()));
- }
-
- ChunkedFolderContent OverlayPartContent;
- std::vector<ChunkBlockDescription> OverlayPartBlockDescriptions;
- std::vector<IoHash> OverlayPartLooseChunkHashes;
-
- ParseBuildPartManifest(Storage,
- BuildId,
- OverlayBuildPartId,
- OverlayBuildPartManifest,
- IncludeWildcards,
- ExcludeWildcards,
- OverlayPartContent,
- OverlayPartBlockDescriptions,
- OverlayPartLooseChunkHashes);
- OutPartContents.push_back(OverlayPartContent);
- OverlayBlockDescriptions.insert(OverlayBlockDescriptions.end(),
- OverlayPartBlockDescriptions.begin(),
- OverlayPartBlockDescriptions.end());
- OverlayLooseChunkHashes.insert(OverlayLooseChunkHashes.end(),
- OverlayPartLooseChunkHashes.begin(),
- OverlayPartLooseChunkHashes.end());
- }
-
- RemoteContent =
- MergeChunkedFolderContents(OutPartContents[0], std::span<const ChunkedFolderContent>(OutPartContents).subspan(1));
- {
- tsl::robin_set<IoHash> AllBlockHashes;
- for (const ChunkBlockDescription& Description : OutBlockDescriptions)
- {
- AllBlockHashes.insert(Description.BlockHash);
- }
- for (const ChunkBlockDescription& Description : OverlayBlockDescriptions)
- {
- if (!AllBlockHashes.contains(Description.BlockHash))
- {
- AllBlockHashes.insert(Description.BlockHash);
- OutBlockDescriptions.push_back(Description);
- }
- }
- }
- {
- tsl::robin_set<IoHash> AllLooseChunkHashes(OutLooseChunkHashes.begin(), OutLooseChunkHashes.end());
- for (const IoHash& OverlayLooseChunkHash : OverlayLooseChunkHashes)
- {
- if (!AllLooseChunkHashes.contains(OverlayLooseChunkHash))
- {
- AllLooseChunkHashes.insert(OverlayLooseChunkHash);
- OutLooseChunkHashes.push_back(OverlayLooseChunkHash);
- }
- }
- }
- }
- else
- {
- RemoteContent = OutPartContents[0];
- }
- return RemoteContent;
- }
-
std::vector<std::filesystem::path> GetNewPaths(const std::span<const std::filesystem::path> KnownPaths,
const std::span<const std::filesystem::path> Paths)
{
@@ -1661,7 +1275,10 @@ namespace {
ChunkController,
PartContents,
BlockDescriptions,
- LooseChunkHashes);
+ LooseChunkHashes,
+ IsQuiet,
+ IsVerbose,
+ DoExtraContentVerify);
#if ZEN_BUILD_DEBUG
ValidateChunkedFolderContent(RemoteContent, BlockDescriptions, LooseChunkHashes, {}, {});
#endif // ZEN_BUILD_DEBUG
@@ -2102,7 +1719,7 @@ namespace {
for (size_t Index : Order)
{
const std::filesystem::path& Path = Paths[Index];
- if (IncludePath(IncludeWildcards, ExcludeWildcards, Path))
+ if (IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(Path.generic_string()), /*CaseSensitive*/ true))
{
const IoHash& RawHash = RawHashes[Index];
const uint64_t RawSize = RawSizes[Index];
@@ -3719,7 +3336,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
/*RequireBucket*/ true,
/*BoostCacheBackgroundWorkerPool */ false);
- if (m_BuildPartName.empty())
+ if (m_BuildPartName.empty() && m_ManifestPath.empty())
{
m_BuildPartName = m_Path.filename().string();
}
@@ -3729,10 +3346,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
m_BuildId = BuildId.ToString();
}
- const Oid BuildPartId = m_BuildPartId.empty() ? Oid::NewOid() : ParseBuildPartId();
- if (m_BuildPartId.empty())
+
+ Oid BuildPartId;
+ if (!m_BuildPartId.empty())
{
- m_BuildPartId = BuildPartId.ToString();
+ BuildPartId = ParseBuildPartId();
}
CbObject MetaData = ParseBuildMetadata();
@@ -3743,30 +3361,34 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions;
ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions);
- UploadFolder(*Output,
- Workers,
- Storage,
- BuildId,
- BuildPartId,
- m_BuildPartName,
- m_Path,
- TempDir,
- m_ManifestPath,
- m_FindBlockMaxCount,
- m_BlockReuseMinPercentLimit,
- m_AllowMultiparts,
- MetaData,
- m_CreateBuild,
- m_Clean,
- m_UploadToZenCache,
- ExcludeFolders,
- ExcludeExtensions);
+ std::vector<std::pair<Oid, std::string>> UploadedParts = UploadFolder(*Output,
+ Workers,
+ Storage,
+ BuildId,
+ BuildPartId,
+ m_BuildPartName,
+ m_Path,
+ TempDir,
+ m_ManifestPath,
+ m_FindBlockMaxCount,
+ m_BlockReuseMinPercentLimit,
+ m_AllowMultiparts,
+ MetaData,
+ m_CreateBuild,
+ m_Clean,
+ m_UploadToZenCache,
+ ExcludeFolders,
+ ExcludeExtensions);
if (!AbortFlag)
{
if (m_PostUploadVerify)
{
- ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+ // TODO: Validate all parts
+ for (const auto& Part : UploadedParts)
+ {
+ ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, Part.first, Part.second);
+ }
}
}
diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp
index 3ca2f72c1..306e24b9a 100644
--- a/src/zenremotestore/builds/buildstorageoperations.cpp
+++ b/src/zenremotestore/builds/buildstorageoperations.cpp
@@ -16,6 +16,7 @@
#include <zencore/compactbinary.h>
#include <zencore/compactbinaryfile.h>
#include <zencore/compactbinaryutil.h>
+#include <zencore/compactbinaryvalue.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/parallelwork.h>
@@ -23,6 +24,7 @@
#include <zencore/string.h>
#include <zencore/timer.h>
#include <zencore/trace.h>
+#include <zenutil/wildcard.h>
#include <numeric>
@@ -31,6 +33,12 @@ ZEN_THIRD_PARTY_INCLUDES_START
#include <tsl/robin_set.h>
ZEN_THIRD_PARTY_INCLUDES_END
+#if ZEN_WITH_TESTS
+# include <zencore/testing.h>
+# include <zencore/testutils.h>
+# include <zenremotestore/builds/filebuildstorage.h>
+#endif // ZEN_WITH_TESTS
+
namespace zen {
using namespace std::literals;
@@ -4554,10 +4562,7 @@ BuildsOperationUploadFolder::BuildsOperationUploadFolder(OperationLogOutput&
WorkerThreadPool& IOWorkerPool,
WorkerThreadPool& NetworkPool,
const Oid& BuildId,
- const Oid& BuildPartId,
- const std::string_view BuildPartName,
const std::filesystem::path& Path,
- const std::filesystem::path& ManifestPath,
bool CreateBuild,
const CbObject& MetaData,
const Options& Options)
@@ -4568,10 +4573,7 @@ BuildsOperationUploadFolder::BuildsOperationUploadFolder(OperationLogOutput&
, m_IOWorkerPool(IOWorkerPool)
, m_NetworkPool(NetworkPool)
, m_BuildId(BuildId)
-, m_BuildPartId(BuildPartId)
-, m_BuildPartName(BuildPartName)
, m_Path(Path)
-, m_ManifestPath(ManifestPath)
, m_CreateBuild(CreateBuild)
, m_MetaData(MetaData)
, m_Options(Options)
@@ -4583,739 +4585,258 @@ BuildsOperationUploadFolder::BuildsOperationUploadFolder(OperationLogOutput&
}
}
-void
-BuildsOperationUploadFolder::Execute()
+BuildsOperationUploadFolder::PrepareBuildResult
+BuildsOperationUploadFolder::PrepareBuild()
{
- ZEN_TRACE_CPU("BuildsOperationUploadFolder::Execute");
- try
- {
- enum class TaskSteps : uint32_t
- {
- PrepareBuild,
- CalculateDelta,
- GenerateBlocks,
- BuildPartManifest,
- UploadBuildPart,
- UploadAttachments,
- FinalizeBuild,
- PutBuildPartStats,
- Cleanup,
- StepCount
- };
+ ZEN_TRACE_CPU("PrepareBuild");
- auto EndProgress =
- MakeGuard([&]() { m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::StepCount, (uint32_t)TaskSteps::StepCount); });
-
- Stopwatch ProcessTimer;
-
- CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir);
- CreateDirectories(m_Options.TempDir);
- auto _ = MakeGuard([&]() { CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir); });
-
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::PrepareBuild, (uint32_t)TaskSteps::StepCount);
-
- std::uint64_t TotalRawSize = 0;
-
- CbObject ChunkerParameters;
+ PrepareBuildResult Result;
+ Result.PreferredMultipartChunkSize = m_Options.PreferredMultipartChunkSize;
+ Stopwatch Timer;
+ if (m_CreateBuild)
+ {
+ ZEN_TRACE_CPU("CreateBuild");
- struct PrepareBuildResult
+ Stopwatch PutBuildTimer;
+ CbObject PutBuildResult = m_Storage.BuildStorage->PutBuild(m_BuildId, m_MetaData);
+ Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs();
+ Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize);
+ Result.PayloadSize = m_MetaData.GetSize();
+ }
+ else
+ {
+ ZEN_TRACE_CPU("PutBuild");
+ Stopwatch GetBuildTimer;
+ CbObject Build = m_Storage.BuildStorage->GetBuild(m_BuildId);
+ Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs();
+ Result.PayloadSize = Build.GetSize();
+ if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0)
{
- std::vector<ChunkBlockDescription> KnownBlocks;
- uint64_t PreferredMultipartChunkSize = 0;
- uint64_t PayloadSize = 0;
- uint64_t PrepareBuildTimeMs = 0;
- uint64_t FindBlocksTimeMs = 0;
- uint64_t ElapsedTimeMs = 0;
- };
-
- std::future<PrepareBuildResult> PrepBuildResultFuture = m_NetworkPool.EnqueueTask(
- std::packaged_task<PrepareBuildResult()>{[this] {
- ZEN_TRACE_CPU("PrepareBuild");
-
- PrepareBuildResult Result;
- Result.PreferredMultipartChunkSize = m_Options.PreferredMultipartChunkSize;
- Stopwatch Timer;
- if (m_CreateBuild)
- {
- ZEN_TRACE_CPU("CreateBuild");
-
- Stopwatch PutBuildTimer;
- CbObject PutBuildResult = m_Storage.BuildStorage->PutBuild(m_BuildId, m_MetaData);
- Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs();
- Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize);
- Result.PayloadSize = m_MetaData.GetSize();
- }
- else
- {
- ZEN_TRACE_CPU("PutBuild");
- Stopwatch GetBuildTimer;
- CbObject Build = m_Storage.BuildStorage->GetBuild(m_BuildId);
- Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs();
- Result.PayloadSize = Build.GetSize();
- if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0)
- {
- Result.PreferredMultipartChunkSize = ChunkSize;
- }
- else if (m_Options.AllowMultiparts)
- {
- ZEN_OPERATION_LOG_WARN(m_LogOutput,
- "PreferredMultipartChunkSize is unknown. Defaulting to '{}'",
- NiceBytes(Result.PreferredMultipartChunkSize));
- }
- }
-
- if (!m_Options.IgnoreExistingBlocks)
- {
- ZEN_TRACE_CPU("FindBlocks");
- Stopwatch KnownBlocksTimer;
- CbObject BlockDescriptionList = m_Storage.BuildStorage->FindBlocks(m_BuildId, m_Options.FindBlockMaxCount);
- if (BlockDescriptionList)
- {
- Result.KnownBlocks = ParseChunkBlockDescriptionList(BlockDescriptionList);
- }
- m_FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs();
- m_FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size();
- Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs();
- }
- Result.ElapsedTimeMs = Timer.GetElapsedTimeMs();
- return Result;
- }},
- WorkerThreadPool::EMode::EnableBacklog);
-
- ChunkedFolderContent LocalContent;
-
+ Result.PreferredMultipartChunkSize = ChunkSize;
+ }
+ else if (m_Options.AllowMultiparts)
{
- Stopwatch ScanTimer;
- FolderContent Content;
- if (m_ManifestPath.empty())
- {
- std::filesystem::path ExcludeManifestPath = m_Path / m_Options.ZenExcludeManifestName;
- tsl::robin_set<std::string> ExcludeAssetPaths;
- if (IsFile(ExcludeManifestPath))
- {
- std::vector<std::filesystem::path> AssetPaths = ParseManifest(m_Path, ExcludeManifestPath);
- ExcludeAssetPaths.reserve(AssetPaths.size());
- for (const std::filesystem::path& AssetPath : AssetPaths)
- {
- ExcludeAssetPaths.insert(AssetPath.generic_string());
- }
- }
- Content = GetFolderContent(
- m_LocalFolderScanStats,
- m_Path,
- [this](const std::string_view& RelativePath) { return IsAcceptedFolder(RelativePath); },
- [this, &ExcludeAssetPaths](const std::string_view& RelativePath, uint64_t Size, uint32_t Attributes) -> bool {
- ZEN_UNUSED(Size, Attributes);
- if (!IsAcceptedFile(RelativePath))
- {
- return false;
- }
- if (ExcludeAssetPaths.contains(std::filesystem::path(RelativePath).generic_string()))
- {
- return false;
- }
- return true;
- },
- m_IOWorkerPool,
- m_LogOutput.GetProgressUpdateDelayMS(),
- [&](bool, std::ptrdiff_t) {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Found {} files in '{}'...",
- m_LocalFolderScanStats.AcceptedFileCount.load(),
- m_Path);
- },
- m_AbortFlag);
- }
- else
- {
- Stopwatch ManifestParseTimer;
- std::vector<std::filesystem::path> AssetPaths = ParseManifest(m_Path, m_ManifestPath);
- for (const std::filesystem::path& AssetPath : AssetPaths)
- {
- Content.Paths.push_back(AssetPath);
- const std::filesystem::path AssetFilePath = (m_Path / AssetPath).make_preferred();
- Content.RawSizes.push_back(FileSizeFromPath(AssetFilePath));
-#if ZEN_PLATFORM_WINDOWS
- Content.Attributes.push_back(GetFileAttributesFromPath(AssetFilePath));
-#endif // ZEN_PLATFORM_WINDOWS
-#if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX
- Content.Attributes.push_back(GetFileMode(AssetFilePath));
-#endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX
- m_LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back();
- m_LocalFolderScanStats.AcceptedFileCount++;
- }
- if (m_ManifestPath.is_relative())
- {
- Content.Paths.push_back(m_ManifestPath);
- const std::filesystem::path ManifestFilePath = (m_Path / m_ManifestPath).make_preferred();
- Content.RawSizes.push_back(FileSizeFromPath(ManifestFilePath));
-#if ZEN_PLATFORM_WINDOWS
- Content.Attributes.push_back(GetFileAttributesFromPath(ManifestFilePath));
-#endif // ZEN_PLATFORM_WINDOWS
-#if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX
- Content.Attributes.push_back(GetFileMode(ManifestFilePath));
-#endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX
-
- m_LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back();
- m_LocalFolderScanStats.AcceptedFileCount++;
- }
- m_LocalFolderScanStats.FoundFileByteCount.store(m_LocalFolderScanStats.AcceptedFileByteCount);
- m_LocalFolderScanStats.FoundFileCount.store(m_LocalFolderScanStats.AcceptedFileCount);
- m_LocalFolderScanStats.ElapsedWallTimeUS = ManifestParseTimer.GetElapsedTimeUs();
- }
-
- std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
- {
- CbObjectWriter ChunkParametersWriter;
- ChunkParametersWriter.AddString("name"sv, ChunkController->GetName());
- ChunkParametersWriter.AddObject("parameters"sv, ChunkController->GetParameters());
- ChunkerParameters = ChunkParametersWriter.Save();
- }
-
- TotalRawSize = std::accumulate(Content.RawSizes.begin(), Content.RawSizes.end(), std::uint64_t(0));
-
- {
- std::unique_ptr<OperationLogOutput::ProgressBar> ProgressBarPtr(m_LogOutput.CreateProgressBar("Scan Folder"));
- OperationLogOutput::ProgressBar& Progress(*ProgressBarPtr);
-
- FilteredRate FilteredBytesHashed;
- FilteredBytesHashed.Start();
- LocalContent = ChunkFolderContent(
- m_ChunkingStats,
- m_IOWorkerPool,
- m_Path,
- Content,
- *ChunkController,
- m_LogOutput.GetProgressUpdateDelayMS(),
- [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
- FilteredBytesHashed.Update(m_ChunkingStats.BytesHashed.load());
- std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
- m_ChunkingStats.FilesProcessed.load(),
- Content.Paths.size(),
- NiceBytes(m_ChunkingStats.BytesHashed.load()),
- NiceBytes(TotalRawSize),
- NiceNum(FilteredBytesHashed.GetCurrent()),
- m_ChunkingStats.UniqueChunksFound.load(),
- NiceBytes(m_ChunkingStats.UniqueBytesFound.load()));
- Progress.UpdateState({.Task = "Scanning files ",
- .Details = Details,
- .TotalCount = TotalRawSize,
- .RemainingCount = TotalRawSize - m_ChunkingStats.BytesHashed.load(),
- .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- },
- m_AbortFlag,
- m_PauseFlag);
- FilteredBytesHashed.Stop();
- Progress.Finish();
- if (m_AbortFlag)
- {
- return;
- }
- }
-
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
- LocalContent.Paths.size(),
- NiceBytes(TotalRawSize),
- m_ChunkingStats.UniqueChunksFound.load(),
- NiceBytes(m_ChunkingStats.UniqueBytesFound.load()),
- m_Path,
- NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()),
- NiceNum(GetBytesPerSecond(m_ChunkingStats.ElapsedWallTimeUS, m_ChunkingStats.BytesHashed)));
- }
+ ZEN_OPERATION_LOG_WARN(m_LogOutput,
+ "PreferredMultipartChunkSize is unknown. Defaulting to '{}'",
+ NiceBytes(Result.PreferredMultipartChunkSize));
}
+ }
- const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent);
-
- std::vector<size_t> ReuseBlockIndexes;
- std::vector<uint32_t> NewBlockChunkIndexes;
-
- PrepareBuildResult PrepBuildResult = PrepBuildResultFuture.get();
-
- if (!m_Options.IsQuiet)
+ if (!m_Options.IgnoreExistingBlocks)
+ {
+ ZEN_TRACE_CPU("FindBlocks");
+ Stopwatch KnownBlocksTimer;
+ CbObject BlockDescriptionList = m_Storage.BuildStorage->FindBlocks(m_BuildId, m_Options.FindBlockMaxCount);
+ if (BlockDescriptionList)
{
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Build prepare took {}. {} took {}, payload size {}{}",
- NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs),
- m_CreateBuild ? "PutBuild" : "GetBuild",
- NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs),
- NiceBytes(PrepBuildResult.PayloadSize),
- m_Options.IgnoreExistingBlocks ? ""
- : fmt::format(". Found {} blocks in {}",
- PrepBuildResult.KnownBlocks.size(),
- NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs)));
+ Result.KnownBlocks = ParseChunkBlockDescriptionList(BlockDescriptionList);
}
+ Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs();
+ }
+ Result.ElapsedTimeMs = Timer.GetElapsedTimeMs();
+ return Result;
+}
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::CalculateDelta, (uint32_t)TaskSteps::StepCount);
+std::vector<BuildsOperationUploadFolder::UploadPart>
+BuildsOperationUploadFolder::ReadFolder()
+{
+ std::vector<UploadPart> UploadParts;
+ std::filesystem::path ExcludeManifestPath = m_Path / m_Options.ZenExcludeManifestName;
+ tsl::robin_set<std::string> ExcludeAssetPaths;
+ if (IsFile(ExcludeManifestPath))
+ {
+ PartManifest Manifest = ParseManifest(m_Path, ExcludeManifestPath);
+ const std::vector<std::filesystem::path>& AssetPaths = Manifest.Parts.front().Files;
+ ExcludeAssetPaths.reserve(AssetPaths.size());
+ for (const std::filesystem::path& AssetPath : AssetPaths)
+ {
+ ExcludeAssetPaths.insert(AssetPath.generic_string());
+ }
+ }
- const std::uint64_t LargeAttachmentSize =
- m_Options.AllowMultiparts ? PrepBuildResult.PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
+ UploadParts.resize(1);
- Stopwatch BlockArrangeTimer;
+ UploadPart& Part = UploadParts.front();
+ GetFolderContentStatistics& LocalFolderScanStats = Part.LocalFolderScanStats;
- std::vector<std::uint32_t> LooseChunkIndexes;
- {
- bool EnableBlocks = true;
- std::vector<std::uint32_t> BlockChunkIndexes;
- for (uint32_t ChunkIndex = 0; ChunkIndex < LocalContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++)
+ Part.Content = GetFolderContent(
+ Part.LocalFolderScanStats,
+ m_Path,
+ [this](const std::string_view& RelativePath) { return IsAcceptedFolder(RelativePath); },
+ [this, &ExcludeAssetPaths](const std::string_view& RelativePath, uint64_t Size, uint32_t Attributes) -> bool {
+ ZEN_UNUSED(Size, Attributes);
+ if (!IsAcceptedFile(RelativePath))
{
- const uint64_t ChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
- if (!EnableBlocks || ChunkRawSize == 0 || ChunkRawSize > m_Options.BlockParameters.MaxChunkEmbedSize)
- {
- LooseChunkIndexes.push_back(ChunkIndex);
- m_LooseChunksStats.ChunkByteCount += ChunkRawSize;
- }
- else
- {
- BlockChunkIndexes.push_back(ChunkIndex);
- m_FindBlocksStats.PotentialChunkByteCount += ChunkRawSize;
- }
- }
- m_FindBlocksStats.PotentialChunkCount = BlockChunkIndexes.size();
- m_LooseChunksStats.ChunkCount = LooseChunkIndexes.size();
-
- if (m_Options.IgnoreExistingBlocks)
- {
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput, "Ignoring any existing blocks in store");
- }
- NewBlockChunkIndexes = std::move(BlockChunkIndexes);
+ return false;
}
- else
+ if (ExcludeAssetPaths.contains(std::filesystem::path(RelativePath).generic_string()))
{
- ReuseBlockIndexes = FindReuseBlocks(m_LogOutput,
- m_Options.BlockReuseMinPercentLimit,
- m_Options.IsVerbose,
- m_ReuseBlocksStats,
- PrepBuildResult.KnownBlocks,
- LocalContent.ChunkedContent.ChunkHashes,
- BlockChunkIndexes,
- NewBlockChunkIndexes);
- m_FindBlocksStats.AcceptedBlockCount = ReuseBlockIndexes.size();
-
- for (const ChunkBlockDescription& Description : PrepBuildResult.KnownBlocks)
- {
- for (uint32_t ChunkRawLength : Description.ChunkRawLengths)
- {
- m_FindBlocksStats.FoundBlockByteCount += ChunkRawLength;
- }
- m_FindBlocksStats.FoundBlockChunkCount += Description.ChunkRawHashes.size();
- }
+ return false;
}
- }
+ return true;
+ },
+ m_IOWorkerPool,
+ m_LogOutput.GetProgressUpdateDelayMS(),
+ [&](bool, std::ptrdiff_t) {
+ ZEN_OPERATION_LOG_INFO(m_LogOutput, "Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), m_Path);
+ },
+ m_AbortFlag);
+ Part.TotalRawSize = std::accumulate(Part.Content.RawSizes.begin(), Part.Content.RawSizes.end(), std::uint64_t(0));
+
+ return UploadParts;
+}
- std::vector<std::vector<uint32_t>> NewBlockChunks;
- ArrangeChunksIntoBlocks(LocalContent, LocalLookup, NewBlockChunkIndexes, NewBlockChunks);
+std::vector<BuildsOperationUploadFolder::UploadPart>
+BuildsOperationUploadFolder::ReadManifestParts(const std::filesystem::path& ManifestPath)
+{
+ std::vector<UploadPart> UploadParts;
+ Stopwatch ManifestParseTimer;
+ PartManifest Manifest = ParseManifest(m_Path, ManifestPath);
+ if (Manifest.Parts.empty())
+ {
+ throw std::runtime_error(fmt::format("Manifest file at '{}' is invalid", ManifestPath));
+ }
- m_FindBlocksStats.NewBlocksCount = NewBlockChunks.size();
- for (uint32_t ChunkIndex : NewBlockChunkIndexes)
- {
- m_FindBlocksStats.NewBlocksChunkByteCount += LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
- }
- m_FindBlocksStats.NewBlocksChunkCount = NewBlockChunkIndexes.size();
+ UploadParts.resize(Manifest.Parts.size());
+ for (size_t PartIndex = 0; PartIndex < Manifest.Parts.size(); PartIndex++)
+ {
+ const PartManifest::Part& PartManifest = Manifest.Parts[PartIndex];
+ UploadPart& Part = UploadParts[PartIndex];
+ FolderContent& Content = Part.Content;
- const double AcceptedByteCountPercent =
- m_FindBlocksStats.PotentialChunkByteCount > 0
- ? (100.0 * m_ReuseBlocksStats.AcceptedRawByteCount / m_FindBlocksStats.PotentialChunkByteCount)
- : 0.0;
+ GetFolderContentStatistics& LocalFolderScanStats = Part.LocalFolderScanStats;
- const double AcceptedReduntantByteCountPercent =
- m_ReuseBlocksStats.AcceptedByteCount > 0
- ? (100.0 * m_ReuseBlocksStats.AcceptedReduntantByteCount) /
- (m_ReuseBlocksStats.AcceptedByteCount + m_ReuseBlocksStats.AcceptedReduntantByteCount)
- : 0.0;
- if (!m_Options.IsQuiet)
+ const std::vector<std::filesystem::path>& AssetPaths = PartManifest.Files;
+ for (const std::filesystem::path& AssetPath : AssetPaths)
{
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n"
- " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n"
- " Accepting {} ({}) redundant chunks ({:.1f}%)\n"
- " Rejected {} ({}) chunks in {} blocks\n"
- " Arranged {} ({}) chunks in {} new blocks\n"
- " Keeping {} ({}) chunks as loose chunks\n"
- " Discovery completed in {}",
- m_FindBlocksStats.FoundBlockChunkCount,
- m_FindBlocksStats.FoundBlockCount,
- NiceBytes(m_FindBlocksStats.FoundBlockByteCount),
- NiceTimeSpanMs(m_FindBlocksStats.FindBlockTimeMS),
-
- m_ReuseBlocksStats.AcceptedChunkCount,
- NiceBytes(m_ReuseBlocksStats.AcceptedRawByteCount),
- m_FindBlocksStats.AcceptedBlockCount,
- AcceptedByteCountPercent,
-
- m_ReuseBlocksStats.AcceptedReduntantChunkCount,
- NiceBytes(m_ReuseBlocksStats.AcceptedReduntantByteCount),
- AcceptedReduntantByteCountPercent,
-
- m_ReuseBlocksStats.RejectedChunkCount,
- NiceBytes(m_ReuseBlocksStats.RejectedByteCount),
- m_ReuseBlocksStats.RejectedBlockCount,
-
- m_FindBlocksStats.NewBlocksChunkCount,
- NiceBytes(m_FindBlocksStats.NewBlocksChunkByteCount),
- m_FindBlocksStats.NewBlocksCount,
-
- m_LooseChunksStats.ChunkCount,
- NiceBytes(m_LooseChunksStats.ChunkByteCount),
+ Content.Paths.push_back(AssetPath);
+ const std::filesystem::path AssetFilePath = (m_Path / AssetPath).make_preferred();
+ Content.RawSizes.push_back(FileSizeFromPath(AssetFilePath));
+#if ZEN_PLATFORM_WINDOWS
+ Content.Attributes.push_back(GetFileAttributesFromPath(AssetFilePath));
+#endif // ZEN_PLATFORM_WINDOWS
+#if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX
+ Content.Attributes.push_back(GetFileMode(AssetFilePath));
+#endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX
+ LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back();
+ LocalFolderScanStats.AcceptedFileCount++;
+ }
+ if (ManifestPath.is_relative())
+ {
+ Content.Paths.push_back(ManifestPath);
+ const std::filesystem::path ManifestFilePath = (m_Path / ManifestPath).make_preferred();
+ Content.RawSizes.push_back(FileSizeFromPath(ManifestFilePath));
+#if ZEN_PLATFORM_WINDOWS
+ Content.Attributes.push_back(GetFileAttributesFromPath(ManifestFilePath));
+#endif // ZEN_PLATFORM_WINDOWS
+#if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX
+ Content.Attributes.push_back(GetFileMode(ManifestFilePath));
+#endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX
- NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs()));
+ LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back();
+ LocalFolderScanStats.AcceptedFileCount++;
}
+ LocalFolderScanStats.FoundFileByteCount.store(LocalFolderScanStats.AcceptedFileByteCount);
+ LocalFolderScanStats.FoundFileCount.store(LocalFolderScanStats.AcceptedFileCount);
+ LocalFolderScanStats.ElapsedWallTimeUS = ManifestParseTimer.GetElapsedTimeUs();
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::GenerateBlocks, (uint32_t)TaskSteps::StepCount);
- GeneratedBlocks NewBlocks;
+ Part.PartId = PartManifest.PartId;
+ Part.PartName = PartManifest.PartName;
+ Part.TotalRawSize = std::accumulate(Part.Content.RawSizes.begin(), Part.Content.RawSizes.end(), std::uint64_t(0));
+ }
- if (!NewBlockChunks.empty())
- {
- Stopwatch GenerateBuildBlocksTimer;
- auto __ = MakeGuard([&]() {
- uint64_t BlockGenerateTimeUs = GenerateBuildBlocksTimer.GetElapsedTimeUs();
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(
- m_LogOutput,
- "Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.",
- m_GenerateBlocksStats.GeneratedBlockCount.load(),
- NiceBytes(m_GenerateBlocksStats.GeneratedBlockByteCount),
- m_UploadStats.BlockCount.load(),
- NiceBytes(m_UploadStats.BlocksBytes.load()),
- NiceTimeSpanMs(BlockGenerateTimeUs / 1000),
- NiceNum(GetBytesPerSecond(m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
- m_GenerateBlocksStats.GeneratedBlockByteCount)),
- NiceNum(GetBytesPerSecond(m_UploadStats.ElapsedWallTimeUS, m_UploadStats.BlocksBytes * 8)));
- }
- });
- GenerateBuildBlocks(LocalContent, LocalLookup, NewBlockChunks, NewBlocks);
- }
+ return UploadParts;
+}
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::BuildPartManifest, (uint32_t)TaskSteps::StepCount);
+std::vector<std::pair<Oid, std::string>>
+BuildsOperationUploadFolder::Execute(const Oid& BuildPartId,
+ const std::string_view BuildPartName,
+ const std::filesystem::path& ManifestPath)
+{
+ ZEN_TRACE_CPU("BuildsOperationUploadFolder::Execute");
+ try
+ {
+ Stopwatch ReadPartsTimer;
+ std::vector<UploadPart> UploadParts = ManifestPath.empty() ? ReadFolder() : ReadManifestParts(ManifestPath);
- CbObject PartManifest;
+ for (UploadPart& Part : UploadParts)
{
- CbObjectWriter PartManifestWriter;
- Stopwatch ManifestGenerationTimer;
- auto __ = MakeGuard([&]() {
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Generated build part manifest in {} ({})",
- NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()),
- NiceBytes(PartManifestWriter.GetSaveSize()));
- }
- });
- PartManifestWriter.AddObject("chunker"sv, ChunkerParameters);
-
- std::vector<IoHash> AllChunkBlockHashes;
- std::vector<ChunkBlockDescription> AllChunkBlockDescriptions;
- AllChunkBlockHashes.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size());
- AllChunkBlockDescriptions.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size());
- for (size_t ReuseBlockIndex : ReuseBlockIndexes)
- {
- AllChunkBlockDescriptions.push_back(PrepBuildResult.KnownBlocks[ReuseBlockIndex]);
- AllChunkBlockHashes.push_back(PrepBuildResult.KnownBlocks[ReuseBlockIndex].BlockHash);
- }
- AllChunkBlockDescriptions.insert(AllChunkBlockDescriptions.end(),
- NewBlocks.BlockDescriptions.begin(),
- NewBlocks.BlockDescriptions.end());
- for (const ChunkBlockDescription& BlockDescription : NewBlocks.BlockDescriptions)
- {
- AllChunkBlockHashes.push_back(BlockDescription.BlockHash);
- }
- std::vector<IoHash> AbsoluteChunkHashes;
- if (m_Options.DoExtraContentValidation)
+ if (Part.PartId == Oid::Zero)
{
- tsl::robin_map<IoHash, size_t, IoHash::Hasher> ChunkHashToAbsoluteChunkIndex;
- AbsoluteChunkHashes.reserve(LocalContent.ChunkedContent.ChunkHashes.size());
- for (uint32_t ChunkIndex : LooseChunkIndexes)
+ if (UploadParts.size() != 1)
{
- ChunkHashToAbsoluteChunkIndex.insert({LocalContent.ChunkedContent.ChunkHashes[ChunkIndex], AbsoluteChunkHashes.size()});
- AbsoluteChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
+ throw std::runtime_error(fmt::format("Multi part upload manifest '{}' must contains build part id", ManifestPath));
}
- for (const ChunkBlockDescription& Block : AllChunkBlockDescriptions)
- {
- for (const IoHash& ChunkHash : Block.ChunkRawHashes)
- {
- ChunkHashToAbsoluteChunkIndex.insert({ChunkHash, AbsoluteChunkHashes.size()});
- AbsoluteChunkHashes.push_back(ChunkHash);
- }
- }
- for (const IoHash& ChunkHash : LocalContent.ChunkedContent.ChunkHashes)
+
+ if (BuildPartId == Oid::Zero)
{
- ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(ChunkHash)] == ChunkHash);
- ZEN_ASSERT(LocalContent.ChunkedContent.ChunkHashes[LocalLookup.ChunkHashToChunkIndex.at(ChunkHash)] == ChunkHash);
+ Part.PartId = Oid::NewOid();
}
- for (const uint32_t ChunkIndex : LocalContent.ChunkedContent.ChunkOrders)
+ else
{
- ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex])] ==
- LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
- ZEN_ASSERT(LocalLookup.ChunkHashToChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]) == ChunkIndex);
+ Part.PartId = BuildPartId;
}
}
- std::vector<uint32_t> AbsoluteChunkOrders = CalculateAbsoluteChunkOrders(LocalContent.ChunkedContent.ChunkHashes,
- LocalContent.ChunkedContent.ChunkOrders,
- LocalLookup.ChunkHashToChunkIndex,
- LooseChunkIndexes,
- AllChunkBlockDescriptions);
-
- if (m_Options.DoExtraContentValidation)
+ if (Part.PartName.empty())
{
- for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); ChunkOrderIndex++)
+ if (UploadParts.size() != 1)
{
- uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndex];
- uint32_t AbsoluteChunkIndex = AbsoluteChunkOrders[ChunkOrderIndex];
- const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
- const IoHash& AbsoluteChunkHash = AbsoluteChunkHashes[AbsoluteChunkIndex];
- ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash);
+ throw std::runtime_error(fmt::format("Multi part upload manifest '{}' must contains build part name", ManifestPath));
}
- }
-
- WriteBuildContentToCompactBinary(PartManifestWriter,
- LocalContent.Platform,
- LocalContent.Paths,
- LocalContent.RawHashes,
- LocalContent.RawSizes,
- LocalContent.Attributes,
- LocalContent.ChunkedContent.SequenceRawHashes,
- LocalContent.ChunkedContent.ChunkCounts,
- LocalContent.ChunkedContent.ChunkHashes,
- LocalContent.ChunkedContent.ChunkRawSizes,
- AbsoluteChunkOrders,
- LooseChunkIndexes,
- AllChunkBlockHashes);
-
- if (m_Options.DoExtraContentValidation)
- {
- ChunkedFolderContent VerifyFolderContent;
-
- std::vector<uint32_t> OutAbsoluteChunkOrders;
- std::vector<IoHash> OutLooseChunkHashes;
- std::vector<uint64_t> OutLooseChunkRawSizes;
- std::vector<IoHash> OutBlockRawHashes;
- ReadBuildContentFromCompactBinary(PartManifestWriter.Save(),
- VerifyFolderContent.Platform,
- VerifyFolderContent.Paths,
- VerifyFolderContent.RawHashes,
- VerifyFolderContent.RawSizes,
- VerifyFolderContent.Attributes,
- VerifyFolderContent.ChunkedContent.SequenceRawHashes,
- VerifyFolderContent.ChunkedContent.ChunkCounts,
- OutAbsoluteChunkOrders,
- OutLooseChunkHashes,
- OutLooseChunkRawSizes,
- OutBlockRawHashes);
- ZEN_ASSERT(OutBlockRawHashes == AllChunkBlockHashes);
-
- for (uint32_t OrderIndex = 0; OrderIndex < OutAbsoluteChunkOrders.size(); OrderIndex++)
- {
- uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex];
- const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
-
- uint32_t VerifyChunkIndex = OutAbsoluteChunkOrders[OrderIndex];
- const IoHash VerifyChunkHash = AbsoluteChunkHashes[VerifyChunkIndex];
-
- ZEN_ASSERT(LocalChunkHash == VerifyChunkHash);
- }
-
- CalculateLocalChunkOrders(OutAbsoluteChunkOrders,
- OutLooseChunkHashes,
- OutLooseChunkRawSizes,
- AllChunkBlockDescriptions,
- VerifyFolderContent.ChunkedContent.ChunkHashes,
- VerifyFolderContent.ChunkedContent.ChunkRawSizes,
- VerifyFolderContent.ChunkedContent.ChunkOrders,
- m_Options.DoExtraContentValidation);
-
- ZEN_ASSERT(LocalContent.Paths == VerifyFolderContent.Paths);
- ZEN_ASSERT(LocalContent.RawHashes == VerifyFolderContent.RawHashes);
- ZEN_ASSERT(LocalContent.RawSizes == VerifyFolderContent.RawSizes);
- ZEN_ASSERT(LocalContent.Attributes == VerifyFolderContent.Attributes);
- ZEN_ASSERT(LocalContent.ChunkedContent.SequenceRawHashes == VerifyFolderContent.ChunkedContent.SequenceRawHashes);
- ZEN_ASSERT(LocalContent.ChunkedContent.ChunkCounts == VerifyFolderContent.ChunkedContent.ChunkCounts);
-
- for (uint32_t OrderIndex = 0; OrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); OrderIndex++)
+ if (BuildPartName.empty())
{
- uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex];
- const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
- uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex];
-
- uint32_t VerifyChunkIndex = VerifyFolderContent.ChunkedContent.ChunkOrders[OrderIndex];
- const IoHash VerifyChunkHash = VerifyFolderContent.ChunkedContent.ChunkHashes[VerifyChunkIndex];
- uint64_t VerifyChunkRawSize = VerifyFolderContent.ChunkedContent.ChunkRawSizes[VerifyChunkIndex];
-
- ZEN_ASSERT(LocalChunkHash == VerifyChunkHash);
- ZEN_ASSERT(LocalChunkRawSize == VerifyChunkRawSize);
+ throw std::runtime_error("Build part name must be set");
}
+ Part.PartName = std::string(BuildPartName);
}
- PartManifest = PartManifestWriter.Save();
}
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::UploadBuildPart, (uint32_t)TaskSteps::StepCount);
-
- Stopwatch PutBuildPartResultTimer;
- std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult =
- m_Storage.BuildStorage->PutBuildPart(m_BuildId, m_BuildPartId, m_BuildPartName, PartManifest);
if (!m_Options.IsQuiet)
{
ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "PutBuildPart took {}, payload size {}. {} attachments are needed.",
- NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()),
- NiceBytes(PartManifest.GetSize()),
- PutBuildPartResult.second.size());
+ "Reading {} parts took {}",
+ UploadParts.size(),
+ NiceTimeSpanMs(ReadPartsTimer.GetElapsedTimeMs()));
}
- IoHash PartHash = PutBuildPartResult.first;
- auto UploadAttachments = [this, &LocalContent, &LocalLookup, &NewBlockChunks, &NewBlocks, &LooseChunkIndexes, &LargeAttachmentSize](
- std::span<IoHash> RawHashes,
- std::vector<IoHash>& OutUnknownChunks) {
- if (!m_AbortFlag)
- {
- UploadStatistics TempUploadStats;
- LooseChunksStatistics TempLooseChunksStats;
-
- Stopwatch TempUploadTimer;
- auto __ = MakeGuard([&]() {
- if (!m_Options.IsQuiet)
- {
- uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs();
- ZEN_OPERATION_LOG_INFO(
- m_LogOutput,
- "Uploaded {} ({}) blocks. "
- "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. "
- "Transferred {} ({}bits/s) in {}",
- TempUploadStats.BlockCount.load(),
- NiceBytes(TempUploadStats.BlocksBytes),
-
- TempLooseChunksStats.CompressedChunkCount.load(),
- NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()),
- NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS,
- TempLooseChunksStats.ChunkByteCount)),
- TempUploadStats.ChunkCount.load(),
- NiceBytes(TempUploadStats.ChunksBytes),
-
- NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes),
- NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)),
- NiceTimeSpanMs(TempChunkUploadTimeUs / 1000));
- }
- });
- UploadPartBlobs(LocalContent,
- LocalLookup,
- RawHashes,
- NewBlockChunks,
- NewBlocks,
- LooseChunkIndexes,
- LargeAttachmentSize,
- TempUploadStats,
- TempLooseChunksStats,
- OutUnknownChunks);
- m_UploadStats += TempUploadStats;
- m_LooseChunksStats += TempLooseChunksStats;
- }
- };
+ const uint32_t PartsUploadStepCount = gsl::narrow<uint32_t>(uint32_t(PartTaskSteps::StepCount) * UploadParts.size());
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::UploadAttachments, (uint32_t)TaskSteps::StepCount);
+ const uint32_t PrepareBuildStep = 0;
+ const uint32_t UploadPartsStep = 1;
+ const uint32_t FinalizeBuildStep = UploadPartsStep + PartsUploadStepCount;
+ const uint32_t CleanupStep = FinalizeBuildStep + 1;
+ const uint32_t StepCount = CleanupStep + 1;
- std::vector<IoHash> UnknownChunks;
- if (m_Options.IgnoreExistingBlocks)
- {
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "PutBuildPart uploading all attachments, needs are: {}",
- FormatArray<IoHash>(PutBuildPartResult.second, "\n "sv));
- }
+ auto EndProgress = MakeGuard([&]() { m_LogOutput.SetLogOperationProgress(StepCount, StepCount); });
- std::vector<IoHash> ForceUploadChunkHashes;
- ForceUploadChunkHashes.reserve(LooseChunkIndexes.size());
+ Stopwatch ProcessTimer;
- for (uint32_t ChunkIndex : LooseChunkIndexes)
- {
- ForceUploadChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
- }
+ CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir);
+ CreateDirectories(m_Options.TempDir);
+ auto _ = MakeGuard([&]() { CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir); });
- for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockHeaders.size(); BlockIndex++)
- {
- if (NewBlocks.BlockHeaders[BlockIndex])
- {
- // Block was not uploaded during generation
- ForceUploadChunkHashes.push_back(NewBlocks.BlockDescriptions[BlockIndex].BlockHash);
- }
- }
- UploadAttachments(ForceUploadChunkHashes, UnknownChunks);
- }
- else if (!PutBuildPartResult.second.empty())
- {
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "PutBuildPart needs attachments: {}",
- FormatArray<IoHash>(PutBuildPartResult.second, "\n "sv));
- }
- UploadAttachments(PutBuildPartResult.second, UnknownChunks);
- }
+ m_LogOutput.SetLogOperationProgress(PrepareBuildStep, StepCount);
- auto BuildUnkownChunksResponse = [](const std::vector<IoHash>& UnknownChunks, bool WillRetry) {
- return fmt::format(
- "The following build blobs was reported as needed for upload but was reported as existing at the start of the "
- "operation.{}{}",
- WillRetry ? " Treating this as a transient inconsistency issue and will attempt to retry finalization."sv : ""sv,
- FormatArray<IoHash>(UnknownChunks, "\n "sv));
- };
+ m_PrepBuildResultFuture = m_NetworkPool.EnqueueTask(std::packaged_task<PrepareBuildResult()>{[this] { return PrepareBuild(); }},
+ WorkerThreadPool::EMode::EnableBacklog);
- if (!UnknownChunks.empty())
{
- ZEN_OPERATION_LOG_WARN(m_LogOutput, "{}", BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ true));
- }
+ std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
- uint32_t FinalizeBuildPartRetryCount = 5;
- while (!m_AbortFlag && (FinalizeBuildPartRetryCount--) > 0)
- {
- Stopwatch FinalizeBuildPartTimer;
- std::vector<IoHash> Needs = m_Storage.BuildStorage->FinalizeBuildPart(m_BuildId, m_BuildPartId, PartHash);
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "FinalizeBuildPart took {}. {} attachments are missing.",
- NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()),
- Needs.size());
- }
- if (Needs.empty())
- {
- break;
- }
- if (m_Options.IsVerbose)
+ for (uint32_t PartIndex = 0; PartIndex < UploadParts.size(); PartIndex++)
{
- ZEN_OPERATION_LOG_INFO(m_LogOutput, "FinalizeBuildPart needs attachments: {}", FormatArray<IoHash>(Needs, "\n "sv));
- }
+ const uint32_t PartStepOffset = UploadPartsStep + (PartIndex * uint32_t(PartTaskSteps::StepCount));
- std::vector<IoHash> RetryUnknownChunks;
- UploadAttachments(Needs, RetryUnknownChunks);
- if (RetryUnknownChunks == UnknownChunks)
- {
- if (FinalizeBuildPartRetryCount > 0)
+ const UploadPart& Part = UploadParts[PartIndex];
+ UploadBuildPart(*ChunkController, PartIndex, Part, PartStepOffset, StepCount);
+ if (m_AbortFlag)
{
- // Back off a bit
- Sleep(1000);
+ return {};
}
}
- else
- {
- UnknownChunks = RetryUnknownChunks;
- ZEN_OPERATION_LOG_WARN(m_LogOutput,
- "{}",
- BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ FinalizeBuildPartRetryCount != 0));
- }
- }
-
- if (!UnknownChunks.empty())
- {
- throw std::runtime_error(BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ false));
}
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::FinalizeBuild, (uint32_t)TaskSteps::StepCount);
+ m_LogOutput.SetLogOperationProgress(FinalizeBuildStep, StepCount);
if (m_CreateBuild && !m_AbortFlag)
{
@@ -5327,79 +4848,15 @@ BuildsOperationUploadFolder::Execute()
}
}
- if (!NewBlocks.BlockDescriptions.empty() && !m_AbortFlag)
- {
- uint64_t UploadBlockMetadataCount = 0;
- Stopwatch UploadBlockMetadataTimer;
+ m_LogOutput.SetLogOperationProgress(CleanupStep, StepCount);
- uint32_t FailedMetadataUploadCount = 1;
- int32_t MetadataUploadRetryCount = 3;
- while ((MetadataUploadRetryCount-- > 0) && (FailedMetadataUploadCount > 0))
- {
- FailedMetadataUploadCount = 0;
- for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++)
- {
- if (m_AbortFlag)
- {
- break;
- }
- const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash;
- if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex])
- {
- const CbObject BlockMetaData =
- BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]);
- if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
- {
- m_Storage.BuildCacheStorage->PutBlobMetadatas(m_BuildId,
- std::vector<IoHash>({BlockHash}),
- std::vector<CbObject>({BlockMetaData}));
- }
- bool MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData);
- if (MetadataSucceeded)
- {
- m_UploadStats.BlocksBytes += BlockMetaData.GetSize();
- NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
- UploadBlockMetadataCount++;
- }
- else
- {
- FailedMetadataUploadCount++;
- }
- }
- }
- }
- if (UploadBlockMetadataCount > 0)
- {
- uint64_t ElapsedUS = UploadBlockMetadataTimer.GetElapsedTimeUs();
- m_UploadStats.ElapsedWallTimeUS += ElapsedUS;
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Uploaded metadata for {} blocks in {}",
- UploadBlockMetadataCount,
- NiceTimeSpanMs(ElapsedUS / 1000));
- }
- }
+ std::vector<std::pair<Oid, std::string>> Result;
+ Result.reserve(UploadParts.size());
+ for (UploadPart& Part : UploadParts)
+ {
+ Result.push_back(std::make_pair(Part.PartId, Part.PartName));
}
-
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::PutBuildPartStats, (uint32_t)TaskSteps::StepCount);
-
- m_Storage.BuildStorage->PutBuildPartStats(
- m_BuildId,
- m_BuildPartId,
- {{"totalSize", double(m_LocalFolderScanStats.FoundFileByteCount.load())},
- {"reusedRatio", AcceptedByteCountPercent / 100.0},
- {"reusedBlockCount", double(m_FindBlocksStats.AcceptedBlockCount)},
- {"reusedBlockByteCount", double(m_ReuseBlocksStats.AcceptedRawByteCount)},
- {"newBlockCount", double(m_FindBlocksStats.NewBlocksCount)},
- {"newBlockByteCount", double(m_FindBlocksStats.NewBlocksChunkByteCount)},
- {"uploadedCount", double(m_UploadStats.BlockCount.load() + m_UploadStats.ChunkCount.load())},
- {"uploadedByteCount", double(m_UploadStats.BlocksBytes.load() + m_UploadStats.ChunksBytes.load())},
- {"uploadedBytesPerSec",
- double(GetBytesPerSecond(m_UploadStats.ElapsedWallTimeUS, m_UploadStats.ChunksBytes + m_UploadStats.BlocksBytes))},
- {"elapsedTimeSec", double(ProcessTimer.GetElapsedTimeMs() / 1000.0)}});
-
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::Cleanup, (uint32_t)TaskSteps::StepCount);
+ return Result;
}
catch (const std::exception&)
{
@@ -5408,41 +4865,90 @@ BuildsOperationUploadFolder::Execute()
}
}
-std::vector<std::filesystem::path>
+BuildsOperationUploadFolder::PartManifest
BuildsOperationUploadFolder::ParseManifest(const std::filesystem::path& Path, const std::filesystem::path& ManifestPath)
{
- std::vector<std::filesystem::path> AssetPaths;
- std::filesystem::path AbsoluteManifestPath = MakeSafeAbsolutePath(ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath);
- IoBuffer ManifestContent = ReadFile(AbsoluteManifestPath).Flatten();
- std::string_view ManifestString((const char*)ManifestContent.GetView().GetData(), ManifestContent.GetSize());
- std::string_view::size_type Offset = 0;
- while (Offset < ManifestContent.GetSize())
+ PartManifest Result;
{
- size_t PathBreakOffset = ManifestString.find_first_of("\t\r\n", Offset);
- if (PathBreakOffset == std::string_view::npos)
- {
- PathBreakOffset = ManifestContent.GetSize();
- }
- std::string_view AssetPath = ManifestString.substr(Offset, PathBreakOffset - Offset);
- if (!AssetPath.empty())
- {
- AssetPaths.emplace_back(std::filesystem::path(AssetPath));
- }
- Offset = PathBreakOffset;
- size_t EolOffset = ManifestString.find_first_of("\r\n", Offset);
- if (EolOffset == std::string_view::npos)
+ std::filesystem::path AbsoluteManifestPath = MakeSafeAbsolutePath(ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath);
+ IoBuffer ManifestContent = ReadFile(AbsoluteManifestPath).Flatten();
+
+ if (ToLower(ManifestPath.extension().string()) == ".json")
{
- break;
+ IoBuffer MetaDataJson = ReadFile(ManifestPath).Flatten();
+ std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
+ std::string JsonError;
+ CbObject Manifest = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
+ if (!JsonError.empty())
+ {
+ throw std::runtime_error(fmt::format("Invalid manifest file at {}. '{}'", ManifestPath, JsonError));
+ }
+ CbObjectView PartsObject = Manifest["parts"sv].AsObjectView();
+ for (CbFieldView PartsField : PartsObject)
+ {
+ std::string_view PartName = PartsField.GetName();
+ if (PartName.empty())
+ {
+ throw std::runtime_error(fmt::format("Part {} in manifest file at {} does not have a name. '{}'",
+ Result.Parts.size() + 1,
+ ManifestPath,
+ JsonError));
+ }
+ CbObjectView Part = PartsField.AsObjectView();
+ Oid PartId = Part["partId"sv].AsObjectId();
+ if (PartId == Oid::Zero)
+ {
+ PartId = Oid::NewOid();
+ }
+ CbArrayView FilesArray = Part["files"sv].AsArrayView();
+ std::vector<std::filesystem::path> Files;
+ Files.reserve(FilesArray.Num());
+ for (CbFieldView FileField : FilesArray)
+ {
+ std::filesystem::path File(FileField.AsU8String());
+ Files.push_back(File);
+ }
+
+ Result.Parts.push_back(PartManifest::Part{.PartId = PartId, .PartName = std::string(PartName), .Files = std::move(Files)});
+ }
+ return Result;
}
- Offset = EolOffset;
- size_t LineBreakOffset = ManifestString.find_first_not_of("\t\r\n", Offset);
- if (LineBreakOffset == std::string_view::npos)
+ else
{
- break;
+ Result.Parts.resize(1);
+ PartManifest::Part& SinglePart = Result.Parts.front();
+
+ std::string_view ManifestString((const char*)ManifestContent.GetView().GetData(), ManifestContent.GetSize());
+ std::string_view::size_type Offset = 0;
+ while (Offset < ManifestContent.GetSize())
+ {
+ size_t PathBreakOffset = ManifestString.find_first_of("\t\r\n", Offset);
+ if (PathBreakOffset == std::string_view::npos)
+ {
+ PathBreakOffset = ManifestContent.GetSize();
+ }
+ std::string_view AssetPath = ManifestString.substr(Offset, PathBreakOffset - Offset);
+ if (!AssetPath.empty())
+ {
+ SinglePart.Files.emplace_back(std::filesystem::path(AssetPath));
+ }
+ Offset = PathBreakOffset;
+ size_t EolOffset = ManifestString.find_first_of("\r\n", Offset);
+ if (EolOffset == std::string_view::npos)
+ {
+ break;
+ }
+ Offset = EolOffset;
+ size_t LineBreakOffset = ManifestString.find_first_not_of("\t\r\n", Offset);
+ if (LineBreakOffset == std::string_view::npos)
+ {
+ break;
+ }
+ Offset = LineBreakOffset;
+ }
}
- Offset = LineBreakOffset;
}
- return AssetPaths;
+ return Result;
}
bool
@@ -5580,7 +5086,9 @@ void
BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& Content,
const ChunkedContentLookup& Lookup,
const std::vector<std::vector<uint32_t>>& NewBlockChunks,
- GeneratedBlocks& OutBlocks)
+ GeneratedBlocks& OutBlocks,
+ GenerateBlocksStatistics& GenerateBlocksStats,
+ UploadStatistics& UploadStats)
{
ZEN_TRACE_CPU("GenerateBuildBlocks");
const std::size_t NewBlockCount = NewBlockChunks.size();
@@ -5626,6 +5134,8 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent&
ChunksInBlock,
&Lock,
&OutBlocks,
+ &GenerateBlocksStats,
+ &UploadStats,
&FilteredGeneratedBytesPerSecond,
&QueuedPendingBlocksForUpload,
&FilteredUploadedBytesPerSecond,
@@ -5655,8 +5165,8 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent&
Writer.AddString("createdBy", "zen");
OutBlocks.BlockMetaDatas[BlockIndex] = Writer.Save();
}
- m_GenerateBlocksStats.GeneratedBlockByteCount += OutBlocks.BlockSizes[BlockIndex];
- m_GenerateBlocksStats.GeneratedBlockCount++;
+ GenerateBlocksStats.GeneratedBlockByteCount += OutBlocks.BlockSizes[BlockIndex];
+ GenerateBlocksStats.GeneratedBlockCount++;
Lock.WithExclusiveLock([&]() {
OutBlocks.BlockHashToBlockIndex.insert_or_assign(OutBlocks.BlockDescriptions[BlockIndex].BlockHash, BlockIndex);
@@ -5668,7 +5178,7 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent&
OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]);
}
- if (m_GenerateBlocksStats.GeneratedBlockCount == NewBlockCount)
+ if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount)
{
FilteredGeneratedBytesPerSecond.Stop();
}
@@ -5689,6 +5199,8 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent&
UploadBlocksPool,
[this,
NewBlockCount,
+ &GenerateBlocksStats,
+ &UploadStats,
&FilteredUploadedBytesPerSecond,
&QueuedPendingBlocksForUpload,
&OutBlocks,
@@ -5697,7 +5209,7 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent&
auto _ = MakeGuard([&QueuedPendingBlocksForUpload] { QueuedPendingBlocksForUpload--; });
if (!m_AbortFlag)
{
- if (m_GenerateBlocksStats.GeneratedBlockCount == NewBlockCount)
+ if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount)
{
ZEN_TRACE_CPU("GenerateBuildBlocks_Save");
@@ -5731,7 +5243,7 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent&
BlockHash,
ZenContentType::kCompressedBinary,
std::move(Payload).GetCompressed());
- m_UploadStats.BlocksBytes += CompressedBlockSize;
+ UploadStats.BlocksBytes += CompressedBlockSize;
if (m_Options.IsVerbose)
{
@@ -5762,11 +5274,11 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent&
}
OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
- m_UploadStats.BlocksBytes += BlockMetaData.GetSize();
+ UploadStats.BlocksBytes += BlockMetaData.GetSize();
}
- m_UploadStats.BlockCount++;
- if (m_UploadStats.BlockCount == NewBlockCount)
+ UploadStats.BlockCount++;
+ if (UploadStats.BlockCount == NewBlockCount)
{
FilteredUploadedBytesPerSecond.Stop();
}
@@ -5782,23 +5294,23 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent&
Work.Wait(m_LogOutput.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
ZEN_UNUSED(PendingWork);
- FilteredGeneratedBytesPerSecond.Update(m_GenerateBlocksStats.GeneratedBlockByteCount.load());
- FilteredUploadedBytesPerSecond.Update(m_UploadStats.BlocksBytes.load());
+ FilteredGeneratedBytesPerSecond.Update(GenerateBlocksStats.GeneratedBlockByteCount.load());
+ FilteredUploadedBytesPerSecond.Update(UploadStats.BlocksBytes.load());
std::string Details = fmt::format("Generated {}/{} ({}, {}B/s). Uploaded {}/{} ({}, {}bits/s)",
- m_GenerateBlocksStats.GeneratedBlockCount.load(),
+ GenerateBlocksStats.GeneratedBlockCount.load(),
NewBlockCount,
- NiceBytes(m_GenerateBlocksStats.GeneratedBlockByteCount.load()),
+ NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()),
NiceNum(FilteredGeneratedBytesPerSecond.GetCurrent()),
- m_UploadStats.BlockCount.load(),
+ UploadStats.BlockCount.load(),
NewBlockCount,
- NiceBytes(m_UploadStats.BlocksBytes.load()),
+ NiceBytes(UploadStats.BlocksBytes.load()),
NiceNum(FilteredUploadedBytesPerSecond.GetCurrent() * 8));
Progress.UpdateState({.Task = "Generating blocks",
.Details = Details,
.TotalCount = gsl::narrow<uint64_t>(NewBlockCount),
- .RemainingCount = gsl::narrow<uint64_t>(NewBlockCount - m_GenerateBlocksStats.GeneratedBlockCount.load()),
+ .RemainingCount = gsl::narrow<uint64_t>(NewBlockCount - GenerateBlocksStats.GeneratedBlockCount.load()),
.Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
false);
});
@@ -5807,8 +5319,8 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent&
Progress.Finish();
- m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGeneratedBytesPerSecond.GetElapsedTimeUS();
- m_UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTimeUS();
+ GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGeneratedBytesPerSecond.GetElapsedTimeUS();
+ UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTimeUS();
}
}
@@ -5985,6 +5497,663 @@ BuildsOperationUploadFolder::RebuildBlock(const ChunkedFolderContent& Content,
};
void
+BuildsOperationUploadFolder::UploadBuildPart(ChunkingController& ChunkController,
+ uint32_t PartIndex,
+ const UploadPart& Part,
+ uint32_t PartStepOffset,
+ uint32_t StepCount)
+{
+ Stopwatch UploadTimer;
+
+ ChunkingStatistics ChunkingStats;
+ FindBlocksStatistics FindBlocksStats;
+ ReuseBlocksStatistics ReuseBlocksStats;
+ UploadStatistics UploadStats;
+ GenerateBlocksStatistics GenerateBlocksStats;
+
+ LooseChunksStatistics LooseChunksStats;
+ ChunkedFolderContent LocalContent;
+
+ m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::ChunkPartContent, StepCount);
+
+ Stopwatch ScanTimer;
+ {
+ std::unique_ptr<OperationLogOutput::ProgressBar> ProgressBarPtr(m_LogOutput.CreateProgressBar("Scan Folder"));
+ OperationLogOutput::ProgressBar& Progress(*ProgressBarPtr);
+
+ FilteredRate FilteredBytesHashed;
+ FilteredBytesHashed.Start();
+ LocalContent = ChunkFolderContent(
+ ChunkingStats,
+ m_IOWorkerPool,
+ m_Path,
+ Part.Content,
+ ChunkController,
+ m_LogOutput.GetProgressUpdateDelayMS(),
+ [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
+ FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load());
+ std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
+ ChunkingStats.FilesProcessed.load(),
+ Part.Content.Paths.size(),
+ NiceBytes(ChunkingStats.BytesHashed.load()),
+ NiceBytes(Part.TotalRawSize),
+ NiceNum(FilteredBytesHashed.GetCurrent()),
+ ChunkingStats.UniqueChunksFound.load(),
+ NiceBytes(ChunkingStats.UniqueBytesFound.load()));
+ Progress.UpdateState({.Task = "Scanning files ",
+ .Details = Details,
+ .TotalCount = Part.TotalRawSize,
+ .RemainingCount = Part.TotalRawSize - ChunkingStats.BytesHashed.load(),
+ .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
+ },
+ m_AbortFlag,
+ m_PauseFlag);
+ FilteredBytesHashed.Stop();
+ Progress.Finish();
+ if (m_AbortFlag)
+ {
+ return;
+ }
+ }
+
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_OPERATION_LOG_INFO(m_LogOutput,
+ "Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
+ Part.Content.Paths.size(),
+ NiceBytes(Part.TotalRawSize),
+ ChunkingStats.UniqueChunksFound.load(),
+ NiceBytes(ChunkingStats.UniqueBytesFound.load()),
+ m_Path,
+ NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()),
+ NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ }
+
+ const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent);
+
+ std::vector<size_t> ReuseBlockIndexes;
+ std::vector<uint32_t> NewBlockChunkIndexes;
+
+ if (PartIndex == 0)
+ {
+ m_PrepBuildResult = m_PrepBuildResultFuture.get();
+
+ m_FindBlocksStats.FindBlockTimeMS = m_PrepBuildResult.ElapsedTimeMs;
+ m_FindBlocksStats.FoundBlockCount = m_PrepBuildResult.KnownBlocks.size();
+ }
+
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_OPERATION_LOG_INFO(m_LogOutput,
+ "Build prepare took {}. {} took {}, payload size {}{}",
+ NiceTimeSpanMs(m_PrepBuildResult.ElapsedTimeMs),
+ m_CreateBuild ? "PutBuild" : "GetBuild",
+ NiceTimeSpanMs(m_PrepBuildResult.PrepareBuildTimeMs),
+ NiceBytes(m_PrepBuildResult.PayloadSize),
+ m_Options.IgnoreExistingBlocks ? ""
+ : fmt::format(". Found {} blocks in {}",
+ m_PrepBuildResult.KnownBlocks.size(),
+ NiceTimeSpanMs(m_PrepBuildResult.FindBlocksTimeMs)));
+ }
+
+ m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::CalculateDelta, StepCount);
+
+ const std::uint64_t LargeAttachmentSize =
+ m_Options.AllowMultiparts ? m_PrepBuildResult.PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
+
+ Stopwatch BlockArrangeTimer;
+
+ std::vector<std::uint32_t> LooseChunkIndexes;
+ {
+ bool EnableBlocks = true;
+ std::vector<std::uint32_t> BlockChunkIndexes;
+ for (uint32_t ChunkIndex = 0; ChunkIndex < LocalContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++)
+ {
+ const uint64_t ChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ if (!EnableBlocks || ChunkRawSize == 0 || ChunkRawSize > m_Options.BlockParameters.MaxChunkEmbedSize)
+ {
+ LooseChunkIndexes.push_back(ChunkIndex);
+ LooseChunksStats.ChunkByteCount += ChunkRawSize;
+ }
+ else
+ {
+ BlockChunkIndexes.push_back(ChunkIndex);
+ FindBlocksStats.PotentialChunkByteCount += ChunkRawSize;
+ }
+ }
+ FindBlocksStats.PotentialChunkCount += BlockChunkIndexes.size();
+ LooseChunksStats.ChunkCount = LooseChunkIndexes.size();
+
+ if (m_Options.IgnoreExistingBlocks)
+ {
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_OPERATION_LOG_INFO(m_LogOutput, "Ignoring any existing blocks in store");
+ }
+ NewBlockChunkIndexes = std::move(BlockChunkIndexes);
+ }
+ else
+ {
+ ReuseBlockIndexes = FindReuseBlocks(m_LogOutput,
+ m_Options.BlockReuseMinPercentLimit,
+ m_Options.IsVerbose,
+ ReuseBlocksStats,
+ m_PrepBuildResult.KnownBlocks,
+ LocalContent.ChunkedContent.ChunkHashes,
+ BlockChunkIndexes,
+ NewBlockChunkIndexes);
+ FindBlocksStats.AcceptedBlockCount += ReuseBlockIndexes.size();
+
+ for (const ChunkBlockDescription& Description : m_PrepBuildResult.KnownBlocks)
+ {
+ for (uint32_t ChunkRawLength : Description.ChunkRawLengths)
+ {
+ FindBlocksStats.FoundBlockByteCount += ChunkRawLength;
+ }
+ FindBlocksStats.FoundBlockChunkCount += Description.ChunkRawHashes.size();
+ }
+ }
+ }
+
+ std::vector<std::vector<uint32_t>> NewBlockChunks;
+ ArrangeChunksIntoBlocks(LocalContent, LocalLookup, NewBlockChunkIndexes, NewBlockChunks);
+
+ FindBlocksStats.NewBlocksCount += NewBlockChunks.size();
+ for (uint32_t ChunkIndex : NewBlockChunkIndexes)
+ {
+ FindBlocksStats.NewBlocksChunkByteCount += LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ }
+ FindBlocksStats.NewBlocksChunkCount += NewBlockChunkIndexes.size();
+
+ const double AcceptedByteCountPercent = FindBlocksStats.PotentialChunkByteCount > 0
+ ? (100.0 * ReuseBlocksStats.AcceptedRawByteCount / FindBlocksStats.PotentialChunkByteCount)
+ : 0.0;
+
+ const double AcceptedReduntantByteCountPercent =
+ ReuseBlocksStats.AcceptedByteCount > 0 ? (100.0 * ReuseBlocksStats.AcceptedReduntantByteCount) /
+ (ReuseBlocksStats.AcceptedByteCount + ReuseBlocksStats.AcceptedReduntantByteCount)
+ : 0.0;
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_OPERATION_LOG_INFO(m_LogOutput,
+ "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n"
+ " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n"
+ " Accepting {} ({}) redundant chunks ({:.1f}%)\n"
+ " Rejected {} ({}) chunks in {} blocks\n"
+ " Arranged {} ({}) chunks in {} new blocks\n"
+ " Keeping {} ({}) chunks as loose chunks\n"
+ " Discovery completed in {}",
+ FindBlocksStats.FoundBlockChunkCount,
+ FindBlocksStats.FoundBlockCount,
+ NiceBytes(FindBlocksStats.FoundBlockByteCount),
+ NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS),
+
+ ReuseBlocksStats.AcceptedChunkCount,
+ NiceBytes(ReuseBlocksStats.AcceptedRawByteCount),
+ FindBlocksStats.AcceptedBlockCount,
+ AcceptedByteCountPercent,
+
+ ReuseBlocksStats.AcceptedReduntantChunkCount,
+ NiceBytes(ReuseBlocksStats.AcceptedReduntantByteCount),
+ AcceptedReduntantByteCountPercent,
+
+ ReuseBlocksStats.RejectedChunkCount,
+ NiceBytes(ReuseBlocksStats.RejectedByteCount),
+ ReuseBlocksStats.RejectedBlockCount,
+
+ FindBlocksStats.NewBlocksChunkCount,
+ NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
+ FindBlocksStats.NewBlocksCount,
+
+ LooseChunksStats.ChunkCount,
+ NiceBytes(LooseChunksStats.ChunkByteCount),
+
+ NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs()));
+ }
+
+ m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::GenerateBlocks, StepCount);
+ GeneratedBlocks NewBlocks;
+
+ if (!NewBlockChunks.empty())
+ {
+ Stopwatch GenerateBuildBlocksTimer;
+ auto __ = MakeGuard([&]() {
+ uint64_t BlockGenerateTimeUs = GenerateBuildBlocksTimer.GetElapsedTimeUs();
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_OPERATION_LOG_INFO(
+ m_LogOutput,
+ "Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.",
+ GenerateBlocksStats.GeneratedBlockCount.load(),
+ NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount),
+ UploadStats.BlockCount.load(),
+ NiceBytes(UploadStats.BlocksBytes.load()),
+ NiceTimeSpanMs(BlockGenerateTimeUs / 1000),
+ NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
+ GenerateBlocksStats.GeneratedBlockByteCount)),
+ NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8)));
+ }
+ });
+ GenerateBuildBlocks(LocalContent, LocalLookup, NewBlockChunks, NewBlocks, GenerateBlocksStats, UploadStats);
+ }
+
+ m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::BuildPartManifest, StepCount);
+
+ CbObject PartManifest;
+ {
+ CbObjectWriter PartManifestWriter;
+ Stopwatch ManifestGenerationTimer;
+ auto __ = MakeGuard([&]() {
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_OPERATION_LOG_INFO(m_LogOutput,
+ "Generated build part manifest in {} ({})",
+ NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()),
+ NiceBytes(PartManifestWriter.GetSaveSize()));
+ }
+ });
+
+ PartManifestWriter.BeginObject("chunker"sv);
+ {
+ PartManifestWriter.AddString("name"sv, ChunkController.GetName());
+ PartManifestWriter.AddObject("parameters"sv, ChunkController.GetParameters());
+ }
+ PartManifestWriter.EndObject(); // chunker
+
+ std::vector<IoHash> AllChunkBlockHashes;
+ std::vector<ChunkBlockDescription> AllChunkBlockDescriptions;
+ AllChunkBlockHashes.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size());
+ AllChunkBlockDescriptions.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size());
+ for (size_t ReuseBlockIndex : ReuseBlockIndexes)
+ {
+ AllChunkBlockDescriptions.push_back(m_PrepBuildResult.KnownBlocks[ReuseBlockIndex]);
+ AllChunkBlockHashes.push_back(m_PrepBuildResult.KnownBlocks[ReuseBlockIndex].BlockHash);
+ }
+ AllChunkBlockDescriptions.insert(AllChunkBlockDescriptions.end(),
+ NewBlocks.BlockDescriptions.begin(),
+ NewBlocks.BlockDescriptions.end());
+ for (const ChunkBlockDescription& BlockDescription : NewBlocks.BlockDescriptions)
+ {
+ AllChunkBlockHashes.push_back(BlockDescription.BlockHash);
+ }
+ std::vector<IoHash> AbsoluteChunkHashes;
+ if (m_Options.DoExtraContentValidation)
+ {
+ tsl::robin_map<IoHash, size_t, IoHash::Hasher> ChunkHashToAbsoluteChunkIndex;
+ AbsoluteChunkHashes.reserve(LocalContent.ChunkedContent.ChunkHashes.size());
+ for (uint32_t ChunkIndex : LooseChunkIndexes)
+ {
+ ChunkHashToAbsoluteChunkIndex.insert({LocalContent.ChunkedContent.ChunkHashes[ChunkIndex], AbsoluteChunkHashes.size()});
+ AbsoluteChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
+ }
+ for (const ChunkBlockDescription& Block : AllChunkBlockDescriptions)
+ {
+ for (const IoHash& ChunkHash : Block.ChunkRawHashes)
+ {
+ ChunkHashToAbsoluteChunkIndex.insert({ChunkHash, AbsoluteChunkHashes.size()});
+ AbsoluteChunkHashes.push_back(ChunkHash);
+ }
+ }
+ for (const IoHash& ChunkHash : LocalContent.ChunkedContent.ChunkHashes)
+ {
+ ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(ChunkHash)] == ChunkHash);
+ ZEN_ASSERT(LocalContent.ChunkedContent.ChunkHashes[LocalLookup.ChunkHashToChunkIndex.at(ChunkHash)] == ChunkHash);
+ }
+ for (const uint32_t ChunkIndex : LocalContent.ChunkedContent.ChunkOrders)
+ {
+ ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex])] ==
+ LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
+ ZEN_ASSERT(LocalLookup.ChunkHashToChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]) == ChunkIndex);
+ }
+ }
+ std::vector<uint32_t> AbsoluteChunkOrders = CalculateAbsoluteChunkOrders(LocalContent.ChunkedContent.ChunkHashes,
+ LocalContent.ChunkedContent.ChunkOrders,
+ LocalLookup.ChunkHashToChunkIndex,
+ LooseChunkIndexes,
+ AllChunkBlockDescriptions);
+
+ if (m_Options.DoExtraContentValidation)
+ {
+ for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); ChunkOrderIndex++)
+ {
+ uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndex];
+ uint32_t AbsoluteChunkIndex = AbsoluteChunkOrders[ChunkOrderIndex];
+ const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
+ const IoHash& AbsoluteChunkHash = AbsoluteChunkHashes[AbsoluteChunkIndex];
+ ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash);
+ }
+ }
+
+ WriteBuildContentToCompactBinary(PartManifestWriter,
+ LocalContent.Platform,
+ LocalContent.Paths,
+ LocalContent.RawHashes,
+ LocalContent.RawSizes,
+ LocalContent.Attributes,
+ LocalContent.ChunkedContent.SequenceRawHashes,
+ LocalContent.ChunkedContent.ChunkCounts,
+ LocalContent.ChunkedContent.ChunkHashes,
+ LocalContent.ChunkedContent.ChunkRawSizes,
+ AbsoluteChunkOrders,
+ LooseChunkIndexes,
+ AllChunkBlockHashes);
+
+ if (m_Options.DoExtraContentValidation)
+ {
+ ChunkedFolderContent VerifyFolderContent;
+
+ std::vector<uint32_t> OutAbsoluteChunkOrders;
+ std::vector<IoHash> OutLooseChunkHashes;
+ std::vector<uint64_t> OutLooseChunkRawSizes;
+ std::vector<IoHash> OutBlockRawHashes;
+ ReadBuildContentFromCompactBinary(PartManifestWriter.Save(),
+ VerifyFolderContent.Platform,
+ VerifyFolderContent.Paths,
+ VerifyFolderContent.RawHashes,
+ VerifyFolderContent.RawSizes,
+ VerifyFolderContent.Attributes,
+ VerifyFolderContent.ChunkedContent.SequenceRawHashes,
+ VerifyFolderContent.ChunkedContent.ChunkCounts,
+ OutAbsoluteChunkOrders,
+ OutLooseChunkHashes,
+ OutLooseChunkRawSizes,
+ OutBlockRawHashes);
+ ZEN_ASSERT(OutBlockRawHashes == AllChunkBlockHashes);
+
+ for (uint32_t OrderIndex = 0; OrderIndex < OutAbsoluteChunkOrders.size(); OrderIndex++)
+ {
+ uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex];
+ const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
+
+ uint32_t VerifyChunkIndex = OutAbsoluteChunkOrders[OrderIndex];
+ const IoHash VerifyChunkHash = AbsoluteChunkHashes[VerifyChunkIndex];
+
+ ZEN_ASSERT(LocalChunkHash == VerifyChunkHash);
+ }
+
+ CalculateLocalChunkOrders(OutAbsoluteChunkOrders,
+ OutLooseChunkHashes,
+ OutLooseChunkRawSizes,
+ AllChunkBlockDescriptions,
+ VerifyFolderContent.ChunkedContent.ChunkHashes,
+ VerifyFolderContent.ChunkedContent.ChunkRawSizes,
+ VerifyFolderContent.ChunkedContent.ChunkOrders,
+ m_Options.DoExtraContentValidation);
+
+ ZEN_ASSERT(LocalContent.Paths == VerifyFolderContent.Paths);
+ ZEN_ASSERT(LocalContent.RawHashes == VerifyFolderContent.RawHashes);
+ ZEN_ASSERT(LocalContent.RawSizes == VerifyFolderContent.RawSizes);
+ ZEN_ASSERT(LocalContent.Attributes == VerifyFolderContent.Attributes);
+ ZEN_ASSERT(LocalContent.ChunkedContent.SequenceRawHashes == VerifyFolderContent.ChunkedContent.SequenceRawHashes);
+ ZEN_ASSERT(LocalContent.ChunkedContent.ChunkCounts == VerifyFolderContent.ChunkedContent.ChunkCounts);
+
+ for (uint32_t OrderIndex = 0; OrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); OrderIndex++)
+ {
+ uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex];
+ const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
+ uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex];
+
+ uint32_t VerifyChunkIndex = VerifyFolderContent.ChunkedContent.ChunkOrders[OrderIndex];
+ const IoHash VerifyChunkHash = VerifyFolderContent.ChunkedContent.ChunkHashes[VerifyChunkIndex];
+ uint64_t VerifyChunkRawSize = VerifyFolderContent.ChunkedContent.ChunkRawSizes[VerifyChunkIndex];
+
+ ZEN_ASSERT(LocalChunkHash == VerifyChunkHash);
+ ZEN_ASSERT(LocalChunkRawSize == VerifyChunkRawSize);
+ }
+ }
+ PartManifest = PartManifestWriter.Save();
+ }
+
+ m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::UploadBuildPart, StepCount);
+
+ Stopwatch PutBuildPartResultTimer;
+ std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult =
+ m_Storage.BuildStorage->PutBuildPart(m_BuildId, Part.PartId, Part.PartName, PartManifest);
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_OPERATION_LOG_INFO(m_LogOutput,
+ "PutBuildPart took {}, payload size {}. {} attachments are needed.",
+ NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()),
+ NiceBytes(PartManifest.GetSize()),
+ PutBuildPartResult.second.size());
+ }
+ IoHash PartHash = PutBuildPartResult.first;
+
+ auto UploadAttachments = [this,
+ &LooseChunksStats,
+ &UploadStats,
+ &LocalContent,
+ &LocalLookup,
+ &NewBlockChunks,
+ &NewBlocks,
+ &LooseChunkIndexes,
+ &LargeAttachmentSize](std::span<IoHash> RawHashes, std::vector<IoHash>& OutUnknownChunks) {
+ if (!m_AbortFlag)
+ {
+ UploadStatistics TempUploadStats;
+ LooseChunksStatistics TempLooseChunksStats;
+
+ Stopwatch TempUploadTimer;
+ auto __ = MakeGuard([&]() {
+ if (!m_Options.IsQuiet)
+ {
+ uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs();
+ ZEN_OPERATION_LOG_INFO(m_LogOutput,
+ "Uploaded {} ({}) blocks. "
+ "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. "
+ "Transferred {} ({}bits/s) in {}",
+ TempUploadStats.BlockCount.load(),
+ NiceBytes(TempUploadStats.BlocksBytes),
+
+ TempLooseChunksStats.CompressedChunkCount.load(),
+ NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()),
+ NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS,
+ TempLooseChunksStats.ChunkByteCount)),
+ TempUploadStats.ChunkCount.load(),
+ NiceBytes(TempUploadStats.ChunksBytes),
+
+ NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes),
+ NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)),
+ NiceTimeSpanMs(TempChunkUploadTimeUs / 1000));
+ }
+ });
+ UploadPartBlobs(LocalContent,
+ LocalLookup,
+ RawHashes,
+ NewBlockChunks,
+ NewBlocks,
+ LooseChunkIndexes,
+ LargeAttachmentSize,
+ TempUploadStats,
+ TempLooseChunksStats,
+ OutUnknownChunks);
+ UploadStats += TempUploadStats;
+ LooseChunksStats += TempLooseChunksStats;
+ }
+ };
+
+ m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::UploadAttachments, StepCount);
+
+ std::vector<IoHash> UnknownChunks;
+ if (m_Options.IgnoreExistingBlocks)
+ {
+ if (m_Options.IsVerbose)
+ {
+ ZEN_OPERATION_LOG_INFO(m_LogOutput,
+ "PutBuildPart uploading all attachments, needs are: {}",
+ FormatArray<IoHash>(PutBuildPartResult.second, "\n "sv));
+ }
+
+ std::vector<IoHash> ForceUploadChunkHashes;
+ ForceUploadChunkHashes.reserve(LooseChunkIndexes.size());
+
+ for (uint32_t ChunkIndex : LooseChunkIndexes)
+ {
+ ForceUploadChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
+ }
+
+ for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockHeaders.size(); BlockIndex++)
+ {
+ if (NewBlocks.BlockHeaders[BlockIndex])
+ {
+ // Block was not uploaded during generation
+ ForceUploadChunkHashes.push_back(NewBlocks.BlockDescriptions[BlockIndex].BlockHash);
+ }
+ }
+ UploadAttachments(ForceUploadChunkHashes, UnknownChunks);
+ }
+ else if (!PutBuildPartResult.second.empty())
+ {
+ if (m_Options.IsVerbose)
+ {
+ ZEN_OPERATION_LOG_INFO(m_LogOutput,
+ "PutBuildPart needs attachments: {}",
+ FormatArray<IoHash>(PutBuildPartResult.second, "\n "sv));
+ }
+ UploadAttachments(PutBuildPartResult.second, UnknownChunks);
+ }
+
+ auto BuildUnkownChunksResponse = [](const std::vector<IoHash>& UnknownChunks, bool WillRetry) {
+ return fmt::format(
+ "The following build blobs was reported as needed for upload but was reported as existing at the start of the "
+ "operation.{}{}",
+ WillRetry ? " Treating this as a transient inconsistency issue and will attempt to retry finalization."sv : ""sv,
+ FormatArray<IoHash>(UnknownChunks, "\n "sv));
+ };
+
+ if (!UnknownChunks.empty())
+ {
+ ZEN_OPERATION_LOG_WARN(m_LogOutput, "{}", BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ true));
+ }
+
+ uint32_t FinalizeBuildPartRetryCount = 5;
+ while (!m_AbortFlag && (FinalizeBuildPartRetryCount--) > 0)
+ {
+ Stopwatch FinalizeBuildPartTimer;
+ std::vector<IoHash> Needs = m_Storage.BuildStorage->FinalizeBuildPart(m_BuildId, Part.PartId, PartHash);
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_OPERATION_LOG_INFO(m_LogOutput,
+ "FinalizeBuildPart took {}. {} attachments are missing.",
+ NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()),
+ Needs.size());
+ }
+ if (Needs.empty())
+ {
+ break;
+ }
+ if (m_Options.IsVerbose)
+ {
+ ZEN_OPERATION_LOG_INFO(m_LogOutput, "FinalizeBuildPart needs attachments: {}", FormatArray<IoHash>(Needs, "\n "sv));
+ }
+
+ std::vector<IoHash> RetryUnknownChunks;
+ UploadAttachments(Needs, RetryUnknownChunks);
+ if (RetryUnknownChunks == UnknownChunks)
+ {
+ if (FinalizeBuildPartRetryCount > 0)
+ {
+ // Back off a bit
+ Sleep(1000);
+ }
+ }
+ else
+ {
+ UnknownChunks = RetryUnknownChunks;
+ ZEN_OPERATION_LOG_WARN(m_LogOutput,
+ "{}",
+ BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ FinalizeBuildPartRetryCount != 0));
+ }
+ }
+
+ if (!UnknownChunks.empty())
+ {
+ throw std::runtime_error(BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ false));
+ }
+
+ if (!NewBlocks.BlockDescriptions.empty() && !m_AbortFlag)
+ {
+ uint64_t UploadBlockMetadataCount = 0;
+ Stopwatch UploadBlockMetadataTimer;
+
+ uint32_t FailedMetadataUploadCount = 1;
+ int32_t MetadataUploadRetryCount = 3;
+ while ((MetadataUploadRetryCount-- > 0) && (FailedMetadataUploadCount > 0))
+ {
+ FailedMetadataUploadCount = 0;
+ for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++)
+ {
+ if (m_AbortFlag)
+ {
+ break;
+ }
+ const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash;
+ if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex])
+ {
+ const CbObject BlockMetaData =
+ BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]);
+ if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
+ {
+ m_Storage.BuildCacheStorage->PutBlobMetadatas(m_BuildId,
+ std::vector<IoHash>({BlockHash}),
+ std::vector<CbObject>({BlockMetaData}));
+ }
+ bool MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData);
+ if (MetadataSucceeded)
+ {
+ UploadStats.BlocksBytes += BlockMetaData.GetSize();
+ NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
+ UploadBlockMetadataCount++;
+ }
+ else
+ {
+ FailedMetadataUploadCount++;
+ }
+ }
+ }
+ }
+ if (UploadBlockMetadataCount > 0)
+ {
+ uint64_t ElapsedUS = UploadBlockMetadataTimer.GetElapsedTimeUs();
+ UploadStats.ElapsedWallTimeUS += ElapsedUS;
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_OPERATION_LOG_INFO(m_LogOutput,
+ "Uploaded metadata for {} blocks in {}",
+ UploadBlockMetadataCount,
+ NiceTimeSpanMs(ElapsedUS / 1000));
+ }
+ }
+ }
+
+ m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::PutBuildPartStats, StepCount);
+
+ m_Storage.BuildStorage->PutBuildPartStats(
+ m_BuildId,
+ Part.PartId,
+ {{"totalSize", double(Part.LocalFolderScanStats.FoundFileByteCount.load())},
+ {"reusedRatio", AcceptedByteCountPercent / 100.0},
+ {"reusedBlockCount", double(FindBlocksStats.AcceptedBlockCount)},
+ {"reusedBlockByteCount", double(ReuseBlocksStats.AcceptedRawByteCount)},
+ {"newBlockCount", double(FindBlocksStats.NewBlocksCount)},
+ {"newBlockByteCount", double(FindBlocksStats.NewBlocksChunkByteCount)},
+ {"uploadedCount", double(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load())},
+ {"uploadedByteCount", double(UploadStats.BlocksBytes.load() + UploadStats.ChunksBytes.load())},
+ {"uploadedBytesPerSec",
+ double(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.ChunksBytes + UploadStats.BlocksBytes))},
+ {"elapsedTimeSec", double(UploadTimer.GetElapsedTimeMs() / 1000.0)}});
+
+ m_LocalFolderScanStats += Part.LocalFolderScanStats;
+ m_ChunkingStats += ChunkingStats;
+ m_FindBlocksStats += FindBlocksStats;
+ m_ReuseBlocksStats += ReuseBlocksStats;
+ m_UploadStats += UploadStats;
+ m_GenerateBlocksStats += GenerateBlocksStats;
+ m_LooseChunksStats += LooseChunksStats;
+}
+
+void
BuildsOperationUploadFolder::UploadPartBlobs(const ChunkedFolderContent& Content,
const ChunkedContentLookup& Lookup,
std::span<IoHash> RawHashes,
@@ -7179,4 +7348,863 @@ BuildsOperationValidateBuildPart::ValidateChunkBlock(IoBuffer&& Payload,
return GetChunkBlockDescription(BlockBuffer.Flatten(), BlobHash);
}
+std::vector<std::pair<Oid, std::string>>
+ResolveBuildPartNames(CbObjectView BuildObject,
+ const Oid& BuildId,
+ const std::vector<Oid>& BuildPartIds,
+ std::span<const std::string> BuildPartNames,
+ std::uint64_t& OutPreferredMultipartChunkSize)
+{
+ std::vector<std::pair<Oid, std::string>> Result;
+ {
+ CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView();
+ if (!PartsObject)
+ {
+ throw std::runtime_error("Build object does not have a 'parts' object");
+ }
+
+ OutPreferredMultipartChunkSize = BuildObject["chunkSize"sv].AsUInt64(OutPreferredMultipartChunkSize);
+
+ std::vector<std::pair<Oid, std::string>> AvailableParts;
+
+ for (CbFieldView PartView : PartsObject)
+ {
+ const std::string BuildPartName = std::string(PartView.GetName());
+ const Oid BuildPartId = PartView.AsObjectId();
+ if (BuildPartId == Oid::Zero)
+ {
+ ExtendableStringBuilder<128> SB;
+ for (CbFieldView ScanPartView : PartsObject)
+ {
+ SB.Append(fmt::format("\n {}: {}", ScanPartView.GetName(), ScanPartView.AsObjectId()));
+ }
+ throw std::runtime_error(fmt::format("Build object parts does not have a '{}' object id{}", BuildPartName, SB.ToView()));
+ }
+ AvailableParts.push_back({BuildPartId, BuildPartName});
+ }
+
+ if (BuildPartIds.empty() && BuildPartNames.empty())
+ {
+ Result = AvailableParts;
+ }
+ else
+ {
+ for (const std::string& BuildPartName : BuildPartNames)
+ {
+ if (auto It = std::find_if(AvailableParts.begin(),
+ AvailableParts.end(),
+ [&BuildPartName](const auto& Part) { return Part.second == BuildPartName; });
+ It != AvailableParts.end())
+ {
+ Result.push_back(*It);
+ }
+ else
+ {
+ throw std::runtime_error(fmt::format("Build {} object does not have a part named '{}'", BuildId, BuildPartName));
+ }
+ }
+ for (const Oid& BuildPartId : BuildPartIds)
+ {
+ if (auto It = std::find_if(AvailableParts.begin(),
+ AvailableParts.end(),
+ [&BuildPartId](const auto& Part) { return Part.first == BuildPartId; });
+ It != AvailableParts.end())
+ {
+ Result.push_back(*It);
+ }
+ else
+ {
+ throw std::runtime_error(fmt::format("Build {} object does not have a part with id '{}'", BuildId, BuildPartId));
+ }
+ }
+ }
+
+ if (Result.empty())
+ {
+ throw std::runtime_error(fmt::format("Build object does not have any parts", BuildId));
+ }
+ }
+ return Result;
+}
+
+ChunkedFolderContent
+GetRemoteContent(OperationLogOutput& Output,
+ StorageInstance& Storage,
+ const Oid& BuildId,
+ const std::vector<std::pair<Oid, std::string>>& BuildParts,
+ std::span<const std::string> IncludeWildcards,
+ std::span<const std::string> ExcludeWildcards,
+ std::unique_ptr<ChunkingController>& OutChunkController,
+ std::vector<ChunkedFolderContent>& OutPartContents,
+ std::vector<ChunkBlockDescription>& OutBlockDescriptions,
+ std::vector<IoHash>& OutLooseChunkHashes,
+ bool IsQuiet,
+ bool IsVerbose,
+ bool DoExtraContentVerify)
+{
+ ZEN_TRACE_CPU("GetRemoteContent");
+
+ Stopwatch GetBuildPartTimer;
+ const Oid BuildPartId = BuildParts[0].first;
+ const std::string_view BuildPartName = BuildParts[0].second;
+ CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
+ if (!IsQuiet)
+ {
+ ZEN_OPERATION_LOG_INFO(Output,
+ "GetBuildPart {} ('{}') took {}. Payload size: {}",
+ BuildPartId,
+ BuildPartName,
+ NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
+ NiceBytes(BuildPartManifest.GetSize()));
+ ZEN_OPERATION_LOG_INFO(Output, "{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv));
+ }
+
+ {
+ CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView();
+ std::string_view ChunkerName = Chunker["name"sv].AsString();
+ CbObjectView Parameters = Chunker["parameters"sv].AsObjectView();
+ OutChunkController = CreateChunkingController(ChunkerName, Parameters);
+ }
+
+ auto ParseBuildPartManifest = [&Output, IsQuiet, IsVerbose, DoExtraContentVerify](
+ StorageInstance& Storage,
+ const Oid& BuildId,
+ const Oid& BuildPartId,
+ CbObject BuildPartManifest,
+ std::span<const std::string> IncludeWildcards,
+ std::span<const std::string> ExcludeWildcards,
+ ChunkedFolderContent& OutRemoteContent,
+ std::vector<ChunkBlockDescription>& OutBlockDescriptions,
+ std::vector<IoHash>& OutLooseChunkHashes) {
+ std::vector<uint32_t> AbsoluteChunkOrders;
+ std::vector<uint64_t> LooseChunkRawSizes;
+ std::vector<IoHash> BlockRawHashes;
+
+ ReadBuildContentFromCompactBinary(BuildPartManifest,
+ OutRemoteContent.Platform,
+ OutRemoteContent.Paths,
+ OutRemoteContent.RawHashes,
+ OutRemoteContent.RawSizes,
+ OutRemoteContent.Attributes,
+ OutRemoteContent.ChunkedContent.SequenceRawHashes,
+ OutRemoteContent.ChunkedContent.ChunkCounts,
+ AbsoluteChunkOrders,
+ OutLooseChunkHashes,
+ LooseChunkRawSizes,
+ BlockRawHashes);
+
+ // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them
+
+ {
+ bool AttemptFallback = false;
+ OutBlockDescriptions = GetBlockDescriptions(Output,
+ *Storage.BuildStorage,
+ Storage.BuildCacheStorage.get(),
+ BuildId,
+ BuildPartId,
+ BlockRawHashes,
+ AttemptFallback,
+ IsQuiet,
+ IsVerbose);
+ }
+
+ CalculateLocalChunkOrders(AbsoluteChunkOrders,
+ OutLooseChunkHashes,
+ LooseChunkRawSizes,
+ OutBlockDescriptions,
+ OutRemoteContent.ChunkedContent.ChunkHashes,
+ OutRemoteContent.ChunkedContent.ChunkRawSizes,
+ OutRemoteContent.ChunkedContent.ChunkOrders,
+ DoExtraContentVerify);
+
+ if (!IncludeWildcards.empty() || !ExcludeWildcards.empty())
+ {
+ std::vector<std::filesystem::path> DeletedPaths;
+ for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths)
+ {
+ if (!IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(RemotePath.generic_string()), /*CaseSensitive*/ true))
+ {
+ DeletedPaths.push_back(RemotePath);
+ }
+ }
+
+ if (!DeletedPaths.empty())
+ {
+ OutRemoteContent = DeletePathsFromChunkedContent(OutRemoteContent, DeletedPaths);
+ InlineRemoveUnusedHashes(OutLooseChunkHashes, OutRemoteContent.ChunkedContent.ChunkHashes);
+ }
+ }
+
+#if ZEN_BUILD_DEBUG
+ ValidateChunkedFolderContent(OutRemoteContent, OutBlockDescriptions, OutLooseChunkHashes, IncludeWildcards, ExcludeWildcards);
+#endif // ZEN_BUILD_DEBUG
+ };
+
+ OutPartContents.resize(1);
+ ParseBuildPartManifest(Storage,
+ BuildId,
+ BuildPartId,
+ BuildPartManifest,
+ IncludeWildcards,
+ ExcludeWildcards,
+ OutPartContents[0],
+ OutBlockDescriptions,
+ OutLooseChunkHashes);
+ ChunkedFolderContent RemoteContent;
+ if (BuildParts.size() > 1)
+ {
+ std::vector<ChunkBlockDescription> OverlayBlockDescriptions;
+ std::vector<IoHash> OverlayLooseChunkHashes;
+ for (size_t PartIndex = 1; PartIndex < BuildParts.size(); PartIndex++)
+ {
+ const Oid& OverlayBuildPartId = BuildParts[PartIndex].first;
+ const std::string& OverlayBuildPartName = BuildParts[PartIndex].second;
+ Stopwatch GetOverlayBuildPartTimer;
+ CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId);
+ if (!IsQuiet)
+ {
+ ZEN_OPERATION_LOG_INFO(Output,
+ "GetBuildPart {} ('{}') took {}. Payload size: {}",
+ OverlayBuildPartId,
+ OverlayBuildPartName,
+ NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()),
+ NiceBytes(OverlayBuildPartManifest.GetSize()));
+ }
+
+ ChunkedFolderContent OverlayPartContent;
+ std::vector<ChunkBlockDescription> OverlayPartBlockDescriptions;
+ std::vector<IoHash> OverlayPartLooseChunkHashes;
+
+ ParseBuildPartManifest(Storage,
+ BuildId,
+ OverlayBuildPartId,
+ OverlayBuildPartManifest,
+ IncludeWildcards,
+ ExcludeWildcards,
+ OverlayPartContent,
+ OverlayPartBlockDescriptions,
+ OverlayPartLooseChunkHashes);
+ OutPartContents.push_back(OverlayPartContent);
+ OverlayBlockDescriptions.insert(OverlayBlockDescriptions.end(),
+ OverlayPartBlockDescriptions.begin(),
+ OverlayPartBlockDescriptions.end());
+ OverlayLooseChunkHashes.insert(OverlayLooseChunkHashes.end(),
+ OverlayPartLooseChunkHashes.begin(),
+ OverlayPartLooseChunkHashes.end());
+ }
+
+ RemoteContent = MergeChunkedFolderContents(OutPartContents[0], std::span<const ChunkedFolderContent>(OutPartContents).subspan(1));
+ {
+ tsl::robin_set<IoHash> AllBlockHashes;
+ for (const ChunkBlockDescription& Description : OutBlockDescriptions)
+ {
+ AllBlockHashes.insert(Description.BlockHash);
+ }
+ for (const ChunkBlockDescription& Description : OverlayBlockDescriptions)
+ {
+ if (!AllBlockHashes.contains(Description.BlockHash))
+ {
+ AllBlockHashes.insert(Description.BlockHash);
+ OutBlockDescriptions.push_back(Description);
+ }
+ }
+ }
+ {
+ tsl::robin_set<IoHash> AllLooseChunkHashes(OutLooseChunkHashes.begin(), OutLooseChunkHashes.end());
+ for (const IoHash& OverlayLooseChunkHash : OverlayLooseChunkHashes)
+ {
+ if (!AllLooseChunkHashes.contains(OverlayLooseChunkHash))
+ {
+ AllLooseChunkHashes.insert(OverlayLooseChunkHash);
+ OutLooseChunkHashes.push_back(OverlayLooseChunkHash);
+ }
+ }
+ }
+ }
+ else
+ {
+ RemoteContent = OutPartContents[0];
+ }
+ return RemoteContent;
+}
+
+std::string
+GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix)
+{
+ ExtendableStringBuilder<512> SB;
+ std::vector<std::pair<std::string, std::string>> NameStringValuePairs;
+ for (CbFieldView Field : Object)
+ {
+ std::string_view Name = Field.GetName();
+ switch (CbValue Accessor = Field.GetValue(); Accessor.GetType())
+ {
+ case CbFieldType::String:
+ NameStringValuePairs.push_back({std::string(Name), std::string(Accessor.AsString())});
+ break;
+ case CbFieldType::IntegerPositive:
+ NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerPositive())});
+ break;
+ case CbFieldType::IntegerNegative:
+ NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerNegative())});
+ break;
+ case CbFieldType::Float32:
+ {
+ const float Value = Accessor.AsFloat32();
+ if (std::isfinite(Value))
+ {
+ NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.9g}", Value)});
+ }
+ else
+ {
+ NameStringValuePairs.push_back({std::string(Name), "null"});
+ }
+ }
+ break;
+ case CbFieldType::Float64:
+ {
+ const double Value = Accessor.AsFloat64();
+ if (std::isfinite(Value))
+ {
+ NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.17g}", Value)});
+ }
+ else
+ {
+ NameStringValuePairs.push_back({std::string(Name), "null"});
+ }
+ }
+ break;
+ case CbFieldType::BoolFalse:
+ NameStringValuePairs.push_back({std::string(Name), "false"});
+ break;
+ case CbFieldType::BoolTrue:
+ NameStringValuePairs.push_back({std::string(Name), "true"});
+ break;
+ case CbFieldType::Hash:
+ {
+ NameStringValuePairs.push_back({std::string(Name), Accessor.AsHash().ToHexString()});
+ }
+ break;
+ case CbFieldType::Uuid:
+ {
+ StringBuilder<Oid::StringLength + 1> Builder;
+ Accessor.AsUuid().ToString(Builder);
+ NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
+ }
+ break;
+ case CbFieldType::DateTime:
+ {
+ ExtendableStringBuilder<64> Builder;
+ Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601();
+ NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
+ }
+ break;
+ case CbFieldType::TimeSpan:
+ {
+ ExtendableStringBuilder<64> Builder;
+ const TimeSpan Span(Accessor.AsTimeSpanTicks());
+ if (Span.GetDays() == 0)
+ {
+ Builder << Span.ToString("%h:%m:%s.%n");
+ }
+ else
+ {
+ Builder << Span.ToString("%d.%h:%m:%s.%n");
+ }
+ NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
+ break;
+ }
+ case CbFieldType::ObjectId:
+ NameStringValuePairs.push_back({std::string(Name), Accessor.AsObjectId().ToString()});
+ break;
+ }
+ }
+ std::string::size_type LongestKey = 0;
+ for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs)
+ {
+ LongestKey = Max(KeyValue.first.length(), LongestKey);
+ }
+ for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs)
+ {
+ SB.Append(fmt::format("{}{:<{}}: {}{}", Prefix, KeyValue.first, LongestKey, KeyValue.second, Suffix));
+ }
+ return SB.ToString();
+}
+
+#if ZEN_WITH_TESTS
+
+namespace buildstorageoperations_testutils {
+ struct TestState
+ {
+ TestState(const std::filesystem::path& InRootPath)
+ : RootPath(InRootPath)
+ , LogOutput(CreateStandardLogOutput(Log))
+ , WorkerPool(2)
+ , NetworkPool(2)
+ {
+ }
+
+ void Initialize()
+ {
+ StoragePath = RootPath / "storage";
+ TempPath = RootPath / "temp";
+ SystemRootDir = RootPath / "sysroot";
+ ZenFolderPath = RootPath / ".zen";
+
+ CreateDirectories(TempPath);
+ CreateDirectories(StoragePath);
+
+ Storage.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false);
+ }
+
+ void CreateSourceData(const std::filesystem::path& Source, std::span<const std::string> Paths, std::span<const uint64_t> Sizes)
+ {
+ const std::filesystem::path SourcePath = RootPath / Source;
+ CreateDirectories(SourcePath);
+ for (size_t FileIndex = 0; FileIndex < Paths.size(); FileIndex++)
+ {
+ const std::string& FilePath = Paths[FileIndex];
+ const uint64_t FileSize = Sizes[FileIndex];
+ IoBuffer FileData = FileSize > 0 ? CreateSemiRandomBlob(FileSize) : IoBuffer{};
+ WriteFile(SourcePath / FilePath, FileData);
+ }
+ }
+
+ std::vector<std::pair<Oid, std::string>> Upload(const Oid& BuildId,
+ const Oid& BuildPartId,
+ const std::string_view BuildPartName,
+ const std::filesystem::path& Source,
+ const std::filesystem::path& ManifestPath)
+ {
+ const std::filesystem::path SourcePath = RootPath / Source;
+ CbObject MetaData;
+ BuildsOperationUploadFolder Upload(*LogOutput,
+ Storage,
+ AbortFlag,
+ PauseFlag,
+ WorkerPool,
+ NetworkPool,
+ BuildId,
+ SourcePath,
+ true,
+ MetaData,
+ BuildsOperationUploadFolder::Options{.TempDir = TempPath});
+ return Upload.Execute(BuildPartId, BuildPartName, ManifestPath);
+ }
+
+ void ValidateUpload(const Oid& BuildId, const std::vector<std::pair<Oid, std::string>>& Parts)
+ {
+ for (auto Part : Parts)
+ {
+ BuildsOperationValidateBuildPart Validate(*LogOutput,
+ *Storage.BuildStorage,
+ AbortFlag,
+ PauseFlag,
+ WorkerPool,
+ NetworkPool,
+ BuildId,
+ Part.first,
+ Part.second,
+ BuildsOperationValidateBuildPart::Options{});
+ Validate.Execute();
+ }
+ }
+
+ FolderContent Download(const Oid& BuildId,
+ const Oid& BuildPartId,
+ const std::string_view BuildPartName,
+ const std::filesystem::path& Target,
+ bool Append)
+ {
+ const std::filesystem::path TargetPath = RootPath / Target;
+
+ CreateDirectories(TargetPath);
+
+ uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
+ CbObject BuildObject = Storage.BuildStorage->GetBuild(BuildId);
+ std::vector<Oid> PartIds;
+ if (BuildPartId != Oid::Zero)
+ {
+ PartIds.push_back(BuildPartId);
+ }
+ std::vector<std::string> PartNames;
+ if (!BuildPartName.empty())
+ {
+ PartNames.push_back(std::string(BuildPartName));
+ }
+ std::vector<std::pair<Oid, std::string>> AllBuildParts =
+ ResolveBuildPartNames(BuildObject, BuildId, PartIds, PartNames, PreferredMultipartChunkSize);
+
+ std::vector<ChunkedFolderContent> PartContents;
+
+ std::unique_ptr<ChunkingController> ChunkController;
+
+ std::vector<ChunkBlockDescription> BlockDescriptions;
+ std::vector<IoHash> LooseChunkHashes;
+
+ ChunkedFolderContent RemoteContent = GetRemoteContent(*LogOutput,
+ Storage,
+ BuildId,
+ AllBuildParts,
+ {},
+ {},
+ ChunkController,
+ PartContents,
+ BlockDescriptions,
+ LooseChunkHashes,
+ /*IsQuiet*/ false,
+ /*IsVerbose*/ false,
+ /*DoExtraContentVerify*/ true);
+
+ GetFolderContentStatistics LocalFolderScanStats;
+
+ struct ContentVisitor : public GetDirectoryContentVisitor
+ {
+ virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content)
+ {
+ RwLock::ExclusiveLockScope _(ExistingPathsLock);
+ for (const std::filesystem::path& FileName : Content.FileNames)
+ {
+ if (RelativeRoot.empty())
+ {
+ ExistingPaths.push_back(FileName);
+ }
+ else
+ {
+ ExistingPaths.push_back(RelativeRoot / FileName);
+ }
+ }
+ }
+
+ RwLock ExistingPathsLock;
+ std::vector<std::filesystem::path> ExistingPaths;
+ } Visitor;
+
+ Latch PendingWorkCount(1);
+
+ GetDirectoryContent(TargetPath,
+ DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive,
+ Visitor,
+ WorkerPool,
+ PendingWorkCount);
+
+ PendingWorkCount.CountDown();
+ PendingWorkCount.Wait();
+
+ FolderContent CurrentLocalFolderState = GetValidFolderContent(
+ WorkerPool,
+ LocalFolderScanStats,
+ TargetPath,
+ Visitor.ExistingPaths,
+ [](uint64_t PathCount, uint64_t CompletedPathCount) { ZEN_UNUSED(PathCount, CompletedPathCount); },
+ 1000,
+ AbortFlag,
+ PauseFlag);
+
+ ChunkingStatistics LocalChunkingStats;
+ ChunkedFolderContent LocalContent = ChunkFolderContent(
+ LocalChunkingStats,
+ WorkerPool,
+ TargetPath,
+ CurrentLocalFolderState,
+ *ChunkController,
+ 1000,
+ [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { ZEN_UNUSED(IsAborted, IsPaused); },
+ AbortFlag,
+ PauseFlag);
+
+ if (Append)
+ {
+ RemoteContent = ApplyChunkedContentOverlay(LocalContent, RemoteContent, {}, {});
+ }
+
+ const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent);
+ const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent);
+
+ BuildsOperationUpdateFolder Download(*LogOutput,
+ Storage,
+ AbortFlag,
+ PauseFlag,
+ WorkerPool,
+ NetworkPool,
+ BuildId,
+ TargetPath,
+ LocalContent,
+ LocalLookup,
+ RemoteContent,
+ RemoteLookup,
+ BlockDescriptions,
+ LooseChunkHashes,
+ BuildsOperationUpdateFolder::Options{.SystemRootDir = SystemRootDir,
+ .ZenFolderPath = ZenFolderPath,
+ .ValidateCompletedSequences = true});
+ FolderContent ResultingState;
+ Download.Execute(ResultingState);
+
+ return ResultingState;
+ }
+
+ void ValidateDownload(std::span<const std::string> Paths,
+ std::span<const uint64_t> Sizes,
+ const std::filesystem::path& Source,
+ const std::filesystem::path& Target,
+ const FolderContent& DownloadContent)
+ {
+ const std::filesystem::path SourcePath = RootPath / Source;
+ const std::filesystem::path TargetPath = RootPath / Target;
+
+ CHECK_EQ(Paths.size(), DownloadContent.Paths.size());
+ tsl::robin_map<std::string, uint64_t> ExpectedSizes;
+ tsl::robin_map<std::string, IoHash> ExpectedHashes;
+ for (size_t Index = 0; Index < Paths.size(); Index++)
+ {
+ const std::string LookupString = std::filesystem::path(Paths[Index]).generic_string();
+ ExpectedSizes.insert_or_assign(LookupString, Sizes[Index]);
+ std::filesystem::path FilePath = SourcePath / Paths[Index];
+ const IoHash SourceHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(FilePath.make_preferred()));
+ ExpectedHashes.insert_or_assign(LookupString, SourceHash);
+ }
+ for (size_t Index = 0; Index < DownloadContent.Paths.size(); Index++)
+ {
+ const std::string LookupString = std::filesystem::path(DownloadContent.Paths[Index]).generic_string();
+ auto SizeIt = ExpectedSizes.find(LookupString);
+ CHECK_NE(SizeIt, ExpectedSizes.end());
+ CHECK_EQ(SizeIt->second, DownloadContent.RawSizes[Index]);
+ std::filesystem::path FilePath = TargetPath / DownloadContent.Paths[Index];
+ const IoHash DownloadedHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(FilePath.make_preferred()));
+ auto HashIt = ExpectedHashes.find(LookupString);
+ CHECK_NE(HashIt, ExpectedHashes.end());
+ CHECK_EQ(HashIt->second, DownloadedHash);
+ }
+ }
+
+ const std::filesystem::path RootPath;
+ std::filesystem::path StoragePath;
+ std::filesystem::path TempPath;
+ std::filesystem::path SystemRootDir;
+ std::filesystem::path ZenFolderPath;
+
+ LoggerRef Log = ConsoleLog();
+ std::unique_ptr<OperationLogOutput> LogOutput;
+
+ StorageInstance Storage;
+ BuildStorageBase::Statistics StorageStats;
+
+ WorkerThreadPool WorkerPool;
+ WorkerThreadPool NetworkPool;
+
+ std::atomic<bool> AbortFlag;
+ std::atomic<bool> PauseFlag;
+ };
+
+} // namespace buildstorageoperations_testutils
+
+TEST_CASE("buildstorageoperations.upload.folder")
+{
+ using namespace buildstorageoperations_testutils;
+
+ FastRandom BaseRandom;
+
+ const size_t FileCount = 11;
+
+ const std::string Paths[FileCount] = {{"file_1"},
+ {"file_2.exe"},
+ {"file_3.txt"},
+ {"dir_1/dir1_file_1.exe"},
+ {"dir_1/dir1_file_2.pdb"},
+ {"dir_1/dir1_file_3.txt"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"},
+ {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"},
+ {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}};
+ const uint64_t Sizes[FileCount] =
+ {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u};
+
+ ScopedTemporaryDirectory SourceFolder;
+ TestState State(SourceFolder.Path());
+ State.Initialize();
+ State.CreateSourceData("source", Paths, Sizes);
+
+ const Oid BuildId = Oid::NewOid();
+ const Oid BuildPartId = Oid::NewOid();
+ const std::string BuildPartName = "default";
+
+ auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", {});
+
+ CHECK_EQ(Result.size(), 1u);
+ CHECK_EQ(Result[0].first, BuildPartId);
+ CHECK_EQ(Result[0].second, BuildPartName);
+ State.ValidateUpload(BuildId, Result);
+
+ FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false);
+ CHECK_EQ(DownloadContent.Paths.size(), FileCount);
+ State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent);
+}
+
+TEST_CASE("buildstorageoperations.upload.manifest")
+{
+ using namespace buildstorageoperations_testutils;
+
+ FastRandom BaseRandom;
+
+ const size_t FileCount = 11;
+
+ const std::string Paths[FileCount] = {{"file_1"},
+ {"file_2.exe"},
+ {"file_3.txt"},
+ {"dir_1/dir1_file_1.exe"},
+ {"dir_1/dir1_file_2.pdb"},
+ {"dir_1/dir1_file_3.txt"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"},
+ {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"},
+ {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}};
+ const uint64_t Sizes[FileCount] =
+ {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u};
+
+ ScopedTemporaryDirectory SourceFolder;
+ TestState State(SourceFolder.Path());
+ State.Initialize();
+ State.CreateSourceData("source", Paths, Sizes);
+
+ std::span<const std::string> ManifestFiles(Paths);
+ ManifestFiles = ManifestFiles.subspan(0, FileCount / 2);
+
+ std::span<const uint64_t> ManifestSizes(Sizes);
+ ManifestSizes = ManifestSizes.subspan(0, FileCount / 2);
+
+ ExtendableStringBuilder<1024> Manifest;
+ for (const std::string& FilePath : ManifestFiles)
+ {
+ Manifest << FilePath << "\n";
+ }
+
+ WriteFile(State.RootPath / "manifest.txt", IoBuffer(IoBuffer::Wrap, Manifest.Data(), Manifest.Size()));
+
+ const Oid BuildId = Oid::NewOid();
+ const Oid BuildPartId = Oid::NewOid();
+ const std::string BuildPartName = "default";
+
+ auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", State.RootPath / "manifest.txt");
+
+ CHECK_EQ(Result.size(), 1u);
+ CHECK_EQ(Result[0].first, BuildPartId);
+ CHECK_EQ(Result[0].second, BuildPartName);
+ State.ValidateUpload(BuildId, Result);
+
+ FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false);
+ State.ValidateDownload(ManifestFiles, ManifestSizes, "source", "download", DownloadContent);
+}
+
+TEST_CASE("buildstorageoperations.upload.multipart")
+{
+ using namespace buildstorageoperations_testutils;
+
+ FastRandom BaseRandom;
+
+ const size_t FileCount = 11;
+
+ const std::string Paths[FileCount] = {{"file_1"},
+ {"file_2.exe"},
+ {"file_3.txt"},
+ {"dir_1/dir1_file_1.exe"},
+ {"dir_1/dir1_file_2.pdb"},
+ {"dir_1/dir1_file_3.txt"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"},
+ {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"},
+ {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}};
+ const uint64_t Sizes[FileCount] =
+ {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u};
+
+ ScopedTemporaryDirectory SourceFolder;
+ TestState State(SourceFolder.Path());
+ State.Initialize();
+ State.CreateSourceData("source", Paths, Sizes);
+
+ std::span<const std::string> ManifestFiles1(Paths);
+ ManifestFiles1 = ManifestFiles1.subspan(0, FileCount / 2);
+
+ std::span<const uint64_t> ManifestSizes1(Sizes);
+ ManifestSizes1 = ManifestSizes1.subspan(0, FileCount / 2);
+
+ std::span<const std::string> ManifestFiles2(Paths);
+ ManifestFiles2 = ManifestFiles2.subspan(FileCount / 2 - 1);
+
+ std::span<const uint64_t> ManifestSizes2(Sizes);
+ ManifestSizes2 = ManifestSizes2.subspan(FileCount / 2 - 1);
+
+ const Oid BuildPart1Id = Oid::NewOid();
+ const std::string BuildPart1Name = "part1";
+ const Oid BuildPart2Id = Oid::NewOid();
+ const std::string BuildPart2Name = "part2";
+ {
+ CbObjectWriter Writer;
+ Writer.BeginObject("parts"sv);
+ {
+ Writer.BeginObject(BuildPart1Name);
+ {
+ Writer.AddObjectId("partId"sv, BuildPart1Id);
+ Writer.BeginArray("files"sv);
+ for (const std::string& ManifestFile : ManifestFiles1)
+ {
+ Writer.AddString(ManifestFile);
+ }
+ Writer.EndArray(); // files
+ }
+ Writer.EndObject(); // part1
+
+ Writer.BeginObject(BuildPart2Name);
+ {
+ Writer.AddObjectId("partId"sv, BuildPart2Id);
+ Writer.BeginArray("files"sv);
+ for (const std::string& ManifestFile : ManifestFiles2)
+ {
+ Writer.AddString(ManifestFile);
+ }
+ Writer.EndArray(); // files
+ }
+ Writer.EndObject(); // part2
+ }
+ Writer.EndObject(); // parts
+
+ ExtendableStringBuilder<1024> Manifest;
+ CompactBinaryToJson(Writer.Save(), Manifest);
+ WriteFile(State.RootPath / "manifest.json", IoBuffer(IoBuffer::Wrap, Manifest.Data(), Manifest.Size()));
+ }
+
+ const Oid BuildId = Oid::NewOid();
+
+ auto Result = State.Upload(BuildId, {}, {}, "source", State.RootPath / "manifest.json");
+
+ CHECK_EQ(Result.size(), 2u);
+ CHECK_EQ(Result[0].first, BuildPart1Id);
+ CHECK_EQ(Result[0].second, BuildPart1Name);
+ CHECK_EQ(Result[1].first, BuildPart2Id);
+ CHECK_EQ(Result[1].second, BuildPart2Name);
+ State.ValidateUpload(BuildId, Result);
+
+ FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false);
+ State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent);
+
+ FolderContent Part1DownloadContent = State.Download(BuildId, BuildPart1Id, {}, "download_part1", /* Append */ false);
+ State.ValidateDownload(ManifestFiles1, ManifestSizes1, "source", "download_part1", Part1DownloadContent);
+
+ FolderContent Part2DownloadContent = State.Download(BuildId, Oid::Zero, BuildPart2Name, "download_part2", /* Append */ false);
+ State.ValidateDownload(ManifestFiles2, ManifestSizes2, "source", "download_part2", Part2DownloadContent);
+
+ (void)State.Download(BuildId, BuildPart1Id, BuildPart1Name, "download_part1+2", /* Append */ false);
+ FolderContent Part1And2DownloadContent = State.Download(BuildId, BuildPart2Id, {}, "download_part1+2", /* Append */ true);
+ State.ValidateDownload(Paths, Sizes, "source", "download_part1+2", Part1And2DownloadContent);
+}
+
+void
+buildstorageoperations_forcelink()
+{
+}
+
+#endif // ZEN_WITH_TESTS
+
} // namespace zen
diff --git a/src/zenremotestore/chunking/chunkblock.cpp b/src/zenremotestore/chunking/chunkblock.cpp
index a5d0db205..c4d8653f4 100644
--- a/src/zenremotestore/chunking/chunkblock.cpp
+++ b/src/zenremotestore/chunking/chunkblock.cpp
@@ -297,6 +297,7 @@ FindReuseBlocks(OperationLogOutput& Output,
if (ChunkCount > 0)
{
+ size_t AcceptedChunkCount = 0;
if (!KnownBlocks.empty())
{
Stopwatch ReuseTimer;
@@ -420,6 +421,7 @@ FindReuseBlocks(OperationLogOutput& Output,
{
ChunkFound[ChunkIndex] = true;
}
+ AcceptedChunkCount += FoundChunkIndexes.size();
Stats.AcceptedChunkCount += FoundChunkIndexes.size();
Stats.AcceptedByteCount += AdjustedReuseSize;
Stats.AcceptedRawByteCount += AdjustedRawReuseSize;
@@ -440,7 +442,8 @@ FindReuseBlocks(OperationLogOutput& Output,
}
}
}
- OutUnusedChunkIndexes.reserve(ChunkIndexes.size() - Stats.AcceptedChunkCount);
+
+ OutUnusedChunkIndexes.reserve(ChunkIndexes.size() - AcceptedChunkCount);
for (uint32_t ChunkIndex : ChunkIndexes)
{
if (!ChunkFound[ChunkIndex])
diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
index 32c8bda01..f9fe949a6 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
@@ -11,6 +11,7 @@
#include <zenutil/bufferedwritefilecache.h>
#include <atomic>
+#include <future>
#include <memory>
ZEN_THIRD_PARTY_INCLUDES_START
@@ -421,6 +422,21 @@ struct FindBlocksStatistics
uint64_t NewBlocksCount = 0;
uint64_t NewBlocksChunkCount = 0;
uint64_t NewBlocksChunkByteCount = 0;
+
+ FindBlocksStatistics& operator+=(const FindBlocksStatistics& Rhs)
+ {
+ FindBlockTimeMS += Rhs.FindBlockTimeMS;
+ PotentialChunkCount += Rhs.PotentialChunkCount;
+ PotentialChunkByteCount += Rhs.PotentialChunkByteCount;
+ FoundBlockCount += Rhs.FoundBlockCount;
+ FoundBlockChunkCount += Rhs.FoundBlockChunkCount;
+ FoundBlockByteCount += Rhs.FoundBlockByteCount;
+ AcceptedBlockCount += Rhs.AcceptedBlockCount;
+ NewBlocksCount += Rhs.NewBlocksCount;
+ NewBlocksChunkCount += Rhs.NewBlocksChunkCount;
+ NewBlocksChunkByteCount += Rhs.NewBlocksChunkByteCount;
+ return *this;
+ }
};
struct UploadStatistics
@@ -528,15 +544,14 @@ public:
WorkerThreadPool& IOWorkerPool,
WorkerThreadPool& NetworkPool,
const Oid& BuildId,
- const Oid& BuildPartId,
- const std::string_view BuildPartName,
const std::filesystem::path& Path,
- const std::filesystem::path& ManifestPath,
bool CreateBuild,
const CbObject& MetaData,
const Options& Options);
- void Execute();
+ std::vector<std::pair<Oid, std::string>> Execute(const Oid& BuildPartId,
+ const std::string_view BuildPartName,
+ const std::filesystem::path& ManifestPath);
DiskStatistics m_DiskStats;
GetFolderContentStatistics m_LocalFolderScanStats;
@@ -548,7 +563,42 @@ public:
LooseChunksStatistics m_LooseChunksStats;
private:
- std::vector<std::filesystem::path> ParseManifest(const std::filesystem::path& Path, const std::filesystem::path& ManifestPath);
+ struct PrepareBuildResult
+ {
+ std::vector<ChunkBlockDescription> KnownBlocks;
+ uint64_t PreferredMultipartChunkSize = 0;
+ uint64_t PayloadSize = 0;
+ uint64_t PrepareBuildTimeMs = 0;
+ uint64_t FindBlocksTimeMs = 0;
+ uint64_t ElapsedTimeMs = 0;
+ };
+
+ PrepareBuildResult PrepareBuild();
+
+ struct PartManifest
+ {
+ struct Part
+ {
+ Oid PartId;
+ std::string PartName;
+ std::vector<std::filesystem::path> Files;
+ };
+ std::vector<Part> Parts;
+ };
+
+ struct UploadPart
+ {
+ Oid PartId = Oid::Zero;
+ std::string PartName;
+ FolderContent Content;
+ uint64_t TotalRawSize = 0;
+ GetFolderContentStatistics LocalFolderScanStats;
+ };
+
+ std::vector<BuildsOperationUploadFolder::UploadPart> ReadFolder();
+ std::vector<UploadPart> ReadManifestParts(const std::filesystem::path& ManifestPath);
+
+ PartManifest ParseManifest(const std::filesystem::path& Path, const std::filesystem::path& ManifestPath);
bool IsAcceptedFolder(const std::string_view& RelativePath) const;
bool IsAcceptedFile(const std::string_view& RelativePath) const;
@@ -571,7 +621,9 @@ private:
void GenerateBuildBlocks(const ChunkedFolderContent& Content,
const ChunkedContentLookup& Lookup,
const std::vector<std::vector<uint32_t>>& NewBlockChunks,
- GeneratedBlocks& OutBlocks);
+ GeneratedBlocks& OutBlocks,
+ GenerateBlocksStatistics& GenerateBlocksStats,
+ UploadStatistics& UploadStats);
std::vector<uint32_t> CalculateAbsoluteChunkOrders(const std::span<const IoHash> LocalChunkHashes,
const std::span<const uint32_t> LocalChunkOrder,
@@ -594,6 +646,24 @@ private:
CompositeBuffer&& HeaderBuffer,
const std::vector<uint32_t>& ChunksInBlock);
+ enum class PartTaskSteps : uint32_t
+ {
+ ChunkPartContent = 0,
+ CalculateDelta,
+ GenerateBlocks,
+ BuildPartManifest,
+ UploadBuildPart,
+ UploadAttachments,
+ PutBuildPartStats,
+ StepCount
+ };
+
+ void UploadBuildPart(ChunkingController& ChunkController,
+ uint32_t PartIndex,
+ const UploadPart& Part,
+ uint32_t PartStepOffset,
+ uint32_t StepCount);
+
void UploadPartBlobs(const ChunkedFolderContent& Content,
const ChunkedContentLookup& Lookup,
std::span<IoHash> RawHashes,
@@ -617,16 +687,16 @@ private:
WorkerThreadPool& m_IOWorkerPool;
WorkerThreadPool& m_NetworkPool;
const Oid m_BuildId;
- const Oid m_BuildPartId;
- const std::string m_BuildPartName;
const std::filesystem::path m_Path;
- const std::filesystem::path m_ManifestPath;
const bool m_CreateBuild; // ?? Member?
const CbObject m_MetaData; // ?? Member
const Options m_Options;
tsl::robin_set<uint32_t> m_NonCompressableExtensionHashes;
+
+ std::future<PrepareBuildResult> m_PrepBuildResultFuture;
+ PrepareBuildResult m_PrepBuildResult;
};
struct ValidateStatistics
@@ -730,4 +800,30 @@ CompositeBuffer ValidateBlob(std::atomic<bool>& AbortFlag,
uint64_t& OutCompressedSize,
uint64_t& OutDecompressedSize);
+std::vector<std::pair<Oid, std::string>> ResolveBuildPartNames(CbObjectView BuildObject,
+ const Oid& BuildId,
+ const std::vector<Oid>& BuildPartIds,
+ std::span<const std::string> BuildPartNames,
+ std::uint64_t& OutPreferredMultipartChunkSize);
+
+ChunkedFolderContent GetRemoteContent(OperationLogOutput& Output,
+ StorageInstance& Storage,
+ const Oid& BuildId,
+ const std::vector<std::pair<Oid, std::string>>& BuildParts,
+ std::span<const std::string> IncludeWildcards,
+ std::span<const std::string> ExcludeWildcards,
+ std::unique_ptr<ChunkingController>& OutChunkController,
+ std::vector<ChunkedFolderContent>& OutPartContents,
+ std::vector<ChunkBlockDescription>& OutBlockDescriptions,
+ std::vector<IoHash>& OutLooseChunkHashes,
+ bool IsQuiet,
+ bool IsVerbose,
+ bool DoExtraContentVerify);
+
+std::string GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix);
+
+#if ZEN_WITH_TESTS
+void buildstorageoperations_forcelink();
+#endif // ZEN_WITH_TESTS
+
} // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h
index 295d275d1..d339b0f94 100644
--- a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h
+++ b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h
@@ -47,6 +47,19 @@ struct ReuseBlocksStatistics
uint64_t RejectedByteCount = 0;
uint64_t AcceptedReduntantChunkCount = 0;
uint64_t AcceptedReduntantByteCount = 0;
+
+ ReuseBlocksStatistics& operator+=(const ReuseBlocksStatistics& Rhs)
+ {
+ AcceptedChunkCount += Rhs.AcceptedChunkCount;
+ AcceptedByteCount += Rhs.AcceptedByteCount;
+ AcceptedRawByteCount += Rhs.AcceptedRawByteCount;
+ RejectedBlockCount += Rhs.RejectedBlockCount;
+ RejectedChunkCount += Rhs.RejectedChunkCount;
+ RejectedByteCount += Rhs.RejectedByteCount;
+ AcceptedReduntantChunkCount += Rhs.AcceptedReduntantChunkCount;
+ AcceptedReduntantByteCount += Rhs.AcceptedReduntantByteCount;
+ return *this;
+ }
};
class OperationLogOutput;
diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h
index 78f20a727..1e8e878df 100644
--- a/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h
+++ b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h
@@ -55,11 +55,30 @@ FolderContent LoadFolderContentToCompactBinary(CbObjectView Input);
struct GetFolderContentStatistics
{
+ GetFolderContentStatistics() {}
+ GetFolderContentStatistics(GetFolderContentStatistics&& Rhs)
+ : FoundFileCount(Rhs.FoundFileCount.load())
+ , FoundFileByteCount(Rhs.FoundFileByteCount.load())
+ , AcceptedFileCount(Rhs.AcceptedFileCount.load())
+ , AcceptedFileByteCount(Rhs.AcceptedFileByteCount.load())
+ , ElapsedWallTimeUS(Rhs.ElapsedWallTimeUS)
+ {
+ }
std::atomic<uint64_t> FoundFileCount = 0;
std::atomic<uint64_t> FoundFileByteCount = 0;
std::atomic<uint64_t> AcceptedFileCount = 0;
std::atomic<uint64_t> AcceptedFileByteCount = 0;
uint64_t ElapsedWallTimeUS = 0;
+
+ inline GetFolderContentStatistics& operator+=(const GetFolderContentStatistics& Rhs)
+ {
+ FoundFileCount += Rhs.FoundFileCount;
+ FoundFileByteCount += Rhs.FoundFileByteCount;
+ AcceptedFileCount += Rhs.AcceptedFileCount;
+ AcceptedFileByteCount += Rhs.AcceptedFileByteCount;
+ ElapsedWallTimeUS += Rhs.ElapsedWallTimeUS;
+ return *this;
+ }
};
FolderContent GetFolderContent(GetFolderContentStatistics& Stats,
diff --git a/src/zenremotestore/zenremotestore.cpp b/src/zenremotestore/zenremotestore.cpp
index 7f785599f..ecf85794d 100644
--- a/src/zenremotestore/zenremotestore.cpp
+++ b/src/zenremotestore/zenremotestore.cpp
@@ -3,6 +3,7 @@
#include <zenremotestore/zenremotestore.h>
#include <zenremotestore/builds/buildsavedstate.h>
+#include <zenremotestore/builds/buildstorageoperations.h>
#include <zenremotestore/chunking/chunkedcontent.h>
#include <zenremotestore/chunking/chunkedfile.h>
#include <zenremotestore/filesystemutils.h>
@@ -16,10 +17,10 @@ void
zenremotestore_forcelinktests()
{
buildsavedstate_forcelink();
+ buildstorageoperations_forcelink();
chunkblock_forcelink();
chunkedcontent_forcelink();
chunkedfile_forcelink();
- chunkedcontent_forcelink();
filesystemutils_forcelink();
remoteprojectstore_forcelink();
}
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-23 18:29:27 +0000