diff options
| author | Dan Engelbrecht <[email protected]> | 2025-05-05 18:57:15 +0200 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-05-05 18:57:15 +0200 |
| commit | fd5b8fde6bd838b17e6100f01d466bcbf01f91fe (patch) | |
| tree | 9fd20b6c4e7ba1da6d654eae74391c84981c354d /src/zen/cmds/builds_cmd.cpp | |
| parent | silence Out Of Disk errors to sentry (#378) (diff) | |
| download | archived-zen-fd5b8fde6bd838b17e6100f01d466bcbf01f91fe.tar.xz archived-zen-fd5b8fde6bd838b17e6100f01d466bcbf01f91fe.zip | |
UE style formatted progress output (#380)
* add UE style @progress style progress
Diffstat (limited to 'src/zen/cmds/builds_cmd.cpp')
| -rw-r--r-- | src/zen/cmds/builds_cmd.cpp | 206 |
1 files changed, 173 insertions, 33 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 15c635594..134cde0f8 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -145,8 +145,24 @@ namespace { const std::vector<std::string_view> DefaultExcludeFolders({UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName}); const std::vector<std::string_view> DefaultExcludeExtensions({}); - static bool IsVerbose = false; - static bool UsePlainProgress = false; + static bool IsVerbose = false; + static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; + + uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode) + { + switch (InMode) + { + case ProgressBar::Mode::Plain: + return 5000; + case ProgressBar::Mode::Pretty: + return 200; + case ProgressBar::Mode::Log: + return 2000; + default: + ZEN_ASSERT(false); + return 0; + } + } #define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ if (IsVerbose) \ @@ -333,7 +349,7 @@ namespace { ZEN_TRACE_CPU("CleanDirectory"); Stopwatch Timer; - ProgressBar Progress(UsePlainProgress); + ProgressBar Progress(ProgressMode, "Clean Folder"); std::atomic<bool> CleanWipe = true; std::atomic<uint64_t> DiscoveredItemCount = 0; @@ -453,7 +469,7 @@ namespace { uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs(); - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); LastUpdateTimeMs = Timer.GetElapsedTimeMs(); @@ -492,7 +508,7 @@ namespace { } uint64_t NowMs = Timer.GetElapsedTimeMs(); - if ((NowMs - LastUpdateTimeMs) >= (UsePlainProgress ? 5000 : 200)) + if ((NowMs - LastUpdateTimeMs) >= GetUpdateDelayMS(ProgressMode)) { LastUpdateTimeMs = NowMs; @@ -1814,15 +1830,34 @@ namespace { ValidateStatistics& ValidateStats, DownloadStatistics& DownloadStats) { + ZEN_TRACE_CPU("ValidateBuildPart"); + + ProgressBar::SetLogOperationName(ProgressMode, "Validate Part"); + + enum TaskSteps : uint32_t + { + FetchBuild, + FetchBuildPart, + ValidateBlobs, + Cleanup, + StepCount + }; + + auto EndProgress = + MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); + Stopwatch Timer; - auto _ = MakeGuard([&]() { + auto _ = MakeGuard([&]() { ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}", BuildId, BuildPartId, BuildPartName, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); - CbObject Build = Storage.GetBuild(BuildId); + + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::FetchBuild, TaskSteps::StepCount); + + CbObject Build = Storage.GetBuild(BuildId); if (!BuildPartName.empty()) { BuildPartId = Build["parts"sv].AsObjectView()[BuildPartName].AsObjectId(); @@ -1837,6 +1872,9 @@ namespace { { PreferredMultipartChunkSize = ChunkSize; } + + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::FetchBuildPart, TaskSteps::StepCount); + CbObject BuildPart = Storage.GetBuildPart(BuildId, BuildPartId); ValidateStats.BuildPartSize = BuildPart.GetSize(); ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize())); @@ -1876,7 +1914,9 @@ namespace { } }); - ProgressBar ProgressBar(UsePlainProgress); + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::ValidateBlobs, TaskSteps::StepCount); + + ProgressBar ProgressBar(ProgressMode, "Validate Blobs"); uint64_t AttachmentsToVerifyCount = ChunkAttachments.size() + BlockAttachments.size(); FilteredRate FilteredDownloadedBytesPerSecond; @@ -1990,7 +2030,7 @@ namespace { Work.DefaultErrorFunction()); } - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); const uint64_t DownloadedAttachmentCount = DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount; @@ -2020,6 +2060,8 @@ namespace { ProgressBar.Finish(); ValidateStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); + + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); } void ArrangeChunksIntoBlocks(const ChunkedFolderContent& Content, @@ -2226,7 +2268,7 @@ namespace { const std::size_t NewBlockCount = NewBlockChunks.size(); if (NewBlockCount > 0) { - ProgressBar ProgressBar(UsePlainProgress); + ProgressBar ProgressBar(ProgressMode, "Generate Blocks"); OutBlocks.BlockDescriptions.resize(NewBlockCount); OutBlocks.BlockSizes.resize(NewBlockCount); @@ -2389,7 +2431,7 @@ namespace { Work.DefaultErrorFunction()); } - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); FilteredGeneratedBytesPerSecond.Update(GenerateBlocksStats.GeneratedBlockByteCount.load()); @@ -2439,7 +2481,7 @@ namespace { { ZEN_TRACE_CPU("UploadPartBlobs"); { - ProgressBar ProgressBar(UsePlainProgress); + ProgressBar ProgressBar(ProgressMode, "Upload Blobs"); WorkerThreadPool& ReadChunkPool = GetIOWorkerPool(); WorkerThreadPool& UploadChunkPool = GetNetworkPool(); @@ -2752,7 +2794,7 @@ namespace { Work.DefaultErrorFunction()); } - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); FilteredCompressedBytesPerSecond.Update(LooseChunksStats.CompressedChunkRawBytes.load()); FilteredGenerateBlockBytesPerSecond.Update(GeneratedBlockByteCount.load()); @@ -2971,6 +3013,23 @@ namespace { bool IgnoreExistingBlocks, bool PostUploadVerify) { + ZEN_TRACE_CPU("UploadFolder"); + + ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder"); + + enum TaskSteps : uint32_t + { + PrepareBuild, + CalculateDelta, + Upload, + Validate, + Cleanup, + StepCount + }; + + auto EndProgress = + MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); + Stopwatch ProcessTimer; const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath); @@ -2986,6 +3045,8 @@ namespace { CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath)); CreateDirectories(ZenTempChunkFolderPath(ZenFolderPath)); + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::PrepareBuild, TaskSteps::StepCount); + std::uint64_t TotalRawSize = 0; CbObject ChunkerParameters; @@ -3157,7 +3218,7 @@ namespace { return true; }, GetIOWorkerPool(), - UsePlainProgress ? 5000 : 200, + GetUpdateDelayMS(ProgressMode), [&](bool, std::ptrdiff_t) { ZEN_CONSOLE_VERBOSE("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); }, @@ -3212,7 +3273,7 @@ namespace { TotalRawSize = std::accumulate(Content.RawSizes.begin(), Content.RawSizes.end(), std::uint64_t(0)); { - ProgressBar ProgressBar(UsePlainProgress); + ProgressBar ProgressBar(ProgressMode, "Scan Files"); FilteredRate FilteredBytesHashed; FilteredBytesHashed.Start(); LocalContent = ChunkFolderContent( @@ -3221,7 +3282,7 @@ namespace { Path, Content, *ChunkController, - UsePlainProgress ? 5000 : 200, + GetUpdateDelayMS(ProgressMode), [&](bool, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", @@ -3277,6 +3338,8 @@ namespace { PrepBuildResult.KnownBlocks.size(), NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs))); + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CalculateDelta, TaskSteps::StepCount); + const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PrepBuildResult.PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; Stopwatch BlockArrangeTimer; @@ -3386,6 +3449,8 @@ namespace { UploadStatistics UploadStats; GeneratedBlocks NewBlocks; + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Upload, TaskSteps::StepCount); + if (!NewBlockChunks.empty()) { Stopwatch GenerateBuildBlocksTimer; @@ -3710,6 +3775,7 @@ namespace { DownloadStatistics ValidateDownloadStats; if (PostUploadVerify && !AbortFlag) { + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Validate, TaskSteps::StepCount); ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, BuildPartName, ValidateStats, ValidateDownloadStats); } @@ -3961,6 +4027,8 @@ namespace { {"uploadedBytesPerSec", double(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.ChunksBytes + UploadStats.BlocksBytes))}, {"elapsedTimeSec", double(ProcessTimer.GetElapsedTimeMs() / 1000.0)}}); + + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); } void VerifyFolder(const ChunkedFolderContent& Content, @@ -3972,7 +4040,7 @@ namespace { Stopwatch Timer; - ProgressBar ProgressBar(UsePlainProgress); + ProgressBar ProgressBar(ProgressMode, "Verify Files"); WorkerThreadPool& VerifyPool = GetIOWorkerPool(); @@ -4114,7 +4182,7 @@ namespace { }); } - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}", VerifyFolderStats.FilesVerified.load(), @@ -5085,7 +5153,7 @@ namespace { Stopwatch Timer; auto _ = MakeGuard([&LocalFolderScanStats, &Timer]() { LocalFolderScanStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); }); - ProgressBar ProgressBar(UsePlainProgress); + ProgressBar ProgressBar(ProgressMode, "Check Files"); ParallellWork Work(AbortFlag); std::atomic<uint64_t> CompletedPathCount = 0; @@ -5751,7 +5819,7 @@ namespace { WorkerThreadPool& NetworkPool = GetNetworkPool(); WorkerThreadPool& WritePool = GetIOWorkerPool(); - ProgressBar WriteProgressBar(UsePlainProgress); + ProgressBar WriteProgressBar(ProgressMode, PrimeCacheOnly ? "Downloading" : "Writing"); ParallellWork Work(AbortFlag); struct LooseChunkHashWorkData @@ -7054,7 +7122,7 @@ namespace { { ZEN_TRACE_CPU("WriteChunks_Wait"); - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() + @@ -7241,7 +7309,7 @@ namespace { WorkerThreadPool& WritePool = GetIOWorkerPool(); - ProgressBar CacheLocalProgressBar(UsePlainProgress); + ProgressBar CacheLocalProgressBar(ProgressMode, "Cache Local Data"); ParallellWork Work(AbortFlag); for (uint32_t LocalPathIndex : FilesToCache) @@ -7272,7 +7340,7 @@ namespace { { ZEN_TRACE_CPU("CacheLocal_Wait"); - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); const uint64_t WorkTotal = FilesToCache.size(); const uint64_t WorkComplete = CachedCount.load(); @@ -7329,7 +7397,7 @@ namespace { WorkerThreadPool& WritePool = GetIOWorkerPool(); - ProgressBar RebuildProgressBar(UsePlainProgress); + ProgressBar RebuildProgressBar(ProgressMode, "Rebuild State"); ParallellWork Work(AbortFlag); OutLocalFolderState.Paths.resize(RemoteContent.Paths.size()); @@ -7566,7 +7634,7 @@ namespace { { ZEN_TRACE_CPU("FinalizeTree_Wait"); - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); const uint64_t WorkTotal = Targets.size() + RemoveLocalPathIndexes.size(); const uint64_t WorkComplete = TargetsComplete.load() + DeletedCount.load(); @@ -8164,7 +8232,7 @@ namespace { { ByteCountToScan += RawSize; } - ProgressBar ProgressBar(false); + ProgressBar ProgressBar(ProgressMode, "Scan Files"); FilteredRate FilteredBytesHashed; FilteredBytesHashed.Start(); ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( @@ -8173,7 +8241,7 @@ namespace { Path, UpdatedContent, ChunkController, - UsePlainProgress ? 5000 : 200, + GetUpdateDelayMS(ProgressMode), [&](bool, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", @@ -8232,7 +8300,7 @@ namespace { { ByteCountToScan += RawSize; } - ProgressBar ProgressBar(false); + ProgressBar ProgressBar(ProgressMode, "Scan Files"); FilteredRate FilteredBytesHashed; FilteredBytesHashed.Start(); LocalContent = ChunkFolderContent( @@ -8241,7 +8309,7 @@ namespace { Path, OutLocalFolderContent, ChunkController, - UsePlainProgress ? 5000 : 200, + GetUpdateDelayMS(ProgressMode), [&](bool, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", @@ -8289,7 +8357,7 @@ namespace { std::move(IsAcceptedFolder), std::move(IsAcceptedFile), GetIOWorkerPool(), - UsePlainProgress ? 5000 : 200, + GetUpdateDelayMS(ProgressMode), [](bool, std::ptrdiff_t) {}, AbortFlag); if (AbortFlag) @@ -8337,10 +8405,27 @@ namespace { { ZEN_TRACE_CPU("DownloadFolder"); + ProgressBar::SetLogOperationName(ProgressMode, "Download Folder"); + + enum TaskSteps : uint32_t + { + CheckState, + CompareState, + Download, + Verify, + Cleanup, + StepCount + }; + + auto EndProgress = + MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); + ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests))); Stopwatch DownloadTimer; + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount); + const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath); CreateDirectories(ZenTempFolder); @@ -8360,6 +8445,8 @@ namespace { std::vector<ChunkBlockDescription> BlockDescriptions; std::vector<IoHash> LooseChunkHashes; + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount); + ChunkedFolderContent RemoteContent = GetRemoteContent(Storage, BuildId, AllBuildParts, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes); @@ -8471,6 +8558,8 @@ namespace { RebuildFolderStateStatistics RebuildFolderStateStats; VerifyFolderStatistics VerifyFolderStats; + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount); + UpdateFolder(SystemRootDir, Storage, BuildId, @@ -8497,6 +8586,8 @@ namespace { { if (!PrimeCacheOnly) { + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount); + VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats); Stopwatch WriteStateTimer; @@ -8564,6 +8655,9 @@ namespace { }); } } + + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); + if (CleanDirectory(ZenTempFolder, {})) { RemoveDirWithRetry(ZenTempFolder); @@ -8572,6 +8666,22 @@ namespace { void DiffFolders(const std::filesystem::path& BasePath, const std::filesystem::path& ComparePath, bool OnlyChunked) { + ZEN_TRACE_CPU("DiffFolders"); + + ProgressBar::SetLogOperationName(ProgressMode, "Diff Folders"); + + enum TaskSteps : uint32_t + { + CheckBase, + CheckCompare, + Diff, + Cleanup, + StepCount + }; + + auto EndProgress = + MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); + ChunkedFolderContent BaseFolderContent; ChunkedFolderContent CompareFolderContent; @@ -8614,6 +8724,8 @@ namespace { return true; }; + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckBase, TaskSteps::StepCount); + GetFolderContentStatistics BaseGetFolderContentStats; ChunkingStatistics BaseChunkingStats; BaseFolderContent = ScanAndChunkFolder(BaseGetFolderContentStats, @@ -8627,6 +8739,8 @@ namespace { return; } + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckCompare, TaskSteps::StepCount); + GetFolderContentStatistics CompareGetFolderContentStats; ChunkingStatistics CompareChunkingStats; CompareFolderContent = ScanAndChunkFolder(CompareGetFolderContentStats, @@ -8642,6 +8756,8 @@ namespace { } } + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Diff, TaskSteps::StepCount); + std::vector<IoHash> AddedHashes; std::vector<IoHash> RemovedHashes; uint64_t RemovedSize = 0; @@ -8728,6 +8844,8 @@ namespace { NewChunkCount, NiceBytes(NewChunkSize), NewPercent); + + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); } } // namespace @@ -8852,7 +8970,18 @@ BuildsCommand::BuildsCommand() }; auto AddOutputOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), "<progress>"); + Ops.add_option("output", + "", + "plain-progress", + "Show progress using plain output", + cxxopts::value(m_PlainProgress), + "<plainprogress>"); + Ops.add_option("output", + "", + "log-progress", + "Write @progress style progress to output", + cxxopts::value(m_LogProgress), + "<logprogress>"); Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>"); }; @@ -9335,8 +9464,19 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; auto ParseOutputOptions = [&]() { - IsVerbose = m_Verbose; - UsePlainProgress = IsVerbose || m_PlainProgress; + IsVerbose = m_Verbose; + if (m_LogProgress) + { + ProgressMode = ProgressBar::Mode::Log; + } + else if (m_Verbose || m_PlainProgress) + { + ProgressMode = ProgressBar::Mode::Plain; + } + else + { + ProgressMode = ProgressBar::Mode::Pretty; + } }; ParseOutputOptions(); |