diff options
| author | Dan Engelbrecht <[email protected]> | 2025-11-07 11:22:45 +0100 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-11-07 11:22:45 +0100 |
| commit | fa2b84a47a305bc3364eb71194b347d981e3c9fe (patch) | |
| tree | e0e3ae5a3c3b22e14dc68bcb08433e6b67ebaa54 /src | |
| parent | remotestore op refactorings (#637) (diff) | |
| download | zen-fa2b84a47a305bc3364eb71194b347d981e3c9fe.tar.xz zen-fa2b84a47a305bc3364eb71194b347d981e3c9fe.zip | |
move progress bar to separate file (#638)
* move progress bar to separate file
Diffstat (limited to 'src')
| -rw-r--r-- | src/zen/cmds/builds_cmd.cpp | 146 | ||||
| -rw-r--r-- | src/zen/cmds/projectstore_cmd.cpp | 2 | ||||
| -rw-r--r-- | src/zen/cmds/wipe_cmd.cpp | 18 | ||||
| -rw-r--r-- | src/zen/progressbar.cpp | 462 | ||||
| -rw-r--r-- | src/zen/progressbar.h | 83 | ||||
| -rw-r--r-- | src/zen/zen.cpp | 360 | ||||
| -rw-r--r-- | src/zen/zen.h | 64 | ||||
| -rw-r--r-- | src/zenremotestore/builds/buildstorageoperations.cpp | 2 | ||||
| -rw-r--r-- | src/zenremotestore/include/zenremotestore/operationlogoutput.h | 14 | ||||
| -rw-r--r-- | src/zenremotestore/operationlogoutput.cpp | 58 |
10 files changed, 591 insertions, 618 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 91eedb756..3a334ed15 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -40,6 +40,8 @@ #include <zenutil/workerpools.h> #include <zenutil/zenserverprocess.h> +#include "../progressbar.h" + #include <signal.h> #include <memory> #include <numeric> @@ -354,22 +356,6 @@ namespace { return *NetworkPool; } - uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode) - { - switch (InMode) - { - case ProgressBar::Mode::Plain: - return 5000; - case ProgressBar::Mode::Pretty: - return 200; - case ProgressBar::Mode::Log: - return 2000; - default: - ZEN_ASSERT(false); - return 0; - } - } - #define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ if (IsVerbose) \ { \ @@ -386,62 +372,6 @@ namespace { ); - class ConsoleOpLogProgressBar : public OperationLogOutput::ProgressBar - { - public: - ConsoleOpLogProgressBar(zen::ProgressBar::Mode InMode, std::string_view InSubTask) : m_Inner(InMode, InSubTask) {} - - virtual void UpdateState(const State& NewState, bool DoLinebreak) - { - zen::ProgressBar::State State = {.Task = NewState.Task, - .Details = NewState.Details, - .TotalCount = NewState.TotalCount, - .RemainingCount = NewState.RemainingCount, - .Status = ConvertStatus(NewState.Status)}; - m_Inner.UpdateState(State, DoLinebreak); - } - virtual void Finish() { m_Inner.Finish(); } - - private: - zen::ProgressBar::State::EStatus ConvertStatus(State::EStatus Status) - { - switch (Status) - { - case State::EStatus::Running: - return zen::ProgressBar::State::EStatus::Running; - case State::EStatus::Aborted: - return zen::ProgressBar::State::EStatus::Aborted; - case State::EStatus::Paused: - return zen::ProgressBar::State::EStatus::Paused; - default: - return (zen::ProgressBar::State::EStatus)Status; - } - } - zen::ProgressBar m_Inner; - }; - - class ConsoleOpLogOutput : public OperationLogOutput - { - public: - ConsoleOpLogOutput(zen::ProgressBar::Mode InMode) : m_Mode(InMode) {} - virtual void EmitLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) - { - logging::EmitConsoleLogMessage(LogLevel, Format, Args); - } - - virtual void SetLogOperationName(std::string_view Name) { zen::ProgressBar::SetLogOperationName(m_Mode, Name); } - virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) - { - zen::ProgressBar::SetLogOperationProgress(m_Mode, StepIndex, StepCount); - } - virtual uint32_t GetProgressUpdateDelayMS() { return GetUpdateDelayMS(m_Mode); } - - virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) { return new ConsoleOpLogProgressBar(m_Mode, InSubTask); } - - private: - zen::ProgressBar::Mode m_Mode; - }; - bool IncludePath(std::span<const std::string> IncludeWildcards, std::span<const std::string> ExcludeWildcards, const std::filesystem::path& Path) @@ -577,9 +507,9 @@ namespace { ProgressBar::SetLogOperationName(ProgressMode, "Validate Part"); - ConsoleOpLogOutput Output(ProgressMode); + std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); - BuildsOperationValidateBuildPart ValidateOp(Output, + BuildsOperationValidateBuildPart ValidateOp(*Output, Storage, AbortFlag, PauseFlag, @@ -620,10 +550,10 @@ namespace { { Stopwatch UploadTimer; - ConsoleOpLogOutput Output(ProgressMode); + std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); BuildsOperationUploadFolder UploadOp( - Output, + *Output, Storage, AbortFlag, PauseFlag, @@ -1377,17 +1307,17 @@ namespace { // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them { - ConsoleLogOutput OperationLogOutput; - bool AttemptFallback = false; - OutBlockDescriptions = GetBlockDescriptions(OperationLogOutput, - *Storage.BuildStorage, - Storage.BuildCacheStorage.get(), - BuildId, - BuildPartId, - BlockRawHashes, - AttemptFallback, - IsQuiet, - IsVerbose); + std::unique_ptr<OperationLogOutput> OperationLogOutput(CreateConsoleLogOutput(ProgressMode)); + bool AttemptFallback = false; + OutBlockDescriptions = GetBlockDescriptions(*OperationLogOutput, + *Storage.BuildStorage, + Storage.BuildCacheStorage.get(), + BuildId, + BuildPartId, + BlockRawHashes, + AttemptFallback, + IsQuiet, + IsVerbose); } CalculateLocalChunkOrders(AbsoluteChunkOrders, @@ -1969,7 +1899,7 @@ namespace { { if (!ChunkController && !IsQuiet) { - ZEN_CONSOLE_WARN("Unspecified chunking algorith, using default"); + ZEN_CONSOLE_INFO("Unspecified chunking algorithm, using default"); ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); } @@ -2033,9 +1963,9 @@ namespace { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount); - ConsoleOpLogOutput Output(ProgressMode); - BuildsOperationUpdateFolder Updater( - Output, + std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + BuildsOperationUpdateFolder Updater( + *Output, Storage, AbortFlag, PauseFlag, @@ -2050,21 +1980,21 @@ namespace { BlockDescriptions, LooseChunkHashes, BuildsOperationUpdateFolder::Options{.IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .AllowFileClone = AllowFileClone, - .UseSparseFiles = UseSparseFiles, - .SystemRootDir = Options.SystemRootDir, - .ZenFolderPath = Options.ZenFolderPath, - .LargeAttachmentSize = LargeAttachmentSize, - .PreferredMultipartChunkSize = PreferredMultipartChunkSize, - .PartialBlockRequestMode = Options.PartialBlockRequestMode, - .WipeTargetFolder = Options.CleanTargetFolder, - .PrimeCacheOnly = Options.PrimeCacheOnly, - .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging, - .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging, - .ValidateCompletedSequences = Options.PostDownloadVerify, - .ExcludeFolders = DefaultExcludeFolders, - .ExcludeExtensions = DefaultExcludeExtensions}); + .IsVerbose = IsVerbose, + .AllowFileClone = AllowFileClone, + .UseSparseFiles = UseSparseFiles, + .SystemRootDir = Options.SystemRootDir, + .ZenFolderPath = Options.ZenFolderPath, + .LargeAttachmentSize = LargeAttachmentSize, + .PreferredMultipartChunkSize = PreferredMultipartChunkSize, + .PartialBlockRequestMode = Options.PartialBlockRequestMode, + .WipeTargetFolder = Options.CleanTargetFolder, + .PrimeCacheOnly = Options.PrimeCacheOnly, + .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging, + .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging, + .ValidateCompletedSequences = Options.PostDownloadVerify, + .ExcludeFolders = DefaultExcludeFolders, + .ExcludeExtensions = DefaultExcludeExtensions}); { ProgressBar::PushLogOperation(ProgressMode, "Download"); auto _ = MakeGuard([]() { ProgressBar::PopLogOperation(ProgressMode); }); @@ -3959,9 +3889,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache"); - ConsoleOpLogOutput Output(ProgressMode); + std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); - BuildsOperationPrimeCache PrimeOp(Output, + BuildsOperationPrimeCache PrimeOp(*Output, Storage, AbortFlag, PauseFlag, diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index af6f9aa7c..6df317823 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -23,6 +23,8 @@ #include <zenremotestore/builds/jupiterbuildstorage.h> #include <zenremotestore/jupiter/jupiterhost.h> +#include "../progressbar.h" + ZEN_THIRD_PARTY_INCLUDES_START #include <json11.hpp> ZEN_THIRD_PARTY_INCLUDES_END diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp index a3d40c142..adf0e61f0 100644 --- a/src/zen/cmds/wipe_cmd.cpp +++ b/src/zen/cmds/wipe_cmd.cpp @@ -11,6 +11,8 @@ #include <zencore/trace.h> #include <zenutil/workerpools.h> +#include "../progressbar.h" + #include <signal.h> #include <iostream> @@ -40,22 +42,6 @@ namespace { const bool SingleThreaded = false; bool BoostWorkerThreads = true; - uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode) - { - switch (InMode) - { - case ProgressBar::Mode::Plain: - return 5000; - case ProgressBar::Mode::Pretty: - return 200; - case ProgressBar::Mode::Log: - return 2000; - default: - ZEN_ASSERT(false); - return 0; - } - } - WorkerThreadPool& GetIOWorkerPool() { return SingleThreaded ? GetSyncWorkerPool() diff --git a/src/zen/progressbar.cpp b/src/zen/progressbar.cpp new file mode 100644 index 000000000..83606df67 --- /dev/null +++ b/src/zen/progressbar.cpp @@ -0,0 +1,462 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +// Zen command line client utility +// + +#include "progressbar.h" + +#include <zencore/logging.h> +#include <zencore/windows.h> +#include <zenremotestore/operationlogoutput.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <gsl/gsl-lite.hpp> +ZEN_THIRD_PARTY_INCLUDES_END + +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC +# include <sys/ioctl.h> +# include <unistd.h> +#endif + +////////////////////////////////////////////////////////////////////////// + +namespace zen { + +#if ZEN_PLATFORM_WINDOWS +static HANDLE +GetConsoleHandle() +{ + static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); + return hStdOut; +} +#endif + +static bool +CheckStdoutTty() +{ +#if ZEN_PLATFORM_WINDOWS + HANDLE hStdOut = GetConsoleHandle(); + DWORD dwMode = 0; + static bool IsConsole = ::GetConsoleMode(hStdOut, &dwMode); + return IsConsole; +#else + return isatty(fileno(stdout)); +#endif +} + +static bool +IsStdoutTty() +{ + static bool StdoutIsTty = CheckStdoutTty(); + return StdoutIsTty; +} + +static void +OutputToConsoleRaw(const char* String, size_t Length) +{ +#if ZEN_PLATFORM_WINDOWS + HANDLE hStdOut = GetConsoleHandle(); +#endif + +#if ZEN_PLATFORM_WINDOWS + if (IsStdoutTty()) + { + WriteConsoleA(hStdOut, String, (DWORD)Length, 0, 0); + } + else + { + ::WriteFile(hStdOut, (LPCVOID)String, (DWORD)Length, 0, 0); + } +#else + fwrite(String, 1, Length, stdout); +#endif +} + +static void +OutputToConsoleRaw(const std::string& String) +{ + OutputToConsoleRaw(String.c_str(), String.length()); +} + +static void +OutputToConsoleRaw(const StringBuilderBase& SB) +{ + OutputToConsoleRaw(SB.c_str(), SB.Size()); +} + +uint32_t +GetConsoleColumns(uint32_t Default) +{ +#if ZEN_PLATFORM_WINDOWS + HANDLE hStdOut = GetConsoleHandle(); + CONSOLE_SCREEN_BUFFER_INFO csbi; + if (GetConsoleScreenBufferInfo(hStdOut, &csbi) == TRUE) + { + return (uint32_t)(csbi.srWindow.Right - csbi.srWindow.Left + 1); + } +#else + struct winsize w; + if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &w) == 0) + { + return (uint32_t)w.ws_col; + } +#endif + return Default; +} + +uint32_t +GetUpdateDelayMS(ProgressBar::Mode InMode) +{ + switch (InMode) + { + case ProgressBar::Mode::Plain: + return 5000; + case ProgressBar::Mode::Pretty: + return 200; + case ProgressBar::Mode::Log: + return 2000; + default: + ZEN_ASSERT(false); + return 0; + } +} + +void +ProgressBar::SetLogOperationName(Mode InMode, std::string_view Name) +{ + ZEN_ASSERT(Name.find('\"') == std::string_view::npos); + if (InMode == Mode::Log) + { + std::string String = fmt::format("@progress \"{}\"\n", Name); + OutputToConsoleRaw(String); + } +} + +void +ProgressBar::SetLogOperationProgress(Mode InMode, uint32_t StepIndex, uint32_t StepCount) +{ + if (InMode == Mode::Log) + { + const size_t PercentDone = StepCount > 0u ? gsl::narrow<uint8_t>((100 * StepIndex) / StepCount) : 0u; + + std::string String = fmt::format("@progress {}%\n", PercentDone); + OutputToConsoleRaw(String); + } +} + +void +ProgressBar::PushLogOperation(Mode InMode, std::string_view Name) +{ + if (InMode == Mode::Log) + { + std::string String = fmt::format("@progress push \"{}\"\n", Name); + OutputToConsoleRaw(String); + } +} + +void +ProgressBar::PopLogOperation(Mode InMode) +{ + if (InMode == Mode::Log) + { + const std::string String("@progress pop\n"); + OutputToConsoleRaw(String); + } +} + +ProgressBar::ProgressBar(Mode InMode, std::string_view InSubTask) +: m_Mode((!IsStdoutTty() && InMode == Mode::Pretty) ? Mode::Plain : InMode) +, m_LastUpdateMS((uint64_t)-1) +, m_PausedMS(0) +, m_SubTask(InSubTask) +{ + ZEN_ASSERT(InSubTask.find('\"') == std::string_view::npos); + if (!m_SubTask.empty()) + { + PushLogOperation(InMode, m_SubTask); + } +} + +ProgressBar::~ProgressBar() +{ + try + { + ForceLinebreak(); + if (!m_SubTask.empty()) + { + PopLogOperation(m_Mode); + } + } + catch (const std::exception& Ex) + { + ZEN_ERROR("ProgressBar::~ProgressBar() failed with {}", Ex.what()); + } +} + +void +ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) +{ + ZEN_ASSERT(NewState.TotalCount >= NewState.RemainingCount); + ZEN_ASSERT(NewState.Task.find('\"') == std::string::npos); + if (DoLinebreak == false && m_State == NewState) + { + return; + } + + uint64_t ElapsedTimeMS = m_SW.GetElapsedTimeMs(); + if (m_LastUpdateMS != (uint64_t)-1) + { + if (!DoLinebreak && (NewState.Status == m_State.Status) && (NewState.Task == m_State.Task) && + ((m_LastUpdateMS + 200) > ElapsedTimeMS)) + { + return; + } + if (m_State.Status == State::EStatus::Paused) + { + uint64_t ElapsedSinceLast = ElapsedTimeMS - m_LastUpdateMS; + m_PausedMS += ElapsedSinceLast; + } + } + + m_LastUpdateMS = ElapsedTimeMS; + + std::string Task = NewState.Task; + switch (NewState.Status) + { + case State::EStatus::Aborted: + Task = "Aborting"; + break; + case State::EStatus::Paused: + Task = "Paused"; + break; + default: + break; + } + if (NewState.Task.length() > Task.length()) + { + Task += std::string(NewState.Task.length() - Task.length(), ' '); + } + + const size_t PercentDone = + NewState.TotalCount > 0u ? gsl::narrow<uint8_t>((100 * (NewState.TotalCount - NewState.RemainingCount)) / NewState.TotalCount) : 0u; + + if (m_Mode == Mode::Plain) + { + const std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; + const std::string Output = fmt::format("{} {}% ({}){}\n", Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), Details); + OutputToConsoleRaw(Output); + } + else if (m_Mode == Mode::Pretty) + { + size_t ProgressBarSize = 20; + + size_t ProgressBarCount = (ProgressBarSize * PercentDone) / 100; + uint64_t Completed = NewState.TotalCount - NewState.RemainingCount; + uint64_t ETAElapsedMS = ElapsedTimeMS -= m_PausedMS; + uint64_t ETAMS = + (NewState.Status == State::EStatus::Running) && (PercentDone > 5) ? (ETAElapsedMS * NewState.RemainingCount) / Completed : 0; + + uint32_t ConsoleColumns = GetConsoleColumns(1024); + + const std::string PercentString = fmt::format("{:#3}%", PercentDone); + + const std::string ProgressBarString = + fmt::format(": |{}{}|", std::string(ProgressBarCount, '#'), std::string(ProgressBarSize - ProgressBarCount, ' ')); + + const std::string ElapsedString = fmt::format(": {}", NiceTimeSpanMs(ElapsedTimeMS)); + + const std::string ETAString = (ETAMS > 0) ? fmt::format(" ETA {}", NiceTimeSpanMs(ETAMS)) : ""; + + const std::string DetailsString = (!NewState.Details.empty()) ? fmt::format(". {}", NewState.Details) : ""; + + ExtendableStringBuilder<256> OutputBuilder; + + OutputBuilder << "\r" << Task << " " << PercentString; + if (OutputBuilder.Size() + 1 < ConsoleColumns) + { + size_t RemainingSpace = ConsoleColumns - (OutputBuilder.Size() + 1); + bool ElapsedFits = RemainingSpace >= ElapsedString.length(); + RemainingSpace -= ElapsedString.length(); + bool ETAFits = ElapsedFits && RemainingSpace >= ETAString.length(); + RemainingSpace -= ETAString.length(); + bool DetailsFits = ETAFits && RemainingSpace >= DetailsString.length(); + RemainingSpace -= DetailsString.length(); + bool ProgressBarFits = DetailsFits && RemainingSpace >= ProgressBarString.length(); + RemainingSpace -= ProgressBarString.length(); + + if (ProgressBarFits) + { + OutputBuilder << ProgressBarString; + } + if (ElapsedFits) + { + OutputBuilder << ElapsedString; + } + if (ETAFits) + { + OutputBuilder << ETAString; + } + if (DetailsFits) + { + OutputBuilder << DetailsString; + } + } + + std::string_view Output = OutputBuilder.ToView(); + std::string::size_type EraseLength = m_LastOutputLength > Output.length() ? (m_LastOutputLength - Output.length()) : 0; + + ExtendableStringBuilder<256> LineToPrint; + + if (Output.length() + EraseLength >= ConsoleColumns) + { + if (m_LastOutputLength > 0) + { + LineToPrint << "\n"; + } + LineToPrint << Output.substr(1); + DoLinebreak = true; + } + else + { + LineToPrint << Output << std::string(EraseLength, ' '); + } + + if (DoLinebreak) + { + LineToPrint << "\n"; + } + + OutputToConsoleRaw(LineToPrint); + + m_LastOutputLength = DoLinebreak ? 0 : Output.length(); + m_State = NewState; + } + else if (m_Mode == Mode::Log) + { + if (m_State.Task != NewState.Task || + m_State.Details != NewState.Details) // TODO: Should we output just because details change? Will this spam the log collector? + { + std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; + for (std::string::value_type& Char : Details) + { + if (Char == '"') + { + Char = '\''; + } + } + const std::string Message = fmt::format("@progress \"{} ({}){}\"\n", NewState.Task, NiceTimeSpanMs(ElapsedTimeMS), Details); + OutputToConsoleRaw(Message); + } + + const size_t OldPercentDone = + m_State.TotalCount > 0u ? gsl::narrow<uint8_t>((100 * (m_State.TotalCount - m_State.RemainingCount)) / m_State.TotalCount) : 0u; + + if (OldPercentDone != PercentDone) + { + const std::string Progress = fmt::format("@progress {}%\n", PercentDone); + OutputToConsoleRaw(Progress); + } + m_State = NewState; + } +} + +void +ProgressBar::ForceLinebreak() +{ + if (m_LastOutputLength > 0) + { + State NewState = m_State; + UpdateState(NewState, /*DoLinebreak*/ true); + } +} + +void +ProgressBar::Finish() +{ + if (m_LastOutputLength > 0 || m_State.RemainingCount > 0) + { + State NewState = m_State; + NewState.RemainingCount = 0; + NewState.Details = ""; + UpdateState(NewState, /*DoLinebreak*/ true); + } + m_State = State{}; + m_LastOutputLength = 0; + m_SW.Reset(); +} + +bool +ProgressBar::IsSameTask(std::string_view Task) const +{ + return Task == m_State.Task; +} + +bool +ProgressBar::HasActiveTask() const +{ + return !m_State.Task.empty(); +} + +class ConsoleOpLogProgressBar : public OperationLogOutput::ProgressBar +{ +public: + ConsoleOpLogProgressBar(zen::ProgressBar::Mode InMode, std::string_view InSubTask) : m_Inner(InMode, InSubTask) {} + + virtual void UpdateState(const State& NewState, bool DoLinebreak) + { + zen::ProgressBar::State State = {.Task = NewState.Task, + .Details = NewState.Details, + .TotalCount = NewState.TotalCount, + .RemainingCount = NewState.RemainingCount, + .Status = ConvertStatus(NewState.Status)}; + m_Inner.UpdateState(State, DoLinebreak); + } + virtual void Finish() { m_Inner.Finish(); } + +private: + zen::ProgressBar::State::EStatus ConvertStatus(State::EStatus Status) + { + switch (Status) + { + case State::EStatus::Running: + return zen::ProgressBar::State::EStatus::Running; + case State::EStatus::Aborted: + return zen::ProgressBar::State::EStatus::Aborted; + case State::EStatus::Paused: + return zen::ProgressBar::State::EStatus::Paused; + default: + return (zen::ProgressBar::State::EStatus)Status; + } + } + zen::ProgressBar m_Inner; +}; + +class ConsoleOpLogOutput : public OperationLogOutput +{ +public: + ConsoleOpLogOutput(zen::ProgressBar::Mode InMode) : m_Mode(InMode) {} + virtual void EmitLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) + { + logging::EmitConsoleLogMessage(LogLevel, Format, Args); + } + + virtual void SetLogOperationName(std::string_view Name) { zen::ProgressBar::SetLogOperationName(m_Mode, Name); } + virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) + { + zen::ProgressBar::SetLogOperationProgress(m_Mode, StepIndex, StepCount); + } + virtual uint32_t GetProgressUpdateDelayMS() { return GetUpdateDelayMS(m_Mode); } + + virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) { return new ConsoleOpLogProgressBar(m_Mode, InSubTask); } + +private: + zen::ProgressBar::Mode m_Mode; +}; + +OperationLogOutput* +CreateConsoleLogOutput(ProgressBar::Mode InMode) +{ + return new ConsoleOpLogOutput(InMode); +} + +} // namespace zen diff --git a/src/zen/progressbar.h b/src/zen/progressbar.h new file mode 100644 index 000000000..bbdb008d4 --- /dev/null +++ b/src/zen/progressbar.h @@ -0,0 +1,83 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/timer.h> +#include <zencore/zencore.h> + +#include <string> + +namespace zen { + +class OperationLogOutput; + +class ProgressBar +{ +public: + struct State + { + bool operator==(const State&) const = default; + std::string Task; + std::string Details; + uint64_t TotalCount = 0; + uint64_t RemainingCount = 0; + enum class EStatus + { + Running, + Aborted, + Paused + }; + EStatus Status = EStatus::Running; + + static EStatus CalculateStatus(bool IsAborted, bool IsPaused) + { + if (IsAborted) + { + return EStatus::Aborted; + } + if (IsPaused) + { + return EStatus::Paused; + } + return EStatus::Running; + } + }; + + enum class Mode + { + Plain, + Pretty, + Log, + Quiet + }; + + static void SetLogOperationName(Mode InMode, std::string_view Name); + static void SetLogOperationProgress(Mode InMode, uint32_t StepIndex, uint32_t StepCount); + static void PushLogOperation(Mode InMode, std::string_view Name); + static void PopLogOperation(Mode InMode); + + explicit ProgressBar(Mode InMode, std::string_view InSubTask); + ~ProgressBar(); + + void UpdateState(const State& NewState, bool DoLinebreak); + void ForceLinebreak(); + void Finish(); + bool IsSameTask(std::string_view Task) const; + bool HasActiveTask() const; + +private: + const Mode m_Mode; + Stopwatch m_SW; + uint64_t m_LastUpdateMS; + uint64_t m_PausedMS; + State m_State; + const std::string m_SubTask; + size_t m_LastOutputLength = 0; +}; + +uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode); +uint32_t GetConsoleColumns(uint32_t Default); + +OperationLogOutput* CreateConsoleLogOutput(ProgressBar::Mode InMode); + +} // namespace zen diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 2661f2173..c47dc642f 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -50,6 +50,8 @@ #include <zencore/memory/memorytrace.h> #include <zencore/memory/newdelete.h> +#include "progressbar.h" + #if ZEN_WITH_TESTS # define ZEN_TEST_WITH_RUNNER 1 # include <zencore/testing.h> @@ -70,88 +72,6 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { -#if ZEN_PLATFORM_WINDOWS -static HANDLE -GetConsoleHandle() -{ - static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); - return hStdOut; -} -#endif - -static bool -CheckStdoutTty() -{ -#if ZEN_PLATFORM_WINDOWS - HANDLE hStdOut = GetConsoleHandle(); - DWORD dwMode = 0; - static bool IsConsole = ::GetConsoleMode(hStdOut, &dwMode); - return IsConsole; -#else - return isatty(fileno(stdout)); -#endif -} - -static bool -IsStdoutTty() -{ - static bool StdoutIsTty = CheckStdoutTty(); - return StdoutIsTty; -} - -static void -OutputToConsoleRaw(const char* String, size_t Length) -{ -#if ZEN_PLATFORM_WINDOWS - HANDLE hStdOut = GetConsoleHandle(); -#endif - -#if ZEN_PLATFORM_WINDOWS - if (IsStdoutTty()) - { - WriteConsoleA(hStdOut, String, (DWORD)Length, 0, 0); - } - else - { - ::WriteFile(hStdOut, (LPCVOID)String, (DWORD)Length, 0, 0); - } -#else - fwrite(String, 1, Length, stdout); -#endif -} - -static void -OutputToConsoleRaw(const std::string& String) -{ - OutputToConsoleRaw(String.c_str(), String.length()); -} - -static void -OutputToConsoleRaw(const StringBuilderBase& SB) -{ - OutputToConsoleRaw(SB.c_str(), SB.Size()); -} - -static uint32_t -GetConsoleColumns(uint32_t Default) -{ -#if ZEN_PLATFORM_WINDOWS - HANDLE hStdOut = GetConsoleHandle(); - CONSOLE_SCREEN_BUFFER_INFO csbi; - if (GetConsoleScreenBufferInfo(hStdOut, &csbi) == TRUE) - { - return (uint32_t)(csbi.srWindow.Right - csbi.srWindow.Left + 1); - } -#else - struct winsize w; - if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &w) == 0) - { - return (uint32_t)w.ws_col; - } -#endif - return Default; -} - enum class ReturnCode : std::int8_t { kSuccess = 0, @@ -367,282 +287,6 @@ ZenCmdBase::LogExecutableVersionAndPid() ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); } -void -ProgressBar::SetLogOperationName(Mode InMode, std::string_view Name) -{ - ZEN_ASSERT(Name.find('\"') == std::string_view::npos); - if (InMode == Mode::Log) - { - std::string String = fmt::format("@progress \"{}\"\n", Name); - OutputToConsoleRaw(String); - } -} - -void -ProgressBar::SetLogOperationProgress(Mode InMode, uint32_t StepIndex, uint32_t StepCount) -{ - if (InMode == Mode::Log) - { - const size_t PercentDone = StepCount > 0u ? gsl::narrow<uint8_t>((100 * StepIndex) / StepCount) : 0u; - - std::string String = fmt::format("@progress {}%\n", PercentDone); - OutputToConsoleRaw(String); - } -} - -void -ProgressBar::PushLogOperation(Mode InMode, std::string_view Name) -{ - if (InMode == Mode::Log) - { - std::string String = fmt::format("@progress push \"{}\"\n", Name); - OutputToConsoleRaw(String); - } -} - -void -ProgressBar::PopLogOperation(Mode InMode) -{ - if (InMode == Mode::Log) - { - const std::string String("@progress pop\n"); - OutputToConsoleRaw(String); - } -} - -ProgressBar::ProgressBar(Mode InMode, std::string_view InSubTask) -: m_Mode((!IsStdoutTty() && InMode == Mode::Pretty) ? Mode::Plain : InMode) -, m_LastUpdateMS((uint64_t)-1) -, m_PausedMS(0) -, m_SubTask(InSubTask) -{ - ZEN_ASSERT(InSubTask.find('\"') == std::string_view::npos); - if (!m_SubTask.empty()) - { - PushLogOperation(InMode, m_SubTask); - } -} - -ProgressBar::~ProgressBar() -{ - try - { - ForceLinebreak(); - if (!m_SubTask.empty()) - { - PopLogOperation(m_Mode); - } - } - catch (const std::exception& Ex) - { - ZEN_ERROR("ProgressBar::~ProgressBar() failed with {}", Ex.what()); - } -} - -void -ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) -{ - ZEN_ASSERT(NewState.TotalCount >= NewState.RemainingCount); - ZEN_ASSERT(NewState.Task.find('\"') == std::string::npos); - if (DoLinebreak == false && m_State == NewState) - { - return; - } - - uint64_t ElapsedTimeMS = m_SW.GetElapsedTimeMs(); - if (m_LastUpdateMS != (uint64_t)-1) - { - if (!DoLinebreak && (NewState.Status == m_State.Status) && (NewState.Task == m_State.Task) && - ((m_LastUpdateMS + 200) > ElapsedTimeMS)) - { - return; - } - if (m_State.Status == State::EStatus::Paused) - { - uint64_t ElapsedSinceLast = ElapsedTimeMS - m_LastUpdateMS; - m_PausedMS += ElapsedSinceLast; - } - } - - m_LastUpdateMS = ElapsedTimeMS; - - std::string Task = NewState.Task; - switch (NewState.Status) - { - case State::EStatus::Aborted: - Task = "Aborting"; - break; - case State::EStatus::Paused: - Task = "Paused"; - break; - default: - break; - } - if (NewState.Task.length() > Task.length()) - { - Task += std::string(NewState.Task.length() - Task.length(), ' '); - } - - const size_t PercentDone = - NewState.TotalCount > 0u ? gsl::narrow<uint8_t>((100 * (NewState.TotalCount - NewState.RemainingCount)) / NewState.TotalCount) : 0u; - - if (m_Mode == Mode::Plain) - { - const std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; - const std::string Output = fmt::format("{} {}% ({}){}\n", Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), Details); - OutputToConsoleRaw(Output); - } - else if (m_Mode == Mode::Pretty) - { - size_t ProgressBarSize = 20; - - size_t ProgressBarCount = (ProgressBarSize * PercentDone) / 100; - uint64_t Completed = NewState.TotalCount - NewState.RemainingCount; - uint64_t ETAElapsedMS = ElapsedTimeMS -= m_PausedMS; - uint64_t ETAMS = - (NewState.Status == State::EStatus::Running) && (PercentDone > 5) ? (ETAElapsedMS * NewState.RemainingCount) / Completed : 0; - - uint32_t ConsoleColumns = GetConsoleColumns(1024); - - const std::string PercentString = fmt::format("{:#3}%", PercentDone); - - const std::string ProgressBarString = - fmt::format(": |{}{}|", std::string(ProgressBarCount, '#'), std::string(ProgressBarSize - ProgressBarCount, ' ')); - - const std::string ElapsedString = fmt::format(": {}", NiceTimeSpanMs(ElapsedTimeMS)); - - const std::string ETAString = (ETAMS > 0) ? fmt::format(" ETA {}", NiceTimeSpanMs(ETAMS)) : ""; - - const std::string DetailsString = (!NewState.Details.empty()) ? fmt::format(". {}", NewState.Details) : ""; - - ExtendableStringBuilder<256> OutputBuilder; - - OutputBuilder << "\r" << Task << " " << PercentString; - if (OutputBuilder.Size() + 1 < ConsoleColumns) - { - size_t RemainingSpace = ConsoleColumns - (OutputBuilder.Size() + 1); - bool ElapsedFits = RemainingSpace >= ElapsedString.length(); - RemainingSpace -= ElapsedString.length(); - bool ETAFits = ElapsedFits && RemainingSpace >= ETAString.length(); - RemainingSpace -= ETAString.length(); - bool DetailsFits = ETAFits && RemainingSpace >= DetailsString.length(); - RemainingSpace -= DetailsString.length(); - bool ProgressBarFits = DetailsFits && RemainingSpace >= ProgressBarString.length(); - RemainingSpace -= ProgressBarString.length(); - - if (ProgressBarFits) - { - OutputBuilder << ProgressBarString; - } - if (ElapsedFits) - { - OutputBuilder << ElapsedString; - } - if (ETAFits) - { - OutputBuilder << ETAString; - } - if (DetailsFits) - { - OutputBuilder << DetailsString; - } - } - - std::string_view Output = OutputBuilder.ToView(); - std::string::size_type EraseLength = m_LastOutputLength > Output.length() ? (m_LastOutputLength - Output.length()) : 0; - - ExtendableStringBuilder<256> LineToPrint; - - if (Output.length() + EraseLength >= ConsoleColumns) - { - if (m_LastOutputLength > 0) - { - LineToPrint << "\n"; - } - LineToPrint << Output.substr(1); - DoLinebreak = true; - } - else - { - LineToPrint << Output << std::string(EraseLength, ' '); - } - - if (DoLinebreak) - { - LineToPrint << "\n"; - } - - OutputToConsoleRaw(LineToPrint); - - m_LastOutputLength = DoLinebreak ? 0 : Output.length(); - m_State = NewState; - } - else if (m_Mode == Mode::Log) - { - if (m_State.Task != NewState.Task || - m_State.Details != NewState.Details) // TODO: Should we output just because details change? Will this spam the log collector? - { - std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; - for (std::string::value_type& Char : Details) - { - if (Char == '"') - { - Char = '\''; - } - } - const std::string Message = fmt::format("@progress \"{} ({}){}\"\n", NewState.Task, NiceTimeSpanMs(ElapsedTimeMS), Details); - OutputToConsoleRaw(Message); - } - - const size_t OldPercentDone = - m_State.TotalCount > 0u ? gsl::narrow<uint8_t>((100 * (m_State.TotalCount - m_State.RemainingCount)) / m_State.TotalCount) : 0u; - - if (OldPercentDone != PercentDone) - { - const std::string Progress = fmt::format("@progress {}%\n", PercentDone); - OutputToConsoleRaw(Progress); - } - m_State = NewState; - } -} - -void -ProgressBar::ForceLinebreak() -{ - if (m_LastOutputLength > 0) - { - State NewState = m_State; - UpdateState(NewState, /*DoLinebreak*/ true); - } -} - -void -ProgressBar::Finish() -{ - if (m_LastOutputLength > 0 || m_State.RemainingCount > 0) - { - State NewState = m_State; - NewState.RemainingCount = 0; - NewState.Details = ""; - UpdateState(NewState, /*DoLinebreak*/ true); - } - m_State = State{}; - m_LastOutputLength = 0; - m_SW.Reset(); -} - -bool -ProgressBar::IsSameTask(std::string_view Task) const -{ - return Task == m_State.Task; -} - -bool -ProgressBar::HasActiveTask() const -{ - return !m_State.Task.empty(); -} - } // namespace zen ////////////////////////////////////////////////////////////////////////// diff --git a/src/zen/zen.h b/src/zen/zen.h index 80df8c17f..05d1e4ec8 100644 --- a/src/zen/zen.h +++ b/src/zen/zen.h @@ -76,68 +76,4 @@ class CacheStoreCommand : public ZenCmdBase virtual ZenCmdCategory& CommandCategory() const override { return g_CacheStoreCategory; } }; -class ProgressBar -{ -public: - struct State - { - bool operator==(const State&) const = default; - std::string Task; - std::string Details; - uint64_t TotalCount = 0; - uint64_t RemainingCount = 0; - enum class EStatus - { - Running, - Aborted, - Paused - }; - EStatus Status = EStatus::Running; - - static EStatus CalculateStatus(bool IsAborted, bool IsPaused) - { - if (IsAborted) - { - return EStatus::Aborted; - } - if (IsPaused) - { - return EStatus::Paused; - } - return EStatus::Running; - } - }; - - enum class Mode - { - Plain, - Pretty, - Log, - Quiet - }; - - static void SetLogOperationName(Mode InMode, std::string_view Name); - static void SetLogOperationProgress(Mode InMode, uint32_t StepIndex, uint32_t StepCount); - static void PushLogOperation(Mode InMode, std::string_view Name); - static void PopLogOperation(Mode InMode); - - explicit ProgressBar(Mode InMode, std::string_view InSubTask); - ~ProgressBar(); - - void UpdateState(const State& NewState, bool DoLinebreak); - void ForceLinebreak(); - void Finish(); - bool IsSameTask(std::string_view Task) const; - bool HasActiveTask() const; - -private: - const Mode m_Mode; - Stopwatch m_SW; - uint64_t m_LastUpdateMS; - uint64_t m_PausedMS; - State m_State; - const std::string m_SubTask; - size_t m_LastOutputLength = 0; -}; - } // namespace zen diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp index 747aa7631..d8ae9c5a7 100644 --- a/src/zenremotestore/builds/buildstorageoperations.cpp +++ b/src/zenremotestore/builds/buildstorageoperations.cpp @@ -1336,7 +1336,7 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) if (!m_Options.IsQuiet) { ZEN_OPERATION_LOG_INFO(m_LogOutput, - "Analisys of partial block requests saves download of {} out of {} ({:.1f}%) using {} extra " + "Analysis of partial block requests saves download of {} out of {} ({:.1f}%) using {} extra " "requests. Completed in {}", NiceBytes(TotalSavedBlocksSize), NiceBytes(NonPartialTotalBlockBytes), diff --git a/src/zenremotestore/include/zenremotestore/operationlogoutput.h b/src/zenremotestore/include/zenremotestore/operationlogoutput.h index b84d77703..49649fc76 100644 --- a/src/zenremotestore/include/zenremotestore/operationlogoutput.h +++ b/src/zenremotestore/include/zenremotestore/operationlogoutput.h @@ -9,6 +9,7 @@ namespace zen { class OperationLogOutput { public: + virtual ~OperationLogOutput() {} virtual void EmitLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) = 0; virtual void SetLogOperationName(std::string_view Name) = 0; @@ -71,17 +72,4 @@ public: #define ZEN_OPERATION_LOG_WARN(OutputTarget, fmtstr, ...) \ ZEN_OPERATION_LOG((OutputTarget), zen::logging::level::Warn, fmtstr, ##__VA_ARGS__) -class ConsoleLogOutput : public OperationLogOutput -{ -public: - ConsoleLogOutput(); - virtual void EmitLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) override; - - virtual void SetLogOperationName(std::string_view Name) override; - virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) override; - virtual uint32_t GetProgressUpdateDelayMS() override; - - virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) override; -}; - } // namespace zen diff --git a/src/zenremotestore/operationlogoutput.cpp b/src/zenremotestore/operationlogoutput.cpp index 967b8e34e..8d8a6dc78 100644 --- a/src/zenremotestore/operationlogoutput.cpp +++ b/src/zenremotestore/operationlogoutput.cpp @@ -6,62 +6,4 @@ namespace zen { -using namespace std::literals; - -ConsoleLogOutput::ConsoleLogOutput() -{ -} - -void -ConsoleLogOutput::EmitLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) -{ - logging::EmitConsoleLogMessage(LogLevel, Format, Args); -} - -void -ConsoleLogOutput::SetLogOperationName(std::string_view Name) -{ - ZEN_OPERATION_LOG_INFO(*this, "{}", Name); -} -void -ConsoleLogOutput::SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) -{ - ZEN_OPERATION_LOG_INFO(*this, "{}/{}", StepIndex, StepCount); -} -uint32_t -ConsoleLogOutput::GetProgressUpdateDelayMS() -{ - return 2000; -} - -class ConsoleLogOutputProgress : public OperationLogOutput::ProgressBar -{ -public: - ConsoleLogOutputProgress(OperationLogOutput& OperationLogOutput) : m_LogOutput(OperationLogOutput) {} - virtual void UpdateState(const State& NewState, bool DoLinebreak) - { - ZEN_UNUSED(DoLinebreak); - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "{} {}/{} {}", - NewState.Task, - NewState.TotalCount - NewState.RemainingCount, - NewState.TotalCount, - NewState.Details); - m_CurrentState = NewState; - } - virtual void Finish() - { - ZEN_OPERATION_LOG_INFO(m_LogOutput, "{} {}/{}", m_CurrentState.Task, m_CurrentState.TotalCount, m_CurrentState.TotalCount); - } - State m_CurrentState; - OperationLogOutput& m_LogOutput; -}; - -OperationLogOutput::ProgressBar* -ConsoleLogOutput::CreateProgressBar(std::string_view InSubTask) -{ - ZEN_UNUSED(InSubTask); - return new ConsoleLogOutputProgress(*this); -} - } // namespace zen |