diff options
Diffstat (limited to 'src/zenremotestore')
6 files changed, 145 insertions, 28 deletions
diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp index 5bfef842e..b8dd18bb5 100644 --- a/src/zenremotestore/builds/buildstorageoperations.cpp +++ b/src/zenremotestore/builds/buildstorageoperations.cpp @@ -2833,7 +2833,7 @@ BuildsOperationUpdateFolder::FindScavengeContent(const ScavengeSource& Source, } catch (const std::exception& Ex) { - ZEN_CONSOLE_DEBUG("Skipping invalid build state at '{}', reason: {}", Source.StateFilePath, Ex.what()); + ZEN_OPERATION_LOG_DEBUG(m_LogOutput, "Skipping invalid build state at '{}', reason: {}", Source.StateFilePath, Ex.what()); return false; } @@ -5374,7 +5374,7 @@ BuildsOperationUploadFolder::Execute() if (!UnknownChunks.empty()) { - ZEN_CONSOLE_WARN("{}", BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ true)); + ZEN_OPERATION_LOG_WARN(m_LogOutput, "{}", BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ true)); } uint32_t FinalizeBuildPartRetryCount = 5; @@ -5411,7 +5411,9 @@ BuildsOperationUploadFolder::Execute() else { UnknownChunks = RetryUnknownChunks; - ZEN_CONSOLE_WARN("{}", BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ FinalizeBuildPartRetryCount != 0)); + ZEN_OPERATION_LOG_WARN(m_LogOutput, + "{}", + BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ FinalizeBuildPartRetryCount != 0)); } } @@ -6896,11 +6898,12 @@ BuildsOperationValidateBuildPart::Execute() auto _ = MakeGuard([&]() { if (!m_Options.IsQuiet) { - ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}", - m_BuildId, - m_BuildPartId, - m_BuildPartName, - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Validated build part {}/{} ('{}') in {}", + m_BuildId, + m_BuildPartId, + m_BuildPartName, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } }); @@ -6928,7 +6931,11 @@ BuildsOperationValidateBuildPart::Execute() m_ValidateStats.BuildPartSize = BuildPart.GetSize(); if (!m_Options.IsQuiet) { - ZEN_CONSOLE("Validating build part {}/{} ({})", m_BuildId, m_BuildPartId, NiceBytes(BuildPart.GetSize())); + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Validating build part {}/{} ({})", + m_BuildId, + m_BuildPartId, + NiceBytes(BuildPart.GetSize())); } std::vector<IoHash> ChunkAttachments; if (const CbObjectView ChunkAttachmentsView = BuildPart["chunkAttachments"sv].AsObjectView()) diff --git a/src/zenremotestore/builds/buildstorageutil.cpp b/src/zenremotestore/builds/buildstorageutil.cpp index 3af323628..15ece2edd 100644 --- a/src/zenremotestore/builds/buildstorageutil.cpp +++ b/src/zenremotestore/builds/buildstorageutil.cpp @@ -32,7 +32,8 @@ namespace { } // namespace BuildStorageResolveResult -ResolveBuildStorage(const HttpClientSettings& ClientSettings, +ResolveBuildStorage(OperationLogOutput& Output, + const HttpClientSettings& ClientSettings, std::string_view Host, std::string_view OverrideHost, std::string_view ZenCacheHost, @@ -75,9 +76,10 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings, { if (Verbose) { - ZEN_CONSOLE("Querying servers at '{}/api/v1/status/servers'\n Connection settings:{}", - DiscoveryHost, - ConnectionSettingsToString(ClientSettings)); + ZEN_OPERATION_LOG_INFO(Output, + "Querying servers at '{}/api/v1/status/servers'\n Connection settings:{}", + DiscoveryHost, + ConnectionSettingsToString(ClientSettings)); } DiscoveryResponse = DiscoverJupiterEndpoints(DiscoveryHost, ClientSettings); @@ -87,14 +89,14 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings, { if (Verbose) { - ZEN_CONSOLE("Testing server endpoint at '{}/health/live'. Assume http2: {}", OverrideHost, HostAssumeHttp2); + ZEN_OPERATION_LOG_INFO(Output, "Testing server endpoint at '{}/health/live'. Assume http2: {}", OverrideHost, HostAssumeHttp2); } if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(OverrideHost, HostAssumeHttp2, ClientSettings.Verbose); TestResult.Success) { if (Verbose) { - ZEN_CONSOLE("Server endpoint at '{}/api/v1/status/servers' succeeded", OverrideHost); + ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", OverrideHost); } HostUrl = OverrideHost; HostName = GetHostNameFromUrl(OverrideHost); @@ -117,16 +119,17 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings, { if (Verbose) { - ZEN_CONSOLE("Testing server endpoint at '{}/health/live'. Assume http2: {}", - ServerEndpoint.BaseUrl, - ServerEndpoint.AssumeHttp2); + ZEN_OPERATION_LOG_INFO(Output, + "Testing server endpoint at '{}/health/live'. Assume http2: {}", + ServerEndpoint.BaseUrl, + ServerEndpoint.AssumeHttp2); } if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2, ClientSettings.Verbose); TestResult.Success) { - ZEN_CONSOLE("Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl); + ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl); HostUrl = ServerEndpoint.BaseUrl; HostAssumeHttp2 = ServerEndpoint.AssumeHttp2; @@ -135,7 +138,10 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings, } else { - ZEN_DEBUG("Unable to reach host {}. Reason: {}", ServerEndpoint.BaseUrl, TestResult.FailureReason); + ZEN_OPERATION_LOG_DEBUG(Output, + "Unable to reach host {}. Reason: {}", + ServerEndpoint.BaseUrl, + TestResult.FailureReason); } } } @@ -156,16 +162,17 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings, { if (Verbose) { - ZEN_CONSOLE("Testing cache endpoint at '{}/status/builds'. Assume http2: {}", - CacheEndpoint.BaseUrl, - CacheEndpoint.AssumeHttp2); + ZEN_OPERATION_LOG_INFO(Output, + "Testing cache endpoint at '{}/status/builds'. Assume http2: {}", + CacheEndpoint.BaseUrl, + CacheEndpoint.AssumeHttp2); } if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2, ClientSettings.Verbose); TestResult.Success) { - ZEN_CONSOLE("Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl); + ZEN_OPERATION_LOG_INFO(Output, "Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl); CacheUrl = CacheEndpoint.BaseUrl; CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2; @@ -201,7 +208,7 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings, { if (Verbose) { - ZEN_CONSOLE("Testing cache endpoint at '{}/status/builds'. Assume http2: {}", ZenCacheHost, false); + ZEN_OPERATION_LOG_INFO(Output, "Testing cache endpoint at '{}/status/builds'. Assume http2: {}", ZenCacheHost, false); } if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenCacheHost, /*AssumeHttp2*/ false, ClientSettings.Verbose); TestResult.Success) @@ -239,7 +246,7 @@ GetBlockDescriptions(OperationLogOutput& Output, if (!IsQuiet) { - ZEN_CONSOLE("Fetching metadata for {} blocks", BlockRawHashes.size()); + ZEN_OPERATION_LOG_INFO(Output, "Fetching metadata for {} blocks", BlockRawHashes.size()); } Stopwatch GetBlockMetadataTimer; diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h index 7ac1f78d4..ab3037c89 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h @@ -31,7 +31,8 @@ enum class ZenCacheResolveMode All }; -BuildStorageResolveResult ResolveBuildStorage(const HttpClientSettings& ClientSettings, +BuildStorageResolveResult ResolveBuildStorage(OperationLogOutput& Output, + const HttpClientSettings& ClientSettings, std::string_view Host, std::string_view OverrideHost, std::string_view ZenCacheHost, diff --git a/src/zenremotestore/include/zenremotestore/operationlogoutput.h b/src/zenremotestore/include/zenremotestore/operationlogoutput.h index 49649fc76..9693e69cf 100644 --- a/src/zenremotestore/include/zenremotestore/operationlogoutput.h +++ b/src/zenremotestore/include/zenremotestore/operationlogoutput.h @@ -57,6 +57,10 @@ public: virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) = 0; }; +struct LoggerRef; + +OperationLogOutput* CreateStandardLogOutput(LoggerRef& Log); + #define ZEN_OPERATION_LOG(OutputTarget, InLevel, fmtstr, ...) \ do \ { \ diff --git a/src/zenremotestore/operationlogoutput.cpp b/src/zenremotestore/operationlogoutput.cpp index 8d8a6dc78..0837ed716 100644 --- a/src/zenremotestore/operationlogoutput.cpp +++ b/src/zenremotestore/operationlogoutput.cpp @@ -4,6 +4,100 @@ #include <zencore/logging.h> +ZEN_THIRD_PARTY_INCLUDES_START +#include <gsl/gsl-lite.hpp> +ZEN_THIRD_PARTY_INCLUDES_END + namespace zen { +class StandardLogOutput; + +class StandardLogOutputProgressBar : public OperationLogOutput::ProgressBar +{ +public: + StandardLogOutputProgressBar(StandardLogOutput& Output, std::string_view InSubTask) : m_Output(Output), m_SubTask(InSubTask) {} + + virtual void UpdateState(const State& NewState, bool DoLinebreak) override; + virtual void Finish() override; + +private: + StandardLogOutput& m_Output; + std::string m_SubTask; + State m_State; +}; + +class StandardLogOutput : public OperationLogOutput +{ +public: + StandardLogOutput(LoggerRef& Log) : m_Log(Log) {} + virtual void EmitLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) override + { + if (m_Log.ShouldLog(LogLevel)) + { + fmt::basic_memory_buffer<char, 250> MessageBuffer; + fmt::vformat_to(fmt::appender(MessageBuffer), Format, Args); + ZEN_LOG(m_Log, LogLevel, "{}", std::string_view(MessageBuffer.data(), MessageBuffer.size())); + } + } + + virtual void SetLogOperationName(std::string_view Name) override + { + m_LogOperationName = Name; + ZEN_OPERATION_LOG_INFO(*this, "{}", m_LogOperationName); + } + virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) override + { + const size_t PercentDone = StepCount > 0u ? gsl::narrow<uint8_t>((100 * StepIndex) / StepCount) : 0u; + ZEN_OPERATION_LOG_INFO(*this, "{}: {}%", m_LogOperationName, PercentDone); + } + virtual uint32_t GetProgressUpdateDelayMS() override { return 2000; } + virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) override + { + return new StandardLogOutputProgressBar(*this, InSubTask); + } + +private: + LoggerRef m_Log; + std::string m_LogOperationName; +}; + +void +StandardLogOutputProgressBar::UpdateState(const State& NewState, bool DoLinebreak) +{ + ZEN_UNUSED(DoLinebreak); + const size_t PercentDone = + NewState.TotalCount > 0u ? gsl::narrow<uint8_t>((100 * (NewState.TotalCount - NewState.RemainingCount)) / NewState.TotalCount) : 0u; + std::string Task = NewState.Task; + switch (NewState.Status) + { + case State::EStatus::Aborted: + Task = "Aborting"; + break; + case State::EStatus::Paused: + Task = "Paused"; + break; + default: + break; + } + ZEN_OPERATION_LOG_INFO(m_Output, "{}: {}%{}", Task, PercentDone, NewState.Details.empty() ? "" : fmt::format(" {}", NewState.Details)); + m_State = NewState; +} +void +StandardLogOutputProgressBar::Finish() +{ + if (m_State.RemainingCount > 0) + { + State NewState = m_State; + NewState.RemainingCount = 0; + NewState.Details = ""; + UpdateState(NewState, /*DoLinebreak*/ true); + } +} + +OperationLogOutput* +CreateStandardLogOutput(LoggerRef& Log) +{ + return new StandardLogOutput(Log); +} + } // namespace zen diff --git a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp index bb3de738a..7a03ce50a 100644 --- a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp @@ -11,6 +11,7 @@ #include <zenremotestore/builds/buildstoragecache.h> #include <zenremotestore/builds/buildstorageutil.h> #include <zenremotestore/builds/jupiterbuildstorage.h> +#include <zenremotestore/operationlogoutput.h> #include <numeric> @@ -668,7 +669,10 @@ CreateJupiterBuildsRemoteStore(LoggerRef InLog, .AllowResume = true, .RetryCount = 2}; - ResolveRes = ResolveBuildStorage(ClientSettings, Host, OverrideUrl, ZenHost, ZenCacheResolveMode::Discovery, /*Verbose*/ false); + std::unique_ptr<OperationLogOutput> Output(CreateStandardLogOutput(InLog)); + + ResolveRes = + ResolveBuildStorage(*Output, ClientSettings, Host, OverrideUrl, ZenHost, ZenCacheResolveMode::Discovery, /*Verbose*/ false); } HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", |