diff options
| author | Dan Engelbrecht <[email protected]> | 2025-12-11 10:02:06 +0100 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-12-11 10:02:06 +0100 |
| commit | a598cb444f311c39bbf8d83e2004dd3545b9412d (patch) | |
| tree | e2527cd9c7fcb62e89605d0a3982ce3b052d5fba /src | |
| parent | HTTP server API changes for improved extensibility (#684) (diff) | |
| download | zen-a598cb444f311c39bbf8d83e2004dd3545b9412d.tar.xz zen-a598cb444f311c39bbf8d83e2004dd3545b9412d.zip | |
remove direct console output from code that is used from service mode (#688)
Diffstat (limited to 'src')
| -rw-r--r-- | src/zen/cmds/builds_cmd.cpp | 165 | ||||
| -rw-r--r-- | src/zen/cmds/projectstore_cmd.cpp | 13 | ||||
| -rw-r--r-- | src/zenremotestore/builds/buildstorageoperations.cpp | 25 | ||||
| -rw-r--r-- | src/zenremotestore/builds/buildstorageutil.cpp | 41 | ||||
| -rw-r--r-- | src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h | 3 | ||||
| -rw-r--r-- | src/zenremotestore/include/zenremotestore/operationlogoutput.h | 4 | ||||
| -rw-r--r-- | src/zenremotestore/operationlogoutput.cpp | 94 | ||||
| -rw-r--r-- | src/zenremotestore/projectstore/buildsremoteprojectstore.cpp | 6 |
8 files changed, 244 insertions, 107 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 7060f092a..b1052654d 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -393,7 +393,8 @@ namespace { return CleanAndRemoveDirectory(WorkerPool, AbortFlag, PauseFlag, Directory); } - void ValidateBuildPart(ThreadWorkers& Workers, + void ValidateBuildPart(OperationLogOutput& Output, + ThreadWorkers& Workers, BuildStorageBase& Storage, const Oid& BuildId, Oid BuildPartId, @@ -403,9 +404,7 @@ namespace { ProgressBar::SetLogOperationName(ProgressMode, "Validate Part"); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); - - BuildsOperationValidateBuildPart ValidateOp(*Output, + BuildsOperationValidateBuildPart ValidateOp(Output, Storage, AbortFlag, PauseFlag, @@ -428,7 +427,8 @@ namespace { NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000)); } - void UploadFolder(ThreadWorkers& Workers, + void UploadFolder(OperationLogOutput& Output, + ThreadWorkers& Workers, StorageInstance& Storage, const Oid& BuildId, const Oid& BuildPartId, @@ -448,10 +448,8 @@ namespace { { Stopwatch UploadTimer; - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); - BuildsOperationUploadFolder UploadOp( - *Output, + Output, Storage, AbortFlag, PauseFlag, @@ -1151,7 +1149,8 @@ namespace { return Result; } - ChunkedFolderContent GetRemoteContent(StorageInstance& Storage, + ChunkedFolderContent GetRemoteContent(OperationLogOutput& Output, + StorageInstance& Storage, const Oid& BuildId, const std::vector<std::pair<Oid, std::string>>& BuildParts, std::span<const std::string> IncludeWildcards, @@ -1184,15 +1183,15 @@ namespace { OutChunkController = CreateChunkingController(ChunkerName, Parameters); } - auto ParseBuildPartManifest = [](StorageInstance& Storage, - const Oid& BuildId, - const Oid& BuildPartId, - CbObject BuildPartManifest, - std::span<const std::string> IncludeWildcards, - std::span<const std::string> ExcludeWildcards, - ChunkedFolderContent& OutRemoteContent, - std::vector<ChunkBlockDescription>& OutBlockDescriptions, - std::vector<IoHash>& OutLooseChunkHashes) { + auto ParseBuildPartManifest = [&Output](StorageInstance& Storage, + const Oid& BuildId, + const Oid& BuildPartId, + CbObject BuildPartManifest, + std::span<const std::string> IncludeWildcards, + std::span<const std::string> ExcludeWildcards, + ChunkedFolderContent& OutRemoteContent, + std::vector<ChunkBlockDescription>& OutBlockDescriptions, + std::vector<IoHash>& OutLooseChunkHashes) { std::vector<uint32_t> AbsoluteChunkOrders; std::vector<uint64_t> LooseChunkRawSizes; std::vector<IoHash> BlockRawHashes; @@ -1213,17 +1212,16 @@ namespace { // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them { - std::unique_ptr<OperationLogOutput> OperationLogOutput(CreateConsoleLogOutput(ProgressMode)); - bool AttemptFallback = false; - OutBlockDescriptions = GetBlockDescriptions(*OperationLogOutput, - *Storage.BuildStorage, - Storage.BuildCacheStorage.get(), - BuildId, - BuildPartId, - BlockRawHashes, - AttemptFallback, - IsQuiet, - IsVerbose); + bool AttemptFallback = false; + OutBlockDescriptions = GetBlockDescriptions(Output, + *Storage.BuildStorage, + Storage.BuildCacheStorage.get(), + BuildId, + BuildPartId, + BlockRawHashes, + AttemptFallback, + IsQuiet, + IsVerbose); } CalculateLocalChunkOrders(AbsoluteChunkOrders, @@ -1669,7 +1667,8 @@ namespace { bool AppendNewContent = false; }; - void DownloadFolder(ThreadWorkers& Workers, + void DownloadFolder(OperationLogOutput& Output, + ThreadWorkers& Workers, StorageInstance& Storage, const Oid& BuildId, const std::vector<Oid>& BuildPartIds, @@ -1720,7 +1719,8 @@ namespace { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount); - ChunkedFolderContent RemoteContent = GetRemoteContent(Storage, + ChunkedFolderContent RemoteContent = GetRemoteContent(Output, + Storage, BuildId, AllBuildParts, Options.IncludeWildcards, @@ -1875,9 +1875,8 @@ namespace { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); - BuildsOperationUpdateFolder Updater( - *Output, + BuildsOperationUpdateFolder Updater( + Output, Storage, AbortFlag, PauseFlag, @@ -1892,24 +1891,24 @@ namespace { BlockDescriptions, LooseChunkHashes, BuildsOperationUpdateFolder::Options{ - .IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .AllowFileClone = Options.AllowFileClone, - .UseSparseFiles = UseSparseFiles, - .SystemRootDir = Options.SystemRootDir, - .ZenFolderPath = Options.ZenFolderPath, - .LargeAttachmentSize = LargeAttachmentSize, - .PreferredMultipartChunkSize = PreferredMultipartChunkSize, - .PartialBlockRequestMode = Options.PartialBlockRequestMode, - .WipeTargetFolder = Options.CleanTargetFolder, - .PrimeCacheOnly = Options.PrimeCacheOnly, - .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging, - .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging || Options.AppendNewContent, - .ValidateCompletedSequences = Options.PostDownloadVerify, - .ExcludeFolders = DefaultExcludeFolders, - .ExcludeExtensions = DefaultExcludeExtensions, - .MaximumInMemoryPayloadSize = Options.MaximumInMemoryPayloadSize, - .PopulateCache = Options.PopulateCache}); + .IsQuiet = IsQuiet, + .IsVerbose = IsVerbose, + .AllowFileClone = Options.AllowFileClone, + .UseSparseFiles = UseSparseFiles, + .SystemRootDir = Options.SystemRootDir, + .ZenFolderPath = Options.ZenFolderPath, + .LargeAttachmentSize = LargeAttachmentSize, + .PreferredMultipartChunkSize = PreferredMultipartChunkSize, + .PartialBlockRequestMode = Options.PartialBlockRequestMode, + .WipeTargetFolder = Options.CleanTargetFolder, + .PrimeCacheOnly = Options.PrimeCacheOnly, + .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging, + .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging || Options.AppendNewContent, + .ValidateCompletedSequences = Options.PostDownloadVerify, + .ExcludeFolders = DefaultExcludeFolders, + .ExcludeExtensions = DefaultExcludeExtensions, + .MaximumInMemoryPayloadSize = Options.MaximumInMemoryPayloadSize, + .PopulateCache = Options.PopulateCache}); { ProgressBar::PushLogOperation(ProgressMode, "Download"); auto _ = MakeGuard([]() { ProgressBar::PopLogOperation(ProgressMode); }); @@ -3001,6 +3000,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::unique_ptr<AuthMgr> Auth; + std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); + auto CreateBuildStorage = [&](BuildStorageBase::Statistics& StorageStats, BuildStorageCache::Statistics& StorageCacheStats, const std::filesystem::path& TempPath, @@ -3035,7 +3036,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_Verbose); BuildStorageResolveResult ResolveRes = - ResolveBuildStorage(ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All, m_Verbose); + ResolveBuildStorage(*Output, ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All, m_Verbose); if (!ResolveRes.HostUrl.empty()) { ClientSettings.AssumeHttp2 = ResolveRes.HostAssumeHttp2; @@ -3607,7 +3608,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const std::filesystem::path TempDir = ZenTempFolderPath(m_ZenFolderPath); - UploadFolder(Workers, + UploadFolder(*Output, + Workers, Storage, BuildId, BuildPartId, @@ -3627,7 +3629,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (m_PostUploadVerify) { - ValidateBuildPart(Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); } } @@ -3743,7 +3745,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw OptionParseException("'--append' conflicts with '--clean'", SubOption->help()); } - DownloadFolder(Workers, + DownloadFolder(*Output, + Workers, Storage, BuildId, BuildPartIds, @@ -3889,8 +3892,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache"); - std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode)); - BuildsOperationPrimeCache PrimeOp(*Output, Storage, AbortFlag, @@ -4019,7 +4020,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(); - ValidateBuildPart(Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); if (AbortFlag) { @@ -4068,7 +4069,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildIdString), SubOption->help()); } - DownloadFolder(Workers, + DownloadFolder(*Output, + Workers, Storage, BuildId, {}, @@ -4234,7 +4236,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path); // std::filesystem::path UploadTempDir = m_ZenFolderPath / "upload_tmp"; - UploadFolder(Workers, + UploadFolder(*Output, + Workers, Storage, BuildId, BuildPartId, @@ -4254,7 +4257,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw std::runtime_error("Test aborted. (Upload build)"); } - ValidateBuildPart(Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); if (!m_IncludeWildcard.empty() || !m_ExcludeWildcard.empty()) { @@ -4266,7 +4269,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::vector<std::string> ExcludeWildcards; ParseFileFilters(IncludeWildcards, ExcludeWildcards); - DownloadFolder(Workers, + DownloadFolder(*Output, + Workers, Storage, BuildId, {BuildPartId}, @@ -4295,7 +4299,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Workers, + DownloadFolder(*Output, + Workers, Storage, BuildId, {BuildPartId}, @@ -4320,7 +4325,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Full Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Workers, + DownloadFolder(*Output, + Workers, Storage, BuildId, {BuildPartId}, @@ -4346,7 +4352,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Workers, + DownloadFolder(*Output, + Workers, Storage, BuildId, {BuildPartId}, @@ -4369,7 +4376,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Workers, + DownloadFolder(*Output, + Workers, Storage, BuildId, {BuildPartId}, @@ -4484,7 +4492,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ScrambleDir(DownloadPath); ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Workers, + DownloadFolder(*Output, + Workers, Storage, BuildId, {BuildPartId}, @@ -4518,7 +4527,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView()); } - UploadFolder(Workers, + UploadFolder(*Output, + Workers, Storage, BuildId2, BuildPartId2, @@ -4538,10 +4548,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw std::runtime_error("Test aborted. (Upload scrambled)"); } - ValidateBuildPart(Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Workers, + DownloadFolder(*Output, + Workers, Storage, BuildId, {BuildPartId}, @@ -4564,7 +4575,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(Workers, + DownloadFolder(*Output, + Workers, Storage, BuildId2, {BuildPartId2}, @@ -4586,7 +4598,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(Workers, + DownloadFolder(*Output, + Workers, Storage, BuildId2, {BuildPartId2}, @@ -4608,7 +4621,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2); - DownloadFolder(Workers, + DownloadFolder(*Output, + Workers, Storage, BuildId, {BuildPartId}, @@ -4630,7 +4644,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath3); - DownloadFolder(Workers, + DownloadFolder(*Output, + Workers, Storage, BuildId, {BuildPartId}, diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index b6af2176e..02436f4da 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -2420,6 +2420,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a ParseStorageOptions(/*RequireNamespace*/ true, /*RequireBucket*/ true); + std::unique_ptr<OperationLogOutput> OperationLogOutput(CreateConsoleLogOutput(ProgressMode)); + ThreadWorkers Workers(m_BoostWorkerThreads, /*SingleThreaded*/ false); if (!m_Quiet) { @@ -2447,8 +2449,13 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a /*Hidden*/ false, m_Verbose); - BuildStorageResolveResult ResolveRes = - ResolveBuildStorage(ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::Discovery, m_Verbose); + BuildStorageResolveResult ResolveRes = ResolveBuildStorage(*OperationLogOutput, + ClientSettings, + m_Host, + m_OverrideHost, + m_ZenCacheHost, + ZenCacheResolveMode::Discovery, + m_Verbose); BuildStorageBase::Statistics StorageStats; @@ -2520,8 +2527,6 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a false ? GetSmallWorkerPool(EWorkloadType::Background) : GetTinyWorkerPool(EWorkloadType::Background)); } - std::unique_ptr<OperationLogOutput> OperationLogOutput(CreateConsoleLogOutput(ProgressMode)); - ProjectStoreOperationOplogState State( *OperationLogOutput, Storage, diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp index 5bfef842e..b8dd18bb5 100644 --- a/src/zenremotestore/builds/buildstorageoperations.cpp +++ b/src/zenremotestore/builds/buildstorageoperations.cpp @@ -2833,7 +2833,7 @@ BuildsOperationUpdateFolder::FindScavengeContent(const ScavengeSource& Source, } catch (const std::exception& Ex) { - ZEN_CONSOLE_DEBUG("Skipping invalid build state at '{}', reason: {}", Source.StateFilePath, Ex.what()); + ZEN_OPERATION_LOG_DEBUG(m_LogOutput, "Skipping invalid build state at '{}', reason: {}", Source.StateFilePath, Ex.what()); return false; } @@ -5374,7 +5374,7 @@ BuildsOperationUploadFolder::Execute() if (!UnknownChunks.empty()) { - ZEN_CONSOLE_WARN("{}", BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ true)); + ZEN_OPERATION_LOG_WARN(m_LogOutput, "{}", BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ true)); } uint32_t FinalizeBuildPartRetryCount = 5; @@ -5411,7 +5411,9 @@ BuildsOperationUploadFolder::Execute() else { UnknownChunks = RetryUnknownChunks; - ZEN_CONSOLE_WARN("{}", BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ FinalizeBuildPartRetryCount != 0)); + ZEN_OPERATION_LOG_WARN(m_LogOutput, + "{}", + BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ FinalizeBuildPartRetryCount != 0)); } } @@ -6896,11 +6898,12 @@ BuildsOperationValidateBuildPart::Execute() auto _ = MakeGuard([&]() { if (!m_Options.IsQuiet) { - ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}", - m_BuildId, - m_BuildPartId, - m_BuildPartName, - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Validated build part {}/{} ('{}') in {}", + m_BuildId, + m_BuildPartId, + m_BuildPartName, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } }); @@ -6928,7 +6931,11 @@ BuildsOperationValidateBuildPart::Execute() m_ValidateStats.BuildPartSize = BuildPart.GetSize(); if (!m_Options.IsQuiet) { - ZEN_CONSOLE("Validating build part {}/{} ({})", m_BuildId, m_BuildPartId, NiceBytes(BuildPart.GetSize())); + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Validating build part {}/{} ({})", + m_BuildId, + m_BuildPartId, + NiceBytes(BuildPart.GetSize())); } std::vector<IoHash> ChunkAttachments; if (const CbObjectView ChunkAttachmentsView = BuildPart["chunkAttachments"sv].AsObjectView()) diff --git a/src/zenremotestore/builds/buildstorageutil.cpp b/src/zenremotestore/builds/buildstorageutil.cpp index 3af323628..15ece2edd 100644 --- a/src/zenremotestore/builds/buildstorageutil.cpp +++ b/src/zenremotestore/builds/buildstorageutil.cpp @@ -32,7 +32,8 @@ namespace { } // namespace BuildStorageResolveResult -ResolveBuildStorage(const HttpClientSettings& ClientSettings, +ResolveBuildStorage(OperationLogOutput& Output, + const HttpClientSettings& ClientSettings, std::string_view Host, std::string_view OverrideHost, std::string_view ZenCacheHost, @@ -75,9 +76,10 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings, { if (Verbose) { - ZEN_CONSOLE("Querying servers at '{}/api/v1/status/servers'\n Connection settings:{}", - DiscoveryHost, - ConnectionSettingsToString(ClientSettings)); + ZEN_OPERATION_LOG_INFO(Output, + "Querying servers at '{}/api/v1/status/servers'\n Connection settings:{}", + DiscoveryHost, + ConnectionSettingsToString(ClientSettings)); } DiscoveryResponse = DiscoverJupiterEndpoints(DiscoveryHost, ClientSettings); @@ -87,14 +89,14 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings, { if (Verbose) { - ZEN_CONSOLE("Testing server endpoint at '{}/health/live'. Assume http2: {}", OverrideHost, HostAssumeHttp2); + ZEN_OPERATION_LOG_INFO(Output, "Testing server endpoint at '{}/health/live'. Assume http2: {}", OverrideHost, HostAssumeHttp2); } if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(OverrideHost, HostAssumeHttp2, ClientSettings.Verbose); TestResult.Success) { if (Verbose) { - ZEN_CONSOLE("Server endpoint at '{}/api/v1/status/servers' succeeded", OverrideHost); + ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", OverrideHost); } HostUrl = OverrideHost; HostName = GetHostNameFromUrl(OverrideHost); @@ -117,16 +119,17 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings, { if (Verbose) { - ZEN_CONSOLE("Testing server endpoint at '{}/health/live'. Assume http2: {}", - ServerEndpoint.BaseUrl, - ServerEndpoint.AssumeHttp2); + ZEN_OPERATION_LOG_INFO(Output, + "Testing server endpoint at '{}/health/live'. Assume http2: {}", + ServerEndpoint.BaseUrl, + ServerEndpoint.AssumeHttp2); } if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2, ClientSettings.Verbose); TestResult.Success) { - ZEN_CONSOLE("Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl); + ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl); HostUrl = ServerEndpoint.BaseUrl; HostAssumeHttp2 = ServerEndpoint.AssumeHttp2; @@ -135,7 +138,10 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings, } else { - ZEN_DEBUG("Unable to reach host {}. Reason: {}", ServerEndpoint.BaseUrl, TestResult.FailureReason); + ZEN_OPERATION_LOG_DEBUG(Output, + "Unable to reach host {}. Reason: {}", + ServerEndpoint.BaseUrl, + TestResult.FailureReason); } } } @@ -156,16 +162,17 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings, { if (Verbose) { - ZEN_CONSOLE("Testing cache endpoint at '{}/status/builds'. Assume http2: {}", - CacheEndpoint.BaseUrl, - CacheEndpoint.AssumeHttp2); + ZEN_OPERATION_LOG_INFO(Output, + "Testing cache endpoint at '{}/status/builds'. Assume http2: {}", + CacheEndpoint.BaseUrl, + CacheEndpoint.AssumeHttp2); } if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2, ClientSettings.Verbose); TestResult.Success) { - ZEN_CONSOLE("Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl); + ZEN_OPERATION_LOG_INFO(Output, "Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl); CacheUrl = CacheEndpoint.BaseUrl; CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2; @@ -201,7 +208,7 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings, { if (Verbose) { - ZEN_CONSOLE("Testing cache endpoint at '{}/status/builds'. Assume http2: {}", ZenCacheHost, false); + ZEN_OPERATION_LOG_INFO(Output, "Testing cache endpoint at '{}/status/builds'. Assume http2: {}", ZenCacheHost, false); } if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenCacheHost, /*AssumeHttp2*/ false, ClientSettings.Verbose); TestResult.Success) @@ -239,7 +246,7 @@ GetBlockDescriptions(OperationLogOutput& Output, if (!IsQuiet) { - ZEN_CONSOLE("Fetching metadata for {} blocks", BlockRawHashes.size()); + ZEN_OPERATION_LOG_INFO(Output, "Fetching metadata for {} blocks", BlockRawHashes.size()); } Stopwatch GetBlockMetadataTimer; diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h index 7ac1f78d4..ab3037c89 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h @@ -31,7 +31,8 @@ enum class ZenCacheResolveMode All }; -BuildStorageResolveResult ResolveBuildStorage(const HttpClientSettings& ClientSettings, +BuildStorageResolveResult ResolveBuildStorage(OperationLogOutput& Output, + const HttpClientSettings& ClientSettings, std::string_view Host, std::string_view OverrideHost, std::string_view ZenCacheHost, diff --git a/src/zenremotestore/include/zenremotestore/operationlogoutput.h b/src/zenremotestore/include/zenremotestore/operationlogoutput.h index 49649fc76..9693e69cf 100644 --- a/src/zenremotestore/include/zenremotestore/operationlogoutput.h +++ b/src/zenremotestore/include/zenremotestore/operationlogoutput.h @@ -57,6 +57,10 @@ public: virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) = 0; }; +struct LoggerRef; + +OperationLogOutput* CreateStandardLogOutput(LoggerRef& Log); + #define ZEN_OPERATION_LOG(OutputTarget, InLevel, fmtstr, ...) \ do \ { \ diff --git a/src/zenremotestore/operationlogoutput.cpp b/src/zenremotestore/operationlogoutput.cpp index 8d8a6dc78..0837ed716 100644 --- a/src/zenremotestore/operationlogoutput.cpp +++ b/src/zenremotestore/operationlogoutput.cpp @@ -4,6 +4,100 @@ #include <zencore/logging.h> +ZEN_THIRD_PARTY_INCLUDES_START +#include <gsl/gsl-lite.hpp> +ZEN_THIRD_PARTY_INCLUDES_END + namespace zen { +class StandardLogOutput; + +class StandardLogOutputProgressBar : public OperationLogOutput::ProgressBar +{ +public: + StandardLogOutputProgressBar(StandardLogOutput& Output, std::string_view InSubTask) : m_Output(Output), m_SubTask(InSubTask) {} + + virtual void UpdateState(const State& NewState, bool DoLinebreak) override; + virtual void Finish() override; + +private: + StandardLogOutput& m_Output; + std::string m_SubTask; + State m_State; +}; + +class StandardLogOutput : public OperationLogOutput +{ +public: + StandardLogOutput(LoggerRef& Log) : m_Log(Log) {} + virtual void EmitLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) override + { + if (m_Log.ShouldLog(LogLevel)) + { + fmt::basic_memory_buffer<char, 250> MessageBuffer; + fmt::vformat_to(fmt::appender(MessageBuffer), Format, Args); + ZEN_LOG(m_Log, LogLevel, "{}", std::string_view(MessageBuffer.data(), MessageBuffer.size())); + } + } + + virtual void SetLogOperationName(std::string_view Name) override + { + m_LogOperationName = Name; + ZEN_OPERATION_LOG_INFO(*this, "{}", m_LogOperationName); + } + virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) override + { + const size_t PercentDone = StepCount > 0u ? gsl::narrow<uint8_t>((100 * StepIndex) / StepCount) : 0u; + ZEN_OPERATION_LOG_INFO(*this, "{}: {}%", m_LogOperationName, PercentDone); + } + virtual uint32_t GetProgressUpdateDelayMS() override { return 2000; } + virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) override + { + return new StandardLogOutputProgressBar(*this, InSubTask); + } + +private: + LoggerRef m_Log; + std::string m_LogOperationName; +}; + +void +StandardLogOutputProgressBar::UpdateState(const State& NewState, bool DoLinebreak) +{ + ZEN_UNUSED(DoLinebreak); + const size_t PercentDone = + NewState.TotalCount > 0u ? gsl::narrow<uint8_t>((100 * (NewState.TotalCount - NewState.RemainingCount)) / NewState.TotalCount) : 0u; + std::string Task = NewState.Task; + switch (NewState.Status) + { + case State::EStatus::Aborted: + Task = "Aborting"; + break; + case State::EStatus::Paused: + Task = "Paused"; + break; + default: + break; + } + ZEN_OPERATION_LOG_INFO(m_Output, "{}: {}%{}", Task, PercentDone, NewState.Details.empty() ? "" : fmt::format(" {}", NewState.Details)); + m_State = NewState; +} +void +StandardLogOutputProgressBar::Finish() +{ + if (m_State.RemainingCount > 0) + { + State NewState = m_State; + NewState.RemainingCount = 0; + NewState.Details = ""; + UpdateState(NewState, /*DoLinebreak*/ true); + } +} + +OperationLogOutput* +CreateStandardLogOutput(LoggerRef& Log) +{ + return new StandardLogOutput(Log); +} + } // namespace zen diff --git a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp index bb3de738a..7a03ce50a 100644 --- a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp @@ -11,6 +11,7 @@ #include <zenremotestore/builds/buildstoragecache.h> #include <zenremotestore/builds/buildstorageutil.h> #include <zenremotestore/builds/jupiterbuildstorage.h> +#include <zenremotestore/operationlogoutput.h> #include <numeric> @@ -668,7 +669,10 @@ CreateJupiterBuildsRemoteStore(LoggerRef InLog, .AllowResume = true, .RetryCount = 2}; - ResolveRes = ResolveBuildStorage(ClientSettings, Host, OverrideUrl, ZenHost, ZenCacheResolveMode::Discovery, /*Verbose*/ false); + std::unique_ptr<OperationLogOutput> Output(CreateStandardLogOutput(InLog)); + + ResolveRes = + ResolveBuildStorage(*Output, ClientSettings, Host, OverrideUrl, ZenHost, ZenCacheResolveMode::Discovery, /*Verbose*/ false); } HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", |