aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2025-12-11 10:02:06 +0100
committerGitHub Enterprise <[email protected]>2025-12-11 10:02:06 +0100
commita598cb444f311c39bbf8d83e2004dd3545b9412d (patch)
treee2527cd9c7fcb62e89605d0a3982ce3b052d5fba /src
parentHTTP server API changes for improved extensibility (#684) (diff)
downloadzen-a598cb444f311c39bbf8d83e2004dd3545b9412d.tar.xz
zen-a598cb444f311c39bbf8d83e2004dd3545b9412d.zip
remove direct console output from code that is used from service mode (#688)
Diffstat (limited to 'src')
-rw-r--r--src/zen/cmds/builds_cmd.cpp165
-rw-r--r--src/zen/cmds/projectstore_cmd.cpp13
-rw-r--r--src/zenremotestore/builds/buildstorageoperations.cpp25
-rw-r--r--src/zenremotestore/builds/buildstorageutil.cpp41
-rw-r--r--src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h3
-rw-r--r--src/zenremotestore/include/zenremotestore/operationlogoutput.h4
-rw-r--r--src/zenremotestore/operationlogoutput.cpp94
-rw-r--r--src/zenremotestore/projectstore/buildsremoteprojectstore.cpp6
8 files changed, 244 insertions, 107 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 7060f092a..b1052654d 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -393,7 +393,8 @@ namespace {
return CleanAndRemoveDirectory(WorkerPool, AbortFlag, PauseFlag, Directory);
}
- void ValidateBuildPart(ThreadWorkers& Workers,
+ void ValidateBuildPart(OperationLogOutput& Output,
+ ThreadWorkers& Workers,
BuildStorageBase& Storage,
const Oid& BuildId,
Oid BuildPartId,
@@ -403,9 +404,7 @@ namespace {
ProgressBar::SetLogOperationName(ProgressMode, "Validate Part");
- std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
-
- BuildsOperationValidateBuildPart ValidateOp(*Output,
+ BuildsOperationValidateBuildPart ValidateOp(Output,
Storage,
AbortFlag,
PauseFlag,
@@ -428,7 +427,8 @@ namespace {
NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000));
}
- void UploadFolder(ThreadWorkers& Workers,
+ void UploadFolder(OperationLogOutput& Output,
+ ThreadWorkers& Workers,
StorageInstance& Storage,
const Oid& BuildId,
const Oid& BuildPartId,
@@ -448,10 +448,8 @@ namespace {
{
Stopwatch UploadTimer;
- std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
-
BuildsOperationUploadFolder UploadOp(
- *Output,
+ Output,
Storage,
AbortFlag,
PauseFlag,
@@ -1151,7 +1149,8 @@ namespace {
return Result;
}
- ChunkedFolderContent GetRemoteContent(StorageInstance& Storage,
+ ChunkedFolderContent GetRemoteContent(OperationLogOutput& Output,
+ StorageInstance& Storage,
const Oid& BuildId,
const std::vector<std::pair<Oid, std::string>>& BuildParts,
std::span<const std::string> IncludeWildcards,
@@ -1184,15 +1183,15 @@ namespace {
OutChunkController = CreateChunkingController(ChunkerName, Parameters);
}
- auto ParseBuildPartManifest = [](StorageInstance& Storage,
- const Oid& BuildId,
- const Oid& BuildPartId,
- CbObject BuildPartManifest,
- std::span<const std::string> IncludeWildcards,
- std::span<const std::string> ExcludeWildcards,
- ChunkedFolderContent& OutRemoteContent,
- std::vector<ChunkBlockDescription>& OutBlockDescriptions,
- std::vector<IoHash>& OutLooseChunkHashes) {
+ auto ParseBuildPartManifest = [&Output](StorageInstance& Storage,
+ const Oid& BuildId,
+ const Oid& BuildPartId,
+ CbObject BuildPartManifest,
+ std::span<const std::string> IncludeWildcards,
+ std::span<const std::string> ExcludeWildcards,
+ ChunkedFolderContent& OutRemoteContent,
+ std::vector<ChunkBlockDescription>& OutBlockDescriptions,
+ std::vector<IoHash>& OutLooseChunkHashes) {
std::vector<uint32_t> AbsoluteChunkOrders;
std::vector<uint64_t> LooseChunkRawSizes;
std::vector<IoHash> BlockRawHashes;
@@ -1213,17 +1212,16 @@ namespace {
// TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them
{
- std::unique_ptr<OperationLogOutput> OperationLogOutput(CreateConsoleLogOutput(ProgressMode));
- bool AttemptFallback = false;
- OutBlockDescriptions = GetBlockDescriptions(*OperationLogOutput,
- *Storage.BuildStorage,
- Storage.BuildCacheStorage.get(),
- BuildId,
- BuildPartId,
- BlockRawHashes,
- AttemptFallback,
- IsQuiet,
- IsVerbose);
+ bool AttemptFallback = false;
+ OutBlockDescriptions = GetBlockDescriptions(Output,
+ *Storage.BuildStorage,
+ Storage.BuildCacheStorage.get(),
+ BuildId,
+ BuildPartId,
+ BlockRawHashes,
+ AttemptFallback,
+ IsQuiet,
+ IsVerbose);
}
CalculateLocalChunkOrders(AbsoluteChunkOrders,
@@ -1669,7 +1667,8 @@ namespace {
bool AppendNewContent = false;
};
- void DownloadFolder(ThreadWorkers& Workers,
+ void DownloadFolder(OperationLogOutput& Output,
+ ThreadWorkers& Workers,
StorageInstance& Storage,
const Oid& BuildId,
const std::vector<Oid>& BuildPartIds,
@@ -1720,7 +1719,8 @@ namespace {
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount);
- ChunkedFolderContent RemoteContent = GetRemoteContent(Storage,
+ ChunkedFolderContent RemoteContent = GetRemoteContent(Output,
+ Storage,
BuildId,
AllBuildParts,
Options.IncludeWildcards,
@@ -1875,9 +1875,8 @@ namespace {
ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount);
- std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
- BuildsOperationUpdateFolder Updater(
- *Output,
+ BuildsOperationUpdateFolder Updater(
+ Output,
Storage,
AbortFlag,
PauseFlag,
@@ -1892,24 +1891,24 @@ namespace {
BlockDescriptions,
LooseChunkHashes,
BuildsOperationUpdateFolder::Options{
- .IsQuiet = IsQuiet,
- .IsVerbose = IsVerbose,
- .AllowFileClone = Options.AllowFileClone,
- .UseSparseFiles = UseSparseFiles,
- .SystemRootDir = Options.SystemRootDir,
- .ZenFolderPath = Options.ZenFolderPath,
- .LargeAttachmentSize = LargeAttachmentSize,
- .PreferredMultipartChunkSize = PreferredMultipartChunkSize,
- .PartialBlockRequestMode = Options.PartialBlockRequestMode,
- .WipeTargetFolder = Options.CleanTargetFolder,
- .PrimeCacheOnly = Options.PrimeCacheOnly,
- .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging,
- .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging || Options.AppendNewContent,
- .ValidateCompletedSequences = Options.PostDownloadVerify,
- .ExcludeFolders = DefaultExcludeFolders,
- .ExcludeExtensions = DefaultExcludeExtensions,
- .MaximumInMemoryPayloadSize = Options.MaximumInMemoryPayloadSize,
- .PopulateCache = Options.PopulateCache});
+ .IsQuiet = IsQuiet,
+ .IsVerbose = IsVerbose,
+ .AllowFileClone = Options.AllowFileClone,
+ .UseSparseFiles = UseSparseFiles,
+ .SystemRootDir = Options.SystemRootDir,
+ .ZenFolderPath = Options.ZenFolderPath,
+ .LargeAttachmentSize = LargeAttachmentSize,
+ .PreferredMultipartChunkSize = PreferredMultipartChunkSize,
+ .PartialBlockRequestMode = Options.PartialBlockRequestMode,
+ .WipeTargetFolder = Options.CleanTargetFolder,
+ .PrimeCacheOnly = Options.PrimeCacheOnly,
+ .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging,
+ .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging || Options.AppendNewContent,
+ .ValidateCompletedSequences = Options.PostDownloadVerify,
+ .ExcludeFolders = DefaultExcludeFolders,
+ .ExcludeExtensions = DefaultExcludeExtensions,
+ .MaximumInMemoryPayloadSize = Options.MaximumInMemoryPayloadSize,
+ .PopulateCache = Options.PopulateCache});
{
ProgressBar::PushLogOperation(ProgressMode, "Download");
auto _ = MakeGuard([]() { ProgressBar::PopLogOperation(ProgressMode); });
@@ -3001,6 +3000,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
std::unique_ptr<AuthMgr> Auth;
+ std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
+
auto CreateBuildStorage = [&](BuildStorageBase::Statistics& StorageStats,
BuildStorageCache::Statistics& StorageCacheStats,
const std::filesystem::path& TempPath,
@@ -3035,7 +3036,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
m_Verbose);
BuildStorageResolveResult ResolveRes =
- ResolveBuildStorage(ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All, m_Verbose);
+ ResolveBuildStorage(*Output, ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All, m_Verbose);
if (!ResolveRes.HostUrl.empty())
{
ClientSettings.AssumeHttp2 = ResolveRes.HostAssumeHttp2;
@@ -3607,7 +3608,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
const std::filesystem::path TempDir = ZenTempFolderPath(m_ZenFolderPath);
- UploadFolder(Workers,
+ UploadFolder(*Output,
+ Workers,
Storage,
BuildId,
BuildPartId,
@@ -3627,7 +3629,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
if (m_PostUploadVerify)
{
- ValidateBuildPart(Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+ ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
}
}
@@ -3743,7 +3745,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
throw OptionParseException("'--append' conflicts with '--clean'", SubOption->help());
}
- DownloadFolder(Workers,
+ DownloadFolder(*Output,
+ Workers,
Storage,
BuildId,
BuildPartIds,
@@ -3889,8 +3892,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ProgressBar::SetLogOperationName(ProgressMode, "Prime Cache");
- std::unique_ptr<OperationLogOutput> Output(CreateConsoleLogOutput(ProgressMode));
-
BuildsOperationPrimeCache PrimeOp(*Output,
Storage,
AbortFlag,
@@ -4019,7 +4020,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId();
- ValidateBuildPart(Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+ ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
if (AbortFlag)
{
@@ -4068,7 +4069,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
throw OptionParseException(fmt::format("'--build-id' ('{}') is malformed", BuildIdString), SubOption->help());
}
- DownloadFolder(Workers,
+ DownloadFolder(*Output,
+ Workers,
Storage,
BuildId,
{},
@@ -4234,7 +4236,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path);
// std::filesystem::path UploadTempDir = m_ZenFolderPath / "upload_tmp";
- UploadFolder(Workers,
+ UploadFolder(*Output,
+ Workers,
Storage,
BuildId,
BuildPartId,
@@ -4254,7 +4257,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
throw std::runtime_error("Test aborted. (Upload build)");
}
- ValidateBuildPart(Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+ ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
if (!m_IncludeWildcard.empty() || !m_ExcludeWildcard.empty())
{
@@ -4266,7 +4269,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
std::vector<std::string> ExcludeWildcards;
ParseFileFilters(IncludeWildcards, ExcludeWildcards);
- DownloadFolder(Workers,
+ DownloadFolder(*Output,
+ Workers,
Storage,
BuildId,
{BuildPartId},
@@ -4295,7 +4299,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
BuildPartId,
m_BuildPartName,
DownloadPath);
- DownloadFolder(Workers,
+ DownloadFolder(*Output,
+ Workers,
Storage,
BuildId,
{BuildPartId},
@@ -4320,7 +4325,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
ZEN_CONSOLE("\nDownload Full Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(Workers,
+ DownloadFolder(*Output,
+ Workers,
Storage,
BuildId,
{BuildPartId},
@@ -4346,7 +4352,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(Workers,
+ DownloadFolder(*Output,
+ Workers,
Storage,
BuildId,
{BuildPartId},
@@ -4369,7 +4376,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(Workers,
+ DownloadFolder(*Output,
+ Workers,
Storage,
BuildId,
{BuildPartId},
@@ -4484,7 +4492,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ScrambleDir(DownloadPath);
ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(Workers,
+ DownloadFolder(*Output,
+ Workers,
Storage,
BuildId,
{BuildPartId},
@@ -4518,7 +4527,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView());
}
- UploadFolder(Workers,
+ UploadFolder(*Output,
+ Workers,
Storage,
BuildId2,
BuildPartId2,
@@ -4538,10 +4548,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
throw std::runtime_error("Test aborted. (Upload scrambled)");
}
- ValidateBuildPart(Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
+ ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName);
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(Workers,
+ DownloadFolder(*Output,
+ Workers,
Storage,
BuildId,
{BuildPartId},
@@ -4564,7 +4575,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
- DownloadFolder(Workers,
+ DownloadFolder(*Output,
+ Workers,
Storage,
BuildId2,
{BuildPartId2},
@@ -4586,7 +4598,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
- DownloadFolder(Workers,
+ DownloadFolder(*Output,
+ Workers,
Storage,
BuildId2,
{BuildPartId2},
@@ -4608,7 +4621,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2);
- DownloadFolder(Workers,
+ DownloadFolder(*Output,
+ Workers,
Storage,
BuildId,
{BuildPartId},
@@ -4630,7 +4644,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath3);
- DownloadFolder(Workers,
+ DownloadFolder(*Output,
+ Workers,
Storage,
BuildId,
{BuildPartId},
diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp
index b6af2176e..02436f4da 100644
--- a/src/zen/cmds/projectstore_cmd.cpp
+++ b/src/zen/cmds/projectstore_cmd.cpp
@@ -2420,6 +2420,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
ParseStorageOptions(/*RequireNamespace*/ true, /*RequireBucket*/ true);
+ std::unique_ptr<OperationLogOutput> OperationLogOutput(CreateConsoleLogOutput(ProgressMode));
+
ThreadWorkers Workers(m_BoostWorkerThreads, /*SingleThreaded*/ false);
if (!m_Quiet)
{
@@ -2447,8 +2449,13 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
/*Hidden*/ false,
m_Verbose);
- BuildStorageResolveResult ResolveRes =
- ResolveBuildStorage(ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::Discovery, m_Verbose);
+ BuildStorageResolveResult ResolveRes = ResolveBuildStorage(*OperationLogOutput,
+ ClientSettings,
+ m_Host,
+ m_OverrideHost,
+ m_ZenCacheHost,
+ ZenCacheResolveMode::Discovery,
+ m_Verbose);
BuildStorageBase::Statistics StorageStats;
@@ -2520,8 +2527,6 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
false ? GetSmallWorkerPool(EWorkloadType::Background) : GetTinyWorkerPool(EWorkloadType::Background));
}
- std::unique_ptr<OperationLogOutput> OperationLogOutput(CreateConsoleLogOutput(ProgressMode));
-
ProjectStoreOperationOplogState State(
*OperationLogOutput,
Storage,
diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp
index 5bfef842e..b8dd18bb5 100644
--- a/src/zenremotestore/builds/buildstorageoperations.cpp
+++ b/src/zenremotestore/builds/buildstorageoperations.cpp
@@ -2833,7 +2833,7 @@ BuildsOperationUpdateFolder::FindScavengeContent(const ScavengeSource& Source,
}
catch (const std::exception& Ex)
{
- ZEN_CONSOLE_DEBUG("Skipping invalid build state at '{}', reason: {}", Source.StateFilePath, Ex.what());
+ ZEN_OPERATION_LOG_DEBUG(m_LogOutput, "Skipping invalid build state at '{}', reason: {}", Source.StateFilePath, Ex.what());
return false;
}
@@ -5374,7 +5374,7 @@ BuildsOperationUploadFolder::Execute()
if (!UnknownChunks.empty())
{
- ZEN_CONSOLE_WARN("{}", BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ true));
+ ZEN_OPERATION_LOG_WARN(m_LogOutput, "{}", BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ true));
}
uint32_t FinalizeBuildPartRetryCount = 5;
@@ -5411,7 +5411,9 @@ BuildsOperationUploadFolder::Execute()
else
{
UnknownChunks = RetryUnknownChunks;
- ZEN_CONSOLE_WARN("{}", BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ FinalizeBuildPartRetryCount != 0));
+ ZEN_OPERATION_LOG_WARN(m_LogOutput,
+ "{}",
+ BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ FinalizeBuildPartRetryCount != 0));
}
}
@@ -6896,11 +6898,12 @@ BuildsOperationValidateBuildPart::Execute()
auto _ = MakeGuard([&]() {
if (!m_Options.IsQuiet)
{
- ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}",
- m_BuildId,
- m_BuildPartId,
- m_BuildPartName,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ ZEN_OPERATION_LOG_INFO(m_LogOutput,
+ "Validated build part {}/{} ('{}') in {}",
+ m_BuildId,
+ m_BuildPartId,
+ m_BuildPartName,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
}
});
@@ -6928,7 +6931,11 @@ BuildsOperationValidateBuildPart::Execute()
m_ValidateStats.BuildPartSize = BuildPart.GetSize();
if (!m_Options.IsQuiet)
{
- ZEN_CONSOLE("Validating build part {}/{} ({})", m_BuildId, m_BuildPartId, NiceBytes(BuildPart.GetSize()));
+ ZEN_OPERATION_LOG_INFO(m_LogOutput,
+ "Validating build part {}/{} ({})",
+ m_BuildId,
+ m_BuildPartId,
+ NiceBytes(BuildPart.GetSize()));
}
std::vector<IoHash> ChunkAttachments;
if (const CbObjectView ChunkAttachmentsView = BuildPart["chunkAttachments"sv].AsObjectView())
diff --git a/src/zenremotestore/builds/buildstorageutil.cpp b/src/zenremotestore/builds/buildstorageutil.cpp
index 3af323628..15ece2edd 100644
--- a/src/zenremotestore/builds/buildstorageutil.cpp
+++ b/src/zenremotestore/builds/buildstorageutil.cpp
@@ -32,7 +32,8 @@ namespace {
} // namespace
BuildStorageResolveResult
-ResolveBuildStorage(const HttpClientSettings& ClientSettings,
+ResolveBuildStorage(OperationLogOutput& Output,
+ const HttpClientSettings& ClientSettings,
std::string_view Host,
std::string_view OverrideHost,
std::string_view ZenCacheHost,
@@ -75,9 +76,10 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings,
{
if (Verbose)
{
- ZEN_CONSOLE("Querying servers at '{}/api/v1/status/servers'\n Connection settings:{}",
- DiscoveryHost,
- ConnectionSettingsToString(ClientSettings));
+ ZEN_OPERATION_LOG_INFO(Output,
+ "Querying servers at '{}/api/v1/status/servers'\n Connection settings:{}",
+ DiscoveryHost,
+ ConnectionSettingsToString(ClientSettings));
}
DiscoveryResponse = DiscoverJupiterEndpoints(DiscoveryHost, ClientSettings);
@@ -87,14 +89,14 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings,
{
if (Verbose)
{
- ZEN_CONSOLE("Testing server endpoint at '{}/health/live'. Assume http2: {}", OverrideHost, HostAssumeHttp2);
+ ZEN_OPERATION_LOG_INFO(Output, "Testing server endpoint at '{}/health/live'. Assume http2: {}", OverrideHost, HostAssumeHttp2);
}
if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(OverrideHost, HostAssumeHttp2, ClientSettings.Verbose);
TestResult.Success)
{
if (Verbose)
{
- ZEN_CONSOLE("Server endpoint at '{}/api/v1/status/servers' succeeded", OverrideHost);
+ ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", OverrideHost);
}
HostUrl = OverrideHost;
HostName = GetHostNameFromUrl(OverrideHost);
@@ -117,16 +119,17 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings,
{
if (Verbose)
{
- ZEN_CONSOLE("Testing server endpoint at '{}/health/live'. Assume http2: {}",
- ServerEndpoint.BaseUrl,
- ServerEndpoint.AssumeHttp2);
+ ZEN_OPERATION_LOG_INFO(Output,
+ "Testing server endpoint at '{}/health/live'. Assume http2: {}",
+ ServerEndpoint.BaseUrl,
+ ServerEndpoint.AssumeHttp2);
}
if (JupiterEndpointTestResult TestResult =
TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2, ClientSettings.Verbose);
TestResult.Success)
{
- ZEN_CONSOLE("Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl);
+ ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl);
HostUrl = ServerEndpoint.BaseUrl;
HostAssumeHttp2 = ServerEndpoint.AssumeHttp2;
@@ -135,7 +138,10 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings,
}
else
{
- ZEN_DEBUG("Unable to reach host {}. Reason: {}", ServerEndpoint.BaseUrl, TestResult.FailureReason);
+ ZEN_OPERATION_LOG_DEBUG(Output,
+ "Unable to reach host {}. Reason: {}",
+ ServerEndpoint.BaseUrl,
+ TestResult.FailureReason);
}
}
}
@@ -156,16 +162,17 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings,
{
if (Verbose)
{
- ZEN_CONSOLE("Testing cache endpoint at '{}/status/builds'. Assume http2: {}",
- CacheEndpoint.BaseUrl,
- CacheEndpoint.AssumeHttp2);
+ ZEN_OPERATION_LOG_INFO(Output,
+ "Testing cache endpoint at '{}/status/builds'. Assume http2: {}",
+ CacheEndpoint.BaseUrl,
+ CacheEndpoint.AssumeHttp2);
}
if (ZenCacheEndpointTestResult TestResult =
TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2, ClientSettings.Verbose);
TestResult.Success)
{
- ZEN_CONSOLE("Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl);
+ ZEN_OPERATION_LOG_INFO(Output, "Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl);
CacheUrl = CacheEndpoint.BaseUrl;
CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2;
@@ -201,7 +208,7 @@ ResolveBuildStorage(const HttpClientSettings& ClientSettings,
{
if (Verbose)
{
- ZEN_CONSOLE("Testing cache endpoint at '{}/status/builds'. Assume http2: {}", ZenCacheHost, false);
+ ZEN_OPERATION_LOG_INFO(Output, "Testing cache endpoint at '{}/status/builds'. Assume http2: {}", ZenCacheHost, false);
}
if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenCacheHost, /*AssumeHttp2*/ false, ClientSettings.Verbose);
TestResult.Success)
@@ -239,7 +246,7 @@ GetBlockDescriptions(OperationLogOutput& Output,
if (!IsQuiet)
{
- ZEN_CONSOLE("Fetching metadata for {} blocks", BlockRawHashes.size());
+ ZEN_OPERATION_LOG_INFO(Output, "Fetching metadata for {} blocks", BlockRawHashes.size());
}
Stopwatch GetBlockMetadataTimer;
diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h
index 7ac1f78d4..ab3037c89 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h
@@ -31,7 +31,8 @@ enum class ZenCacheResolveMode
All
};
-BuildStorageResolveResult ResolveBuildStorage(const HttpClientSettings& ClientSettings,
+BuildStorageResolveResult ResolveBuildStorage(OperationLogOutput& Output,
+ const HttpClientSettings& ClientSettings,
std::string_view Host,
std::string_view OverrideHost,
std::string_view ZenCacheHost,
diff --git a/src/zenremotestore/include/zenremotestore/operationlogoutput.h b/src/zenremotestore/include/zenremotestore/operationlogoutput.h
index 49649fc76..9693e69cf 100644
--- a/src/zenremotestore/include/zenremotestore/operationlogoutput.h
+++ b/src/zenremotestore/include/zenremotestore/operationlogoutput.h
@@ -57,6 +57,10 @@ public:
virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) = 0;
};
+struct LoggerRef;
+
+OperationLogOutput* CreateStandardLogOutput(LoggerRef& Log);
+
#define ZEN_OPERATION_LOG(OutputTarget, InLevel, fmtstr, ...) \
do \
{ \
diff --git a/src/zenremotestore/operationlogoutput.cpp b/src/zenremotestore/operationlogoutput.cpp
index 8d8a6dc78..0837ed716 100644
--- a/src/zenremotestore/operationlogoutput.cpp
+++ b/src/zenremotestore/operationlogoutput.cpp
@@ -4,6 +4,100 @@
#include <zencore/logging.h>
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
namespace zen {
+class StandardLogOutput;
+
+class StandardLogOutputProgressBar : public OperationLogOutput::ProgressBar
+{
+public:
+ StandardLogOutputProgressBar(StandardLogOutput& Output, std::string_view InSubTask) : m_Output(Output), m_SubTask(InSubTask) {}
+
+ virtual void UpdateState(const State& NewState, bool DoLinebreak) override;
+ virtual void Finish() override;
+
+private:
+ StandardLogOutput& m_Output;
+ std::string m_SubTask;
+ State m_State;
+};
+
+class StandardLogOutput : public OperationLogOutput
+{
+public:
+ StandardLogOutput(LoggerRef& Log) : m_Log(Log) {}
+ virtual void EmitLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) override
+ {
+ if (m_Log.ShouldLog(LogLevel))
+ {
+ fmt::basic_memory_buffer<char, 250> MessageBuffer;
+ fmt::vformat_to(fmt::appender(MessageBuffer), Format, Args);
+ ZEN_LOG(m_Log, LogLevel, "{}", std::string_view(MessageBuffer.data(), MessageBuffer.size()));
+ }
+ }
+
+ virtual void SetLogOperationName(std::string_view Name) override
+ {
+ m_LogOperationName = Name;
+ ZEN_OPERATION_LOG_INFO(*this, "{}", m_LogOperationName);
+ }
+ virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) override
+ {
+ const size_t PercentDone = StepCount > 0u ? gsl::narrow<uint8_t>((100 * StepIndex) / StepCount) : 0u;
+ ZEN_OPERATION_LOG_INFO(*this, "{}: {}%", m_LogOperationName, PercentDone);
+ }
+ virtual uint32_t GetProgressUpdateDelayMS() override { return 2000; }
+ virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) override
+ {
+ return new StandardLogOutputProgressBar(*this, InSubTask);
+ }
+
+private:
+ LoggerRef m_Log;
+ std::string m_LogOperationName;
+};
+
+void
+StandardLogOutputProgressBar::UpdateState(const State& NewState, bool DoLinebreak)
+{
+ ZEN_UNUSED(DoLinebreak);
+ const size_t PercentDone =
+ NewState.TotalCount > 0u ? gsl::narrow<uint8_t>((100 * (NewState.TotalCount - NewState.RemainingCount)) / NewState.TotalCount) : 0u;
+ std::string Task = NewState.Task;
+ switch (NewState.Status)
+ {
+ case State::EStatus::Aborted:
+ Task = "Aborting";
+ break;
+ case State::EStatus::Paused:
+ Task = "Paused";
+ break;
+ default:
+ break;
+ }
+ ZEN_OPERATION_LOG_INFO(m_Output, "{}: {}%{}", Task, PercentDone, NewState.Details.empty() ? "" : fmt::format(" {}", NewState.Details));
+ m_State = NewState;
+}
+void
+StandardLogOutputProgressBar::Finish()
+{
+ if (m_State.RemainingCount > 0)
+ {
+ State NewState = m_State;
+ NewState.RemainingCount = 0;
+ NewState.Details = "";
+ UpdateState(NewState, /*DoLinebreak*/ true);
+ }
+}
+
+OperationLogOutput*
+CreateStandardLogOutput(LoggerRef& Log)
+{
+ return new StandardLogOutput(Log);
+}
+
} // namespace zen
diff --git a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp
index bb3de738a..7a03ce50a 100644
--- a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp
+++ b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp
@@ -11,6 +11,7 @@
#include <zenremotestore/builds/buildstoragecache.h>
#include <zenremotestore/builds/buildstorageutil.h>
#include <zenremotestore/builds/jupiterbuildstorage.h>
+#include <zenremotestore/operationlogoutput.h>
#include <numeric>
@@ -668,7 +669,10 @@ CreateJupiterBuildsRemoteStore(LoggerRef InLog,
.AllowResume = true,
.RetryCount = 2};
- ResolveRes = ResolveBuildStorage(ClientSettings, Host, OverrideUrl, ZenHost, ZenCacheResolveMode::Discovery, /*Verbose*/ false);
+ std::unique_ptr<OperationLogOutput> Output(CreateStandardLogOutput(InLog));
+
+ ResolveRes =
+ ResolveBuildStorage(*Output, ClientSettings, Host, OverrideUrl, ZenHost, ZenCacheResolveMode::Discovery, /*Verbose*/ false);
}
HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient",