aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2025-08-26 14:53:31 +0200
committerStefan Boberg <[email protected]>2025-08-26 14:53:31 +0200
commit71b9f80cc72fae4cf0476bfca1646ec8311c9272 (patch)
tree59c1e0b8911814445e6b78005a43207593393323 /src
parentdefault Sentry to off in debug (diff)
parentMerge branch 'main' into sb/zen-master (diff)
downloadzen-71b9f80cc72fae4cf0476bfca1646ec8311c9272.tar.xz
zen-71b9f80cc72fae4cf0476bfca1646ec8311c9272.zip
Merge branch 'sb/zen-master' of https://github.ol.epicgames.net/ue-foundation/zen into sb/zen-master
Diffstat (limited to 'src')
-rw-r--r--src/zen/cmds/admin_cmd.cpp32
-rw-r--r--src/zen/cmds/bench_cmd.cpp10
-rw-r--r--src/zen/cmds/builds_cmd.cpp1666
-rw-r--r--src/zen/cmds/cache_cmd.cpp24
-rw-r--r--src/zen/cmds/copy_cmd.cpp4
-rw-r--r--src/zen/cmds/dedup_cmd.cpp6
-rw-r--r--src/zen/cmds/info_cmd.cpp4
-rw-r--r--src/zen/cmds/print_cmd.cpp28
-rw-r--r--src/zen/cmds/projectstore_cmd.cpp155
-rw-r--r--src/zen/cmds/rpcreplay_cmd.cpp2
-rw-r--r--src/zen/cmds/serve_cmd.cpp12
-rw-r--r--src/zen/cmds/status_cmd.cpp7
-rw-r--r--src/zen/cmds/top_cmd.cpp4
-rw-r--r--src/zen/cmds/trace_cmd.cpp6
-rw-r--r--src/zen/cmds/up_cmd.cpp41
-rw-r--r--src/zen/cmds/version_cmd.cpp2
-rw-r--r--src/zen/cmds/wipe_cmd.cpp15
-rw-r--r--src/zen/cmds/workspaces_cmd.cpp18
-rw-r--r--src/zen/zen.cpp59
-rw-r--r--src/zencore/include/zencore/logging.h1
-rw-r--r--src/zencore/logging.cpp23
-rw-r--r--src/zenserver/cache/httpstructuredcache.cpp286
-rw-r--r--src/zenserver/cache/httpstructuredcache.h5
-rw-r--r--src/zenserver/config.cpp49
-rw-r--r--src/zenserver/config.h1
-rw-r--r--src/zenserver/diag/logging.cpp1
-rw-r--r--src/zenserver/main.cpp61
-rw-r--r--src/zenserver/projectstore/httpprojectstore.cpp167
-rw-r--r--src/zenserver/projectstore/httpprojectstore.h7
-rw-r--r--src/zenserver/projectstore/projectstore.cpp169
-rw-r--r--src/zenserver/projectstore/projectstore.h8
-rw-r--r--src/zenserver/upstream/upstreamcache.cpp38
-rw-r--r--src/zenserver/upstream/upstreamcache.h3
-rw-r--r--src/zenserver/zenserver.cpp57
-rw-r--r--src/zenserver/zenserver.h28
-rw-r--r--src/zenstore/cache/cacherpc.cpp54
-rw-r--r--src/zenstore/include/zenstore/cache/cacherpc.h8
-rw-r--r--src/zenstore/include/zenstore/cache/upstreamcacheclient.h2
-rw-r--r--src/zenutil/include/zenutil/logging.h1
-rw-r--r--src/zenutil/logging.cpp4
40 files changed, 1400 insertions, 1668 deletions
diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp
index fe2bbbdc7..dca6da6c4 100644
--- a/src/zen/cmds/admin_cmd.cpp
+++ b/src/zen/cmds/admin_cmd.cpp
@@ -51,17 +51,17 @@ ScrubCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (HttpClient::Response Response = Http.Post("/admin/scrub"sv, /* headers */ HttpClient::KeyValueMap{}, Params))
{
- ZEN_CONSOLE("scrub started OK: {}", Response.ToText());
+ ZEN_CONSOLE("Scrub started OK: {}", Response.ToText());
return 0;
}
else if (int StatusCode = (int)Response.StatusCode)
{
- ZEN_ERROR("scrub start failed: {}: {} ({})", StatusCode, ReasonStringForHttpResultCode(StatusCode), Response.ToText());
+ ZEN_CONSOLE_ERROR("Scrub start failed: {}: {} ({})", StatusCode, ReasonStringForHttpResultCode(StatusCode), Response.ToText());
}
else
{
- ZEN_ERROR("scrub start failed: {}", Response.ToText());
+ ZEN_CONSOLE_ERROR("Scrub start failed: {}", Response.ToText());
}
return 1;
@@ -259,11 +259,11 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (Result.status_code)
{
- ZEN_ERROR("GC start failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
+ ZEN_CONSOLE_ERROR("GC start failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
}
else
{
- ZEN_ERROR("GC start failed: {}", Result.error.message);
+ ZEN_CONSOLE_ERROR("GC start failed: {}", Result.error.message);
}
return 1;
@@ -315,11 +315,11 @@ GcStatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (Result.status_code)
{
- ZEN_ERROR("GC status failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
+ ZEN_CONSOLE_ERROR("GC status failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
}
else
{
- ZEN_ERROR("GC status failed: {}", Result.error.message);
+ ZEN_CONSOLE_ERROR("GC status failed: {}", Result.error.message);
}
return 1;
@@ -369,11 +369,11 @@ GcStopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (Result.status_code)
{
- ZEN_ERROR("GC status failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
+ ZEN_CONSOLE_ERROR("GC status failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
}
else
{
- ZEN_ERROR("GC status failed: {}", Result.error.message);
+ ZEN_CONSOLE_ERROR("GC status failed: {}", Result.error.message);
}
return 1;
@@ -416,7 +416,7 @@ JobCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
if (m_JobId == 0)
{
- ZEN_ERROR("Job id must be given");
+ throw OptionParseException("Job id must be given");
return 1;
}
}
@@ -514,7 +514,7 @@ LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- ZEN_ERROR("Invalid value for parameter 'cache-write-log'. Use 'enable' or 'disable'");
+ ZEN_CONSOLE_ERROR("Invalid value for parameter 'cache-write-log'. Use 'enable' or 'disable'");
return 1;
}
}
@@ -531,7 +531,7 @@ LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- ZEN_ERROR("Invalid value for parameter 'cache-access-log'. Use 'enable' or 'disable'");
+ ZEN_CONSOLE_ERROR("Invalid value for parameter 'cache-access-log'. Use 'enable' or 'disable'");
return 1;
}
}
@@ -551,12 +551,12 @@ LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
auto CopyLog = [](std::string_view SourceName, std::string_view SourcePath, std::string_view TargetPath) -> bool {
if (SourcePath.empty())
{
- ZEN_ERROR("Failed to retrieve {} log path", SourceName);
+ ZEN_CONSOLE_ERROR("Failed to retrieve {} log path", SourceName);
return false;
}
if (!CopyFile(SourcePath, TargetPath, {}))
{
- ZEN_ERROR("Failed to copy {} log file {} to output file '{}'", SourceName, SourcePath, TargetPath);
+ ZEN_CONSOLE_ERROR("Failed to copy {} log file {} to output file '{}'", SourceName, SourcePath, TargetPath);
return false;
}
return true;
@@ -642,11 +642,11 @@ FlushCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else if (int StatusCode = (int)Response.StatusCode)
{
- ZEN_ERROR("flush failed: {}: {} ({})", StatusCode, ReasonStringForHttpResultCode(StatusCode), Response.ToText());
+ ZEN_CONSOLE_ERROR("Flush failed: {}: {} ({})", StatusCode, ReasonStringForHttpResultCode(StatusCode), Response.ToText());
}
else
{
- ZEN_ERROR("flush failed: {}", Response.ToText());
+ ZEN_CONSOLE_ERROR("Flush failed: {}", Response.ToText());
}
return 1;
diff --git a/src/zen/cmds/bench_cmd.cpp b/src/zen/cmds/bench_cmd.cpp
index 86b82d838..d904a51f6 100644
--- a/src/zen/cmds/bench_cmd.cpp
+++ b/src/zen/cmds/bench_cmd.cpp
@@ -50,11 +50,11 @@ BenchCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
catch (const zen::bench::util::elevation_required_exception&)
{
- ZEN_CONSOLE("purging standby lists requires elevation. Will try launch as elevated process");
+ ZEN_CONSOLE_WARN("Purging standby lists requires elevation. Will try launch as elevated process");
}
catch (const std::exception& Ex)
{
- ZEN_CONSOLE("ERROR: {}", Ex.what());
+ ZEN_CONSOLE_ERROR("{}", Ex.what());
}
if (!Ok && !m_SingleProcess)
@@ -79,20 +79,20 @@ BenchCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- ZEN_CONSOLE("ERROR: Elevated child process failed with return code {}", ExitCode);
+ ZEN_CONSOLE_ERROR("Elevated child process failed with return code {}", ExitCode);
}
}
}
catch (const std::exception& Ex)
{
- ZEN_CONSOLE("ERROR: {}", Ex.what());
+ ZEN_CONSOLE_ERROR("{}", Ex.what());
}
}
if (Ok)
{
// TODO: could also add reporting on just how much memory was purged
- ZEN_CONSOLE("purged standby lists! (took {})", zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ ZEN_CONSOLE("Purged standby lists! (took {})", zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
}
}
#endif
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 8692e5941..2c6b8e50e 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -258,7 +258,7 @@ namespace {
}
catch (const std::exception& Ex)
{
- ZEN_ERROR("ZenState::~ZenState threw exception: {}", Ex.what());
+ ZEN_CONSOLE_ERROR("ZenState::~ZenState threw exception: {}", Ex.what());
}
}
@@ -469,7 +469,7 @@ namespace {
ZEN_ASSERT_SLOW(IsFile(SourcePath));
if (Retries > 5)
{
- ZEN_CONSOLE("Unable to overwrite file {} ({}: {}), retrying...", TargetPath, Ec.value(), Ec.message());
+ ZEN_CONSOLE_WARN("Unable to overwrite file {} ({}: {}), retrying...", TargetPath, Ec.value(), Ec.message());
}
Sleep(50 + int(Retries * 150));
Ec.clear();
@@ -701,7 +701,7 @@ namespace {
}
catch (const std::exception& Ex)
{
- ZEN_WARN("Failed removing file {}. Reason: {}", FilePath, Ex.what());
+ ZEN_CONSOLE_WARN("Failed removing file {}. Reason: {}", FilePath, Ex.what());
CleanWipe = false;
}
}
@@ -784,7 +784,7 @@ namespace {
}
catch (const std::exception& Ex)
{
- ZEN_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what());
+ ZEN_CONSOLE_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what());
CleanWipe = false;
}
@@ -1569,7 +1569,7 @@ namespace {
}
catch (const std::exception& Ex)
{
- ZEN_CONSOLE("Unable to read local state: ", Ex.what());
+ ZEN_CONSOLE_WARN("Unable to read local state: ", Ex.what());
return false;
}
}
@@ -1707,7 +1707,7 @@ namespace {
}
else
{
- ZEN_WARN("Invalid download state file at {}. '{}'", EntryPath, JsonError);
+ ZEN_CONSOLE_WARN("Invalid download state file at {}. '{}'", EntryPath, JsonError);
DeleteEntry = true;
}
}
@@ -2927,8 +2927,8 @@ namespace {
}
else
{
- ZEN_CONSOLE(
- "Warning: Build blob {} was reported as needed for upload but it was reported as existing at the start of the "
+ ZEN_CONSOLE_WARN(
+ "Build blob {} was reported as needed for upload but it was reported as existing at the start of the "
"operation. Treating it as a transient inconsistent issue and will attempt to retry finalization",
RawHash);
}
@@ -3570,8 +3570,8 @@ namespace {
}
else if (AllowMultiparts)
{
- ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'",
- NiceBytes(Result.PreferredMultipartChunkSize));
+ ZEN_CONSOLE_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'",
+ NiceBytes(Result.PreferredMultipartChunkSize));
}
}
@@ -4765,7 +4765,7 @@ namespace {
for (const std::string& Error : Errors)
{
- ZEN_CONSOLE("{}", Error);
+ ZEN_CONSOLE_ERROR("{}", Error);
}
if (!Errors.empty())
{
@@ -5671,7 +5671,7 @@ namespace {
}
catch (const std::exception& Ex)
{
- ZEN_CONSOLE("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what());
+ ZEN_CONSOLE_WARN("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what());
}
return HasLocalState;
}
@@ -8194,7 +8194,7 @@ namespace {
// Clean target folder
if (!CleanDirectory(Path, DefaultExcludeFolders))
{
- ZEN_WARN("Some files in {} could not be removed", Path);
+ ZEN_CONSOLE_WARN("Some files in {} could not be removed", Path);
}
RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs();
}
@@ -8750,7 +8750,7 @@ namespace {
ChunkBlockDescription Description = ParseChunkBlockDescription(CacheBlockMetadata);
if (Description.BlockHash == IoHash::Zero)
{
- ZEN_WARN("Unexpected/invalid block metadata received from remote cache, skipping block");
+ ZEN_CONSOLE_WARN("Unexpected/invalid block metadata received from remote cache, skipping block");
}
else
{
@@ -8787,7 +8787,7 @@ namespace {
if (Description.BlockHash == IoHash::Zero)
{
- ZEN_WARN("Unexpected/invalid block metadata received from remote store, skipping block");
+ ZEN_CONSOLE_WARN("Unexpected/invalid block metadata received from remote store, skipping block");
}
else
{
@@ -8843,7 +8843,7 @@ namespace {
BlockRawHashes.size() - OutBlockDescriptions.size());
if (AttemptFallback)
{
- ZEN_CONSOLE("{} Attemping fallback options.", ErrorDescription);
+ ZEN_CONSOLE_WARN("{} Attemping fallback options.", ErrorDescription);
std::vector<ChunkBlockDescription> AugmentedBlockDescriptions;
AugmentedBlockDescriptions.reserve(BlockRawHashes.size());
std::vector<ChunkBlockDescription> FoundBlocks =
@@ -9465,7 +9465,7 @@ namespace {
{
if (!ChunkController && !IsQuiet)
{
- ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default");
+ ZEN_CONSOLE_WARN("Unspecified chunking algorith, using default");
ChunkController = CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{});
}
@@ -10576,7 +10576,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456";
if (!IsQuiet)
{
- ZEN_CONSOLE("Warning: Using default encryption key");
+ ZEN_CONSOLE_WARN("Using default encryption key");
}
}
@@ -10585,7 +10585,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
m_EncryptionIV = "0123456789abcdef";
if (!IsQuiet)
{
- ZEN_CONSOLE("Warning: Using default encryption initialization vector");
+ ZEN_CONSOLE_WARN("Using default encryption initialization vector");
}
}
@@ -10897,7 +10897,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
if (m_ZenCacheHost.empty() && !IsQuiet)
{
- ZEN_CONSOLE("Warning: Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host);
+ ZEN_CONSOLE_WARN("Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host);
}
}
}
@@ -10917,7 +10917,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- ZEN_WARN("Unable to reach cache host {}. Reason: {}", m_ZenCacheHost, TestResult.second);
+ ZEN_CONSOLE_WARN("Unable to reach cache host {}. Reason: {}", m_ZenCacheHost, TestResult.second);
}
}
}
@@ -11163,163 +11163,73 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
BoostWorkerThreads = m_BoostWorkerThreads;
UseSparseFiles = m_UseSparseFiles;
- try
+ if (SubOption == &m_ListNamespacesOptions)
{
- if (SubOption == &m_ListNamespacesOptions)
+ if (!m_ListResultPath.empty())
{
- if (!m_ListResultPath.empty())
+ if (!IsQuiet)
{
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Running {}: {} (pid {})",
- GetRunningExecutablePath(),
- ZEN_CFG_VERSION_BUILD_STRING_FULL,
- GetCurrentProcessId());
- }
+ ZEN_CONSOLE("Running {}: {} (pid {})",
+ GetRunningExecutablePath(),
+ ZEN_CFG_VERSION_BUILD_STRING_FULL,
+ GetCurrentProcessId());
}
+ }
- BuildStorage::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty()
- ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName
- : MakeSafeAbsolutePath(m_ZenFolderPath);
- CreateDirectories(ZenFolderPath);
- auto _ = MakeGuard([ZenFolderPath]() {
- if (CleanDirectory(ZenFolderPath, {}))
- {
- std::error_code DummyEc;
- RemoveDir(ZenFolderPath, DummyEc);
- }
- });
-
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(ZenFolderPath),
- /*RequriesNamespace*/ false,
- /*RequireBucket*/ false);
+ BuildStorage::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive);
- ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
- if (m_ListResultPath.empty())
+ const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty()
+ ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName
+ : MakeSafeAbsolutePath(m_ZenFolderPath);
+ CreateDirectories(ZenFolderPath);
+ auto _ = MakeGuard([ZenFolderPath]() {
+ if (CleanDirectory(ZenFolderPath, {}))
{
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- ZEN_CONSOLE("{}", SB.ToView());
- }
- else
- {
- std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath);
- if (ToLower(ListResultPath.extension().string()) == ".cbo")
- {
- MemoryView ResponseView = Response.GetView();
- WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
- }
- else
- {
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
- }
+ std::error_code DummyEc;
+ RemoveDir(ZenFolderPath, DummyEc);
}
+ });
- return 0;
- }
+ StorageInstance Storage = CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(ZenFolderPath),
+ /*RequriesNamespace*/ false,
+ /*RequireBucket*/ false);
- if (SubOption == &m_ListOptions)
+ CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive);
+ ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
+ if (m_ListResultPath.empty())
{
- MakeSafeAbsolutePathÍnPlace(m_ListQueryPath);
- MakeSafeAbsolutePathÍnPlace(m_ListResultPath);
-
- if (!m_ListResultPath.empty())
- {
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Running {}: {} (pid {})",
- GetRunningExecutablePath(),
- ZEN_CFG_VERSION_BUILD_STRING_FULL,
- GetCurrentProcessId());
- }
- }
- CbObject QueryObject;
- if (m_ListQueryPath.empty())
+ ExtendableStringBuilder<1024> SB;
+ CompactBinaryToJson(Response.GetView(), SB);
+ ZEN_CONSOLE("{}", SB.ToView());
+ }
+ else
+ {
+ std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath);
+ if (ToLower(ListResultPath.extension().string()) == ".cbo")
{
- CbObjectWriter QueryWriter;
- QueryWriter.BeginObject("query");
- QueryWriter.EndObject(); // query
- QueryObject = QueryWriter.Save();
+ MemoryView ResponseView = Response.GetView();
+ WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
}
else
{
- if (ToLower(m_ListQueryPath.extension().string()) == ".cbo")
- {
- QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_ListQueryPath));
- }
- else
- {
- IoBuffer MetaDataJson = ReadFile(m_ListQueryPath).Flatten();
- std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
- std::string JsonError;
- QueryObject = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
- if (!JsonError.empty())
- {
- throw std::runtime_error(
- fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_ListQueryPath, JsonError));
- }
- }
- }
-
- BuildStorage::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- if (m_ZenFolderPath.empty())
- {
- m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
- }
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
-
- CreateDirectories(m_ZenFolderPath);
- auto _ = MakeGuard([this]() {
- if (CleanDirectory(m_ZenFolderPath, {}))
- {
- std::error_code DummyEc;
- RemoveDir(m_ZenFolderPath, DummyEc);
- }
- });
-
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ false);
-
- CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject);
- ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
- if (m_ListResultPath.empty())
- {
ExtendableStringBuilder<1024> SB;
CompactBinaryToJson(Response.GetView(), SB);
- ZEN_CONSOLE("{}", SB.ToView());
+ WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
}
- else
- {
- if (ToLower(m_ListResultPath.extension().string()) == ".cbo")
- {
- MemoryView ResponseView = Response.GetView();
- WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
- }
- else
- {
- ExtendableStringBuilder<1024> SB;
- CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
- }
- }
-
- return 0;
}
- if (SubOption == &m_UploadOptions)
+ return 0;
+ }
+
+ if (SubOption == &m_ListOptions)
+ {
+ MakeSafeAbsolutePathÍnPlace(m_ListQueryPath);
+ MakeSafeAbsolutePathÍnPlace(m_ListResultPath);
+
+ if (!m_ListResultPath.empty())
{
if (!IsQuiet)
{
@@ -11328,544 +11238,439 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZEN_CFG_VERSION_BUILD_STRING_FULL,
GetCurrentProcessId());
}
-
- ZenState InstanceState;
-
- ParsePath();
-
- BuildStorage::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
-
- if (m_ZenFolderPath.empty())
+ }
+ CbObject QueryObject;
+ if (m_ListQueryPath.empty())
+ {
+ CbObjectWriter QueryWriter;
+ QueryWriter.BeginObject("query");
+ QueryWriter.EndObject(); // query
+ QueryObject = QueryWriter.Save();
+ }
+ else
+ {
+ if (ToLower(m_ListQueryPath.extension().string()) == ".cbo")
{
- m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
+ QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_ListQueryPath));
}
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
-
- CreateDirectories(m_ZenFolderPath);
- auto _ = MakeGuard([this]() {
- if (CleanDirectory(m_ZenFolderPath, {}))
+ else
+ {
+ IoBuffer MetaDataJson = ReadFile(m_ListQueryPath).Flatten();
+ std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
+ std::string JsonError;
+ QueryObject = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
+ if (!JsonError.empty())
{
- std::error_code DummyEc;
- RemoveDir(m_ZenFolderPath, DummyEc);
+ throw std::runtime_error(
+ fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_ListQueryPath, JsonError));
}
- });
+ }
+ }
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true);
+ BuildStorage::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- if (m_BuildPartName.empty())
- {
- m_BuildPartName = m_Path.filename().string();
- }
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
- const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId();
- if (m_BuildId.empty())
- {
- m_BuildId = BuildId.ToString();
- }
- const Oid BuildPartId = m_BuildPartId.empty() ? Oid::NewOid() : ParseBuildPartId();
- if (m_BuildPartId.empty())
+ CreateDirectories(m_ZenFolderPath);
+ auto _ = MakeGuard([this]() {
+ if (CleanDirectory(m_ZenFolderPath, {}))
{
- m_BuildPartId = BuildPartId.ToString();
+ std::error_code DummyEc;
+ RemoveDir(m_ZenFolderPath, DummyEc);
}
+ });
- CbObject MetaData = ParseBuildMetadata();
-
- const std::filesystem::path TempDir = ZenTempFolderPath(m_ZenFolderPath);
-
- m_ManifestPath = RemoveQuotes(m_ManifestPath);
+ StorageInstance Storage = CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_ZenFolderPath),
+ /*RequriesNamespace*/ true,
+ /*RequireBucket*/ false);
- UploadFolder(Storage,
- BuildId,
- BuildPartId,
- m_BuildPartName,
- m_Path,
- TempDir,
- m_ManifestPath,
- m_FindBlockMaxCount,
- m_BlockReuseMinPercentLimit,
- m_AllowMultiparts,
- MetaData,
- m_CreateBuild,
- m_Clean,
- m_PostUploadVerify);
-
- if (false)
- {
- if (!IsQuiet)
- {
- ZEN_CONSOLE(
- "{}:\n"
- "Read: {}\n"
- "Write: {}\n"
- "Requests: {}\n"
- "Avg Request Time: {}\n"
- "Avg I/O Time: {}",
- Storage.StorageName,
- NiceBytes(StorageStats.TotalBytesRead.load()),
- NiceBytes(StorageStats.TotalBytesWritten.load()),
- StorageStats.TotalRequestCount.load(),
- StorageStats.TotalExecutionTimeUs.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0,
- StorageStats.TotalRequestCount.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0);
- }
- }
- return AbortFlag ? 11 : 0;
+ CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject);
+ ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None);
+ if (m_ListResultPath.empty())
+ {
+ ExtendableStringBuilder<1024> SB;
+ CompactBinaryToJson(Response.GetView(), SB);
+ ZEN_CONSOLE("{}", SB.ToView());
}
-
- if (SubOption == &m_DownloadOptions)
+ else
{
- if (!IsQuiet)
+ if (ToLower(m_ListResultPath.extension().string()) == ".cbo")
{
- ZEN_CONSOLE("Running {}: {} (pid {})",
- GetRunningExecutablePath(),
- ZEN_CFG_VERSION_BUILD_STRING_FULL,
- GetCurrentProcessId());
+ MemoryView ResponseView = Response.GetView();
+ WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
}
-
- ZenState InstanceState;
-
- ParsePath();
- ParseFileFilters();
-
- if (m_ZenFolderPath.empty())
+ else
{
- m_ZenFolderPath = m_Path / ZenFolderName;
+ ExtendableStringBuilder<1024> SB;
+ CompactBinaryToJson(Response.GetView(), SB);
+ WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
}
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+ }
- BuildStorage::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ return 0;
+ }
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true);
+ if (SubOption == &m_UploadOptions)
+ {
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ }
- const Oid BuildId = ParseBuildId();
+ ZenState InstanceState;
- if (m_PostDownloadVerify && m_PrimeCacheOnly)
- {
- throw zen::OptionParseException(
- fmt::format("'cache-prime-only' option is not compatible with 'verify' option\n{}", SubOption->help()));
- }
+ ParsePath();
- if (m_Clean && m_PrimeCacheOnly)
- {
- ZEN_WARN("ignoring 'clean' option when 'cache-prime-only' is enabled");
- }
+ BuildStorage::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
+
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
- if (m_AllowPartialBlockRequests && m_PrimeCacheOnly)
+ CreateDirectories(m_ZenFolderPath);
+ auto _ = MakeGuard([this]() {
+ if (CleanDirectory(m_ZenFolderPath, {}))
{
- ZEN_WARN("ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled");
+ std::error_code DummyEc;
+ RemoveDir(m_ZenFolderPath, DummyEc);
}
+ });
- std::vector<Oid> BuildPartIds = ParseBuildPartIds();
- std::vector<std::string> BuildPartNames = ParseBuildPartNames();
+ StorageInstance Storage = CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_ZenFolderPath),
+ /*RequriesNamespace*/ true,
+ /*RequireBucket*/ true);
- // TODO: Add file filters
- DownloadFolder(Storage,
- BuildId,
- BuildPartIds,
- BuildPartNames,
- m_Path,
- m_ZenFolderPath,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests && !m_PrimeCacheOnly,
- m_Clean,
- m_PostDownloadVerify,
- m_PrimeCacheOnly,
- m_EnableScavenging,
- m_IncludeWildcard,
- m_ExcludeWildcard);
+ if (m_BuildPartName.empty())
+ {
+ m_BuildPartName = m_Path.filename().string();
+ }
- return AbortFlag ? 11 : 0;
+ const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId();
+ if (m_BuildId.empty())
+ {
+ m_BuildId = BuildId.ToString();
}
+ const Oid BuildPartId = m_BuildPartId.empty() ? Oid::NewOid() : ParseBuildPartId();
+ if (m_BuildPartId.empty())
+ {
+ m_BuildPartId = BuildPartId.ToString();
+ }
+
+ CbObject MetaData = ParseBuildMetadata();
+
+ const std::filesystem::path TempDir = ZenTempFolderPath(m_ZenFolderPath);
- if (SubOption == &m_LsOptions)
+ m_ManifestPath = RemoveQuotes(m_ManifestPath);
+
+ UploadFolder(Storage,
+ BuildId,
+ BuildPartId,
+ m_BuildPartName,
+ m_Path,
+ TempDir,
+ m_ManifestPath,
+ m_FindBlockMaxCount,
+ m_BlockReuseMinPercentLimit,
+ m_AllowMultiparts,
+ MetaData,
+ m_CreateBuild,
+ m_Clean,
+ m_PostUploadVerify);
+
+ if (false)
{
if (!IsQuiet)
{
- ZEN_CONSOLE("Running {}: {} (pid {})",
- GetRunningExecutablePath(),
- ZEN_CFG_VERSION_BUILD_STRING_FULL,
- GetCurrentProcessId());
- }
-
- ZenState InstanceState;
+ ZEN_CONSOLE(
+ "{}:\n"
+ "Read: {}\n"
+ "Write: {}\n"
+ "Requests: {}\n"
+ "Avg Request Time: {}\n"
+ "Avg I/O Time: {}",
+ Storage.StorageName,
+ NiceBytes(StorageStats.TotalBytesRead.load()),
+ NiceBytes(StorageStats.TotalBytesWritten.load()),
+ StorageStats.TotalRequestCount.load(),
+ StorageStats.TotalExecutionTimeUs.load() > 0
+ ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
+ : 0,
+ StorageStats.TotalRequestCount.load() > 0
+ ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
+ : 0);
+ }
+ }
+ return AbortFlag ? 11 : 0;
+ }
- ParseFileFilters();
+ if (SubOption == &m_DownloadOptions)
+ {
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ }
- if (m_ZenFolderPath.empty())
- {
- m_ZenFolderPath = m_Path / ZenFolderName;
- }
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+ ZenState InstanceState;
- BuildStorage::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ ParsePath();
+ ParseFileFilters();
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true);
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = m_Path / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
- const Oid BuildId = ParseBuildId();
+ BuildStorage::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- std::vector<Oid> BuildPartIds = ParseBuildPartIds();
- std::vector<std::string> BuildPartNames = ParseBuildPartNames();
+ StorageInstance Storage = CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_ZenFolderPath),
+ /*RequriesNamespace*/ true,
+ /*RequireBucket*/ true);
- ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, m_IncludeWildcard, m_ExcludeWildcard);
+ const Oid BuildId = ParseBuildId();
- return AbortFlag ? 11 : 0;
+ if (m_PostDownloadVerify && m_PrimeCacheOnly)
+ {
+ throw zen::OptionParseException(
+ fmt::format("'cache-prime-only' option is not compatible with 'verify' option\n{}", SubOption->help()));
}
- if (SubOption == &m_DiffOptions)
+ if (m_Clean && m_PrimeCacheOnly)
{
- ParsePath();
- ParseDiffPath();
-
- DiffFolders(m_Path, m_DiffPath, m_OnlyChunked);
- return AbortFlag ? 11 : 0;
+ ZEN_CONSOLE_WARN("Ignoring 'clean' option when 'cache-prime-only' is enabled");
}
- if (SubOption == &m_FetchBlobOptions)
+ if (m_AllowPartialBlockRequests && m_PrimeCacheOnly)
{
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Running {}: {} (pid {})",
- GetRunningExecutablePath(),
- ZEN_CFG_VERSION_BUILD_STRING_FULL,
- GetCurrentProcessId());
- }
+ ZEN_CONSOLE_WARN("Ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled");
+ }
- BuildStorage::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ std::vector<Oid> BuildPartIds = ParseBuildPartIds();
+ std::vector<std::string> BuildPartNames = ParseBuildPartNames();
- if (m_ZenFolderPath.empty())
- {
- m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
- }
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+ // TODO: Add file filters
+ DownloadFolder(Storage,
+ BuildId,
+ BuildPartIds,
+ BuildPartNames,
+ m_Path,
+ m_ZenFolderPath,
+ m_SystemRootDir,
+ m_AllowMultiparts,
+ m_AllowPartialBlockRequests && !m_PrimeCacheOnly,
+ m_Clean,
+ m_PostDownloadVerify,
+ m_PrimeCacheOnly,
+ m_EnableScavenging,
+ m_IncludeWildcard,
+ m_ExcludeWildcard);
- CreateDirectories(m_ZenFolderPath);
- auto _ = MakeGuard([this]() {
- if (CleanDirectory(m_ZenFolderPath, {}))
- {
- std::error_code DummyEc;
- RemoveDir(m_ZenFolderPath, DummyEc);
- }
- });
-
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true);
+ return AbortFlag ? 11 : 0;
+ }
- IoHash BlobHash = ParseBlobHash();
+ if (SubOption == &m_LsOptions)
+ {
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
+ }
- const Oid BuildId = Oid::FromHexString(m_BuildId);
+ ZenState InstanceState;
- uint64_t CompressedSize;
- uint64_t DecompressedSize;
- ValidateBlob(*Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize);
- if (AbortFlag)
- {
- return 11;
- }
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes",
- BlobHash,
- CompressedSize,
- DecompressedSize);
- }
- return 0;
- }
+ ParseFileFilters();
- if (SubOption == &m_ValidateBuildPartOptions)
+ if (m_ZenFolderPath.empty())
{
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Running {}: {} (pid {})",
- GetRunningExecutablePath(),
- ZEN_CFG_VERSION_BUILD_STRING_FULL,
- GetCurrentProcessId());
- }
+ m_ZenFolderPath = m_Path / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
- ZenState InstanceState;
+ BuildStorage::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- BuildStorage::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ StorageInstance Storage = CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_ZenFolderPath),
+ /*RequriesNamespace*/ true,
+ /*RequireBucket*/ true);
- if (m_ZenFolderPath.empty())
- {
- m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
- }
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
-
- CreateDirectories(m_ZenFolderPath);
- auto _ = MakeGuard([this]() {
- if (CleanDirectory(m_ZenFolderPath, {}))
- {
- std::error_code DummyEc;
- RemoveDir(m_ZenFolderPath, DummyEc);
- }
- });
+ const Oid BuildId = ParseBuildId();
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true);
+ std::vector<Oid> BuildPartIds = ParseBuildPartIds();
+ std::vector<std::string> BuildPartNames = ParseBuildPartNames();
- Oid BuildId = ParseBuildId();
+ ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, m_IncludeWildcard, m_ExcludeWildcard);
- if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
- {
- throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", SubOption->help()));
- }
+ return AbortFlag ? 11 : 0;
+ }
- const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId();
+ if (SubOption == &m_DiffOptions)
+ {
+ ParsePath();
+ ParseDiffPath();
- ValidateStatistics ValidateStats;
- DownloadStatistics DownloadStats;
- ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats);
+ DiffFolders(m_Path, m_DiffPath, m_OnlyChunked);
+ return AbortFlag ? 11 : 0;
+ }
- return AbortFlag ? 13 : 0;
+ if (SubOption == &m_FetchBlobOptions)
+ {
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
}
- if (SubOption == &m_MultiTestDownloadOptions)
- {
- m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
- CreateDirectories(m_SystemRootDir);
- CleanDirectory(m_SystemRootDir, {});
- auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); });
+ BuildStorage::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- ParsePath();
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
- if (m_ZenFolderPath.empty())
+ CreateDirectories(m_ZenFolderPath);
+ auto _ = MakeGuard([this]() {
+ if (CleanDirectory(m_ZenFolderPath, {}))
{
- m_ZenFolderPath = m_Path / ZenFolderName;
+ std::error_code DummyEc;
+ RemoveDir(m_ZenFolderPath, DummyEc);
}
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+ });
- BuildStorage::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ StorageInstance Storage = CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_ZenFolderPath),
+ /*RequriesNamespace*/ true,
+ /*RequireBucket*/ true);
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true);
+ IoHash BlobHash = ParseBlobHash();
- Stopwatch Timer;
- for (const std::string& BuildIdString : m_BuildIds)
- {
- Oid BuildId = Oid::FromHexString(RemoveQuotes(BuildIdString));
- if (BuildId == Oid::Zero)
- {
- throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, SubOption->help()));
- }
- DownloadFolder(Storage,
- BuildId,
- {},
- {},
- m_Path,
- m_ZenFolderPath,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- BuildIdString == m_BuildIds.front(),
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
- if (AbortFlag)
- {
- ZEN_CONSOLE("Download cancelled");
- return 11;
- }
- if (!IsQuiet)
- {
- ZEN_CONSOLE("\n");
- }
- }
- if (!IsQuiet)
- {
- ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
- }
- return 0;
- }
+ const Oid BuildId = Oid::FromHexString(m_BuildId);
- auto ParseZenProcessId = [&]() {
- if (m_ZenProcessId == -1)
- {
- const std::filesystem::path RunningExecutablePath = GetRunningExecutablePath();
- ProcessHandle RunningProcess;
- std::error_code Ec = FindProcess(RunningExecutablePath, RunningProcess, /*IncludeSelf*/ false);
- if (Ec)
- {
- throw std::runtime_error(
- fmt::format("Failed finding process running '{}', reason: '{}'", RunningExecutablePath, Ec.message()));
- }
- if (!RunningProcess.IsValid())
- {
- throw zen::OptionParseException(
- fmt::format("Unable to find a running instance of the zen executable '{}'", RunningExecutablePath));
- }
- m_ZenProcessId = RunningProcess.Pid();
- }
- };
-
- if (SubOption == &m_PauseOptions)
+ uint64_t CompressedSize;
+ uint64_t DecompressedSize;
+ ValidateBlob(*Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize);
+ if (AbortFlag)
{
- ParseZenProcessId();
- ZenState RunningState(m_ZenProcessId);
- RunningState.StateData().Pause.store(true);
- return 0;
+ return 11;
}
-
- if (SubOption == &m_ResumeOptions)
+ if (!IsQuiet)
{
- ParseZenProcessId();
- ZenState RunningState(m_ZenProcessId);
- RunningState.StateData().Pause.store(false);
- return 0;
+ ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes",
+ BlobHash,
+ CompressedSize,
+ DecompressedSize);
}
+ return 0;
+ }
- if (SubOption == &m_AbortOptions)
+ if (SubOption == &m_ValidateBuildPartOptions)
+ {
+ if (!IsQuiet)
{
- ParseZenProcessId();
- ZenState RunningState(m_ZenProcessId);
- RunningState.StateData().Abort.store(true);
- return 0;
+ ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId());
}
- if (SubOption == &m_TestOptions)
- {
- m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
- CreateDirectories(m_SystemRootDir);
- CleanDirectory(m_SystemRootDir, {});
- auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); });
+ ZenState InstanceState;
- ParsePath();
+ BuildStorage::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- if (m_OverrideHost.empty() && m_StoragePath.empty())
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+ CreateDirectories(m_ZenFolderPath);
+ auto _ = MakeGuard([this]() {
+ if (CleanDirectory(m_ZenFolderPath, {}))
{
- m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
- CreateDirectories(m_StoragePath);
- CleanDirectory(m_StoragePath, {});
- m_StoragePath = m_StoragePath.generic_string();
+ std::error_code DummyEc;
+ RemoveDir(m_ZenFolderPath, DummyEc);
}
+ });
- auto __ = MakeGuard([&]() {
- if (m_OverrideHost.empty() && m_StoragePath.empty())
- {
- DeleteDirectories(m_StoragePath);
- }
- });
+ StorageInstance Storage = CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_ZenFolderPath),
+ /*RequriesNamespace*/ true,
+ /*RequireBucket*/ true);
- BuildStorage::Statistics StorageStats;
- BuildStorageCache::Statistics StorageCacheStats;
+ Oid BuildId = ParseBuildId();
- const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_test");
- const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2");
+ if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
+ {
+ throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", SubOption->help()));
+ }
- auto ___ = MakeGuard([DownloadPath, DownloadPath2]() {
- CleanDirectory(DownloadPath, true);
- DeleteDirectories(DownloadPath);
- CleanDirectory(DownloadPath2, true);
- DeleteDirectories(DownloadPath2);
- });
+ const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId();
- if (m_ZenFolderPath.empty())
- {
- m_ZenFolderPath = m_Path / ZenFolderName;
- }
- MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+ ValidateStatistics ValidateStats;
+ DownloadStatistics DownloadStats;
+ ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats);
- StorageInstance Storage = CreateBuildStorage(StorageStats,
- StorageCacheStats,
- ZenTempFolderPath(m_ZenFolderPath),
- /*RequriesNamespace*/ true,
- /*RequireBucket*/ true);
+ return AbortFlag ? 13 : 0;
+ }
- m_BuildId = Oid::NewOid().ToString();
- m_BuildPartName = m_Path.filename().string();
- m_BuildPartId = Oid::NewOid().ToString();
- m_CreateBuild = true;
+ if (SubOption == &m_MultiTestDownloadOptions)
+ {
+ m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
+ CreateDirectories(m_SystemRootDir);
+ CleanDirectory(m_SystemRootDir, {});
+ auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); });
- const Oid BuildId = Oid::FromHexString(m_BuildId);
- const Oid BuildPartId = Oid::FromHexString(m_BuildPartId);
+ ParsePath();
- auto MakeMetaData = [](const Oid& BuildId) -> CbObject {
- CbObjectWriter BuildMetaDataWriter;
- {
- const uint32_t CL = BuildId.OidBits[2];
- BuildMetaDataWriter.AddString("name", fmt::format("++Test+Main-CL-{}", CL));
- BuildMetaDataWriter.AddString("branch", "ZenTestBuild");
- BuildMetaDataWriter.AddString("baselineBranch", "ZenTestBuild");
- BuildMetaDataWriter.AddString("platform", "Windows");
- BuildMetaDataWriter.AddString("project", "Test");
- BuildMetaDataWriter.AddInteger("changelist", CL);
- BuildMetaDataWriter.AddString("buildType", "test-folder");
- }
- return BuildMetaDataWriter.Save();
- };
- CbObject MetaData = MakeMetaData(Oid::TryFromHexString(m_BuildId));
- {
- ExtendableStringBuilder<256> SB;
- CompactBinaryToJson(MetaData, SB);
- ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView());
- }
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = m_Path / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
- const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path);
- // std::filesystem::path UploadTempDir = m_ZenFolderPath / "upload_tmp";
+ BuildStorage::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- UploadFolder(Storage,
- BuildId,
- BuildPartId,
- m_BuildPartName,
- m_Path,
- UploadTempDir,
- {},
- m_FindBlockMaxCount,
- m_BlockReuseMinPercentLimit,
- m_AllowMultiparts,
- MetaData,
- true,
- false,
- true);
- if (AbortFlag)
+ StorageInstance Storage = CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_ZenFolderPath),
+ /*RequriesNamespace*/ true,
+ /*RequireBucket*/ true);
+
+ Stopwatch Timer;
+ for (const std::string& BuildIdString : m_BuildIds)
+ {
+ Oid BuildId = Oid::FromHexString(RemoveQuotes(BuildIdString));
+ if (BuildId == Oid::Zero)
{
- ZEN_CONSOLE("Upload failed.");
- return 11;
+ throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, SubOption->help()));
}
-
- ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
DownloadFolder(Storage,
BuildId,
- {BuildPartId},
{},
- DownloadPath,
- DownloadPath / ZenFolderName,
+ {},
+ m_Path,
+ m_ZenFolderPath,
m_SystemRootDir,
m_AllowMultiparts,
m_AllowPartialBlockRequests,
- true,
+ BuildIdString == m_BuildIds.front(),
true,
false,
m_EnableScavenging,
@@ -11873,306 +11678,445 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
""sv);
if (AbortFlag)
{
- ZEN_CONSOLE("Download failed.");
+ ZEN_CONSOLE("Download cancelled");
return 11;
}
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("\n");
+ }
+ }
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ }
+ return 0;
+ }
- ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)",
- BuildId,
- BuildPartId,
- m_BuildPartName,
- DownloadPath);
- DownloadFolder(Storage,
- BuildId,
- {BuildPartId},
- {},
- DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
- if (AbortFlag)
+ auto ParseZenProcessId = [&]() {
+ if (m_ZenProcessId == -1)
+ {
+ const std::filesystem::path RunningExecutablePath = GetRunningExecutablePath();
+ ProcessHandle RunningProcess;
+ std::error_code Ec = FindProcess(RunningExecutablePath, RunningProcess, /*IncludeSelf*/ false);
+ if (Ec)
{
- ZEN_CONSOLE("Re-download failed. (identical target)");
- return 11;
+ throw std::runtime_error(
+ fmt::format("Failed finding process running '{}', reason: '{}'", RunningExecutablePath, Ec.message()));
+ }
+ if (!RunningProcess.IsValid())
+ {
+ throw zen::OptionParseException(
+ fmt::format("Unable to find a running instance of the zen executable '{}'", RunningExecutablePath));
}
+ m_ZenProcessId = RunningProcess.Pid();
+ }
+ };
- auto ScrambleDir = [](const std::filesystem::path& Path) {
- ZEN_CONSOLE("\nScrambling '{}'", Path);
- Stopwatch Timer;
- DirectoryContent DownloadContent;
- GetDirectoryContent(
- Path,
- DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes,
- DownloadContent);
- auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders, Path](const std::filesystem::path& AbsolutePath) -> bool {
- std::string RelativePath = std::filesystem::relative(AbsolutePath, Path).generic_string();
- for (const std::string_view& ExcludeFolder : ExcludeFolders)
- {
- if (RelativePath.starts_with(ExcludeFolder))
- {
- if (RelativePath.length() == ExcludeFolder.length())
- {
- return false;
- }
- else if (RelativePath[ExcludeFolder.length()] == '/')
- {
- return false;
- }
- }
- }
- return true;
- };
+ if (SubOption == &m_PauseOptions)
+ {
+ ParseZenProcessId();
+ ZenState RunningState(m_ZenProcessId);
+ RunningState.StateData().Pause.store(true);
+ return 0;
+ }
- ParallelWork Work(AbortFlag, PauseFlag);
+ if (SubOption == &m_ResumeOptions)
+ {
+ ParseZenProcessId();
+ ZenState RunningState(m_ZenProcessId);
+ RunningState.StateData().Pause.store(false);
+ return 0;
+ }
- uint32_t Randomizer = 0;
- auto FileSizeIt = DownloadContent.FileSizes.begin();
- for (const std::filesystem::path& FilePath : DownloadContent.Files)
- {
- if (IsAcceptedFolder(FilePath))
- {
- uint32_t Case = (Randomizer++) % 7;
- switch (Case)
- {
- case 0:
- {
- uint64_t SourceSize = *FileSizeIt;
- if (SourceSize > 256)
- {
- Work.ScheduleWork(
- GetIOWorkerPool(),
- [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) {
- if (!AbortFlag)
- {
- bool IsReadOnly = SetFileReadOnlyWithRetry(FilePath, false);
- {
- BasicFile Source(FilePath, BasicFile::Mode::kWrite);
- uint64_t RangeSize = Min(SourceSize / 3, 512u * 1024u);
- IoBuffer TempBuffer1(RangeSize);
- IoBuffer TempBuffer2(RangeSize);
- IoBuffer TempBuffer3(RangeSize);
- Source.Read(TempBuffer1.GetMutableView().GetData(), RangeSize, 0);
- Source.Read(TempBuffer2.GetMutableView().GetData(), RangeSize, SourceSize / 2);
- Source.Read(TempBuffer3.GetMutableView().GetData(),
- RangeSize,
- SourceSize - RangeSize);
- Source.Write(TempBuffer1, SourceSize / 2);
- Source.Write(TempBuffer2, SourceSize - RangeSize);
- Source.Write(TempBuffer3, SourceSize - 0);
- }
- if (IsReadOnly)
- {
- SetFileReadOnly(FilePath, true);
- }
- }
- });
- }
- }
- break;
- case 1:
- {
- (void)SetFileReadOnlyWithRetry(FilePath, false);
- RemoveFileWithRetry(FilePath);
- }
- break;
- default:
- break;
- }
- }
- FileSizeIt++;
- }
- Work.Wait(5000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
- ZEN_UNUSED(IsAborted, IsPaused);
- ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork);
- });
- ZEN_ASSERT(!AbortFlag.load());
- ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
- };
+ if (SubOption == &m_AbortOptions)
+ {
+ ParseZenProcessId();
+ ZenState RunningState(m_ZenProcessId);
+ RunningState.StateData().Abort.store(true);
+ return 0;
+ }
- ScrambleDir(DownloadPath);
- ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)",
- BuildId,
- BuildPartId,
- m_BuildPartName,
- DownloadPath);
- DownloadFolder(Storage,
- BuildId,
- {BuildPartId},
- {},
- DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
- if (AbortFlag)
- {
- ZEN_CONSOLE("Re-download failed. (scrambled target)");
- return 11;
- }
+ if (SubOption == &m_TestOptions)
+ {
+ m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
+ CreateDirectories(m_SystemRootDir);
+ CleanDirectory(m_SystemRootDir, {});
+ auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); });
- ScrambleDir(DownloadPath);
+ ParsePath();
- Oid BuildId2 = Oid::NewOid();
- Oid BuildPartId2 = Oid::NewOid();
+ if (m_OverrideHost.empty() && m_StoragePath.empty())
+ {
+ m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
+ CreateDirectories(m_StoragePath);
+ CleanDirectory(m_StoragePath, {});
+ m_StoragePath = m_StoragePath.generic_string();
+ }
- CbObject MetaData2 = MakeMetaData(BuildId2);
+ auto __ = MakeGuard([&]() {
+ if (m_OverrideHost.empty() && m_StoragePath.empty())
{
- ExtendableStringBuilder<256> SB;
- CompactBinaryToJson(MetaData, SB);
- ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView());
+ DeleteDirectories(m_StoragePath);
}
+ });
- UploadFolder(Storage,
- BuildId2,
- BuildPartId2,
- m_BuildPartName,
- DownloadPath,
- UploadTempDir,
- {},
- m_FindBlockMaxCount,
- m_BlockReuseMinPercentLimit,
- m_AllowMultiparts,
- MetaData2,
- true,
- false,
- true);
- if (AbortFlag)
- {
- ZEN_CONSOLE("Upload of scrambled failed.");
- return 11;
- }
+ BuildStorage::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
- ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
- DownloadFolder(Storage,
- BuildId,
- {BuildPartId},
- {},
- DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
- if (AbortFlag)
- {
- ZEN_CONSOLE("Re-download failed.");
- return 11;
- }
+ const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_test");
+ const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2");
- ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
- DownloadFolder(Storage,
- BuildId2,
- {BuildPartId2},
- {},
- DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
- if (AbortFlag)
- {
- ZEN_CONSOLE("Re-download failed.");
- return 11;
- }
+ auto ___ = MakeGuard([DownloadPath, DownloadPath2]() {
+ CleanDirectory(DownloadPath, true);
+ DeleteDirectories(DownloadPath);
+ CleanDirectory(DownloadPath2, true);
+ DeleteDirectories(DownloadPath2);
+ });
- ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
- DownloadFolder(Storage,
- BuildId2,
- {BuildPartId2},
- {},
- DownloadPath,
- DownloadPath / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
- if (AbortFlag)
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = m_Path / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+ StorageInstance Storage = CreateBuildStorage(StorageStats,
+ StorageCacheStats,
+ ZenTempFolderPath(m_ZenFolderPath),
+ /*RequriesNamespace*/ true,
+ /*RequireBucket*/ true);
+
+ m_BuildId = Oid::NewOid().ToString();
+ m_BuildPartName = m_Path.filename().string();
+ m_BuildPartId = Oid::NewOid().ToString();
+ m_CreateBuild = true;
+
+ const Oid BuildId = Oid::FromHexString(m_BuildId);
+ const Oid BuildPartId = Oid::FromHexString(m_BuildPartId);
+
+ auto MakeMetaData = [](const Oid& BuildId) -> CbObject {
+ CbObjectWriter BuildMetaDataWriter;
{
- ZEN_CONSOLE("Re-download failed.");
- return 11;
+ const uint32_t CL = BuildId.OidBits[2];
+ BuildMetaDataWriter.AddString("name", fmt::format("++Test+Main-CL-{}", CL));
+ BuildMetaDataWriter.AddString("branch", "ZenTestBuild");
+ BuildMetaDataWriter.AddString("baselineBranch", "ZenTestBuild");
+ BuildMetaDataWriter.AddString("platform", "Windows");
+ BuildMetaDataWriter.AddString("project", "Test");
+ BuildMetaDataWriter.AddInteger("changelist", CL);
+ BuildMetaDataWriter.AddString("buildType", "test-folder");
}
+ return BuildMetaDataWriter.Save();
+ };
+ CbObject MetaData = MakeMetaData(Oid::TryFromHexString(m_BuildId));
+ {
+ ExtendableStringBuilder<256> SB;
+ CompactBinaryToJson(MetaData, SB);
+ ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView());
+ }
- ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2);
- DownloadFolder(Storage,
- BuildId,
- {BuildPartId},
- {},
- DownloadPath2,
- DownloadPath2 / ZenFolderName,
- m_SystemRootDir,
- m_AllowMultiparts,
- m_AllowPartialBlockRequests,
- false,
- true,
- false,
- m_EnableScavenging,
- ""sv,
- ""sv);
- if (AbortFlag)
+ const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path);
+ // std::filesystem::path UploadTempDir = m_ZenFolderPath / "upload_tmp";
+
+ UploadFolder(Storage,
+ BuildId,
+ BuildPartId,
+ m_BuildPartName,
+ m_Path,
+ UploadTempDir,
+ {},
+ m_FindBlockMaxCount,
+ m_BlockReuseMinPercentLimit,
+ m_AllowMultiparts,
+ MetaData,
+ true,
+ false,
+ true);
+ if (AbortFlag)
+ {
+ ZEN_CONSOLE_ERROR("Upload failed.");
+ return 11;
+ }
+
+ ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
+ DownloadFolder(Storage,
+ BuildId,
+ {BuildPartId},
+ {},
+ DownloadPath,
+ DownloadPath / ZenFolderName,
+ m_SystemRootDir,
+ m_AllowMultiparts,
+ m_AllowPartialBlockRequests,
+ true,
+ true,
+ false,
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
+ if (AbortFlag)
+ {
+ ZEN_CONSOLE_ERROR("Download failed.");
+ return 11;
+ }
+
+ ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
+ DownloadFolder(Storage,
+ BuildId,
+ {BuildPartId},
+ {},
+ DownloadPath,
+ DownloadPath / ZenFolderName,
+ m_SystemRootDir,
+ m_AllowMultiparts,
+ m_AllowPartialBlockRequests,
+ false,
+ true,
+ false,
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
+ if (AbortFlag)
+ {
+ ZEN_CONSOLE_ERROR("Re-download failed. (identical target)");
+ return 11;
+ }
+
+ auto ScrambleDir = [](const std::filesystem::path& Path) {
+ ZEN_CONSOLE("\nScrambling '{}'", Path);
+ Stopwatch Timer;
+ DirectoryContent DownloadContent;
+ GetDirectoryContent(
+ Path,
+ DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes,
+ DownloadContent);
+ auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders, Path](const std::filesystem::path& AbsolutePath) -> bool {
+ std::string RelativePath = std::filesystem::relative(AbsolutePath, Path).generic_string();
+ for (const std::string_view& ExcludeFolder : ExcludeFolders)
+ {
+ if (RelativePath.starts_with(ExcludeFolder))
+ {
+ if (RelativePath.length() == ExcludeFolder.length())
+ {
+ return false;
+ }
+ else if (RelativePath[ExcludeFolder.length()] == '/')
+ {
+ return false;
+ }
+ }
+ }
+ return true;
+ };
+
+ ParallelWork Work(AbortFlag, PauseFlag);
+
+ uint32_t Randomizer = 0;
+ auto FileSizeIt = DownloadContent.FileSizes.begin();
+ for (const std::filesystem::path& FilePath : DownloadContent.Files)
{
- ZEN_CONSOLE("Re-download failed.");
- return 11;
+ if (IsAcceptedFolder(FilePath))
+ {
+ uint32_t Case = (Randomizer++) % 7;
+ switch (Case)
+ {
+ case 0:
+ {
+ uint64_t SourceSize = *FileSizeIt;
+ if (SourceSize > 256)
+ {
+ Work.ScheduleWork(
+ GetIOWorkerPool(),
+ [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) {
+ if (!AbortFlag)
+ {
+ bool IsReadOnly = SetFileReadOnlyWithRetry(FilePath, false);
+ {
+ BasicFile Source(FilePath, BasicFile::Mode::kWrite);
+ uint64_t RangeSize = Min(SourceSize / 3, 512u * 1024u);
+ IoBuffer TempBuffer1(RangeSize);
+ IoBuffer TempBuffer2(RangeSize);
+ IoBuffer TempBuffer3(RangeSize);
+ Source.Read(TempBuffer1.GetMutableView().GetData(), RangeSize, 0);
+ Source.Read(TempBuffer2.GetMutableView().GetData(), RangeSize, SourceSize / 2);
+ Source.Read(TempBuffer3.GetMutableView().GetData(), RangeSize, SourceSize - RangeSize);
+ Source.Write(TempBuffer1, SourceSize / 2);
+ Source.Write(TempBuffer2, SourceSize - RangeSize);
+ Source.Write(TempBuffer3, SourceSize - 0);
+ }
+ if (IsReadOnly)
+ {
+ SetFileReadOnly(FilePath, true);
+ }
+ }
+ });
+ }
+ }
+ break;
+ case 1:
+ {
+ (void)SetFileReadOnlyWithRetry(FilePath, false);
+ RemoveFileWithRetry(FilePath);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ FileSizeIt++;
}
+ Work.Wait(5000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+ ZEN_UNUSED(IsAborted, IsPaused);
+ ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork);
+ });
+ ZEN_ASSERT(!AbortFlag.load());
+ ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ };
- return 0;
- }
- }
- catch (const std::system_error& SysErr)
- {
- if (IsOOD(SysErr))
+ ScrambleDir(DownloadPath);
+ ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
+ DownloadFolder(Storage,
+ BuildId,
+ {BuildPartId},
+ {},
+ DownloadPath,
+ DownloadPath / ZenFolderName,
+ m_SystemRootDir,
+ m_AllowMultiparts,
+ m_AllowPartialBlockRequests,
+ false,
+ true,
+ false,
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
+ if (AbortFlag)
{
- ZEN_CONSOLE("Operation failed due to out of disk space: {}", SysErr.what());
- return 3;
+ ZEN_CONSOLE_ERROR("Re-download failed. (scrambled target)");
+ return 11;
}
- else if (IsOOM(SysErr))
+
+ ScrambleDir(DownloadPath);
+
+ Oid BuildId2 = Oid::NewOid();
+ Oid BuildPartId2 = Oid::NewOid();
+
+ CbObject MetaData2 = MakeMetaData(BuildId2);
{
- ZEN_CONSOLE("Operation failed due to out of memory: {}", SysErr.what());
- return 3;
+ ExtendableStringBuilder<256> SB;
+ CompactBinaryToJson(MetaData, SB);
+ ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView());
}
- else
+
+ UploadFolder(Storage,
+ BuildId2,
+ BuildPartId2,
+ m_BuildPartName,
+ DownloadPath,
+ UploadTempDir,
+ {},
+ m_FindBlockMaxCount,
+ m_BlockReuseMinPercentLimit,
+ m_AllowMultiparts,
+ MetaData2,
+ true,
+ false,
+ true);
+ if (AbortFlag)
+ {
+ ZEN_CONSOLE_ERROR("Upload of scrambled failed.");
+ return 11;
+ }
+
+ ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath);
+ DownloadFolder(Storage,
+ BuildId,
+ {BuildPartId},
+ {},
+ DownloadPath,
+ DownloadPath / ZenFolderName,
+ m_SystemRootDir,
+ m_AllowMultiparts,
+ m_AllowPartialBlockRequests,
+ false,
+ true,
+ false,
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
+ if (AbortFlag)
+ {
+ ZEN_CONSOLE_ERROR("Re-download failed.");
+ return 11;
+ }
+
+ ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
+ DownloadFolder(Storage,
+ BuildId2,
+ {BuildPartId2},
+ {},
+ DownloadPath,
+ DownloadPath / ZenFolderName,
+ m_SystemRootDir,
+ m_AllowMultiparts,
+ m_AllowPartialBlockRequests,
+ false,
+ true,
+ false,
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
+ if (AbortFlag)
+ {
+ ZEN_CONSOLE_ERROR("Re-download failed.");
+ return 11;
+ }
+
+ ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath);
+ DownloadFolder(Storage,
+ BuildId2,
+ {BuildPartId2},
+ {},
+ DownloadPath,
+ DownloadPath / ZenFolderName,
+ m_SystemRootDir,
+ m_AllowMultiparts,
+ m_AllowPartialBlockRequests,
+ false,
+ true,
+ false,
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
+ if (AbortFlag)
{
- ZEN_ERROR("{}", SysErr.what());
- return 3;
+ ZEN_CONSOLE_ERROR("Re-download failed.");
+ return 11;
+ }
+
+ ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2);
+ DownloadFolder(Storage,
+ BuildId,
+ {BuildPartId},
+ {},
+ DownloadPath2,
+ DownloadPath2 / ZenFolderName,
+ m_SystemRootDir,
+ m_AllowMultiparts,
+ m_AllowPartialBlockRequests,
+ false,
+ true,
+ false,
+ m_EnableScavenging,
+ ""sv,
+ ""sv);
+ if (AbortFlag)
+ {
+ ZEN_CONSOLE_ERROR("Re-download failed.");
+ return 11;
}
- }
- catch (const HttpClientError& HttpEx)
- {
- ZEN_CONSOLE("Operation failed: {}", HttpEx.what());
- return HttpEx.m_Error != 0 ? HttpEx.m_Error : (int)HttpEx.m_ResponseCode;
- }
- catch (const std::exception& Ex)
- {
- ZEN_ERROR("{}", Ex.what());
- return 3;
+
+ return 0;
}
ZEN_ASSERT(false);
}
diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp
index 4412eaf34..90f428b29 100644
--- a/src/zen/cmds/cache_cmd.cpp
+++ b/src/zen/cmds/cache_cmd.cpp
@@ -115,11 +115,11 @@ DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (Result.status_code)
{
- ZEN_ERROR("Drop failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
+ ZEN_CONSOLE_ERROR("Drop failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
}
else
{
- ZEN_ERROR("Drop failed: {}", Result.error.message);
+ ZEN_CONSOLE_ERROR("Drop failed: {}", Result.error.message);
}
return 1;
@@ -217,11 +217,11 @@ CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (Result.status_code)
{
- ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
+ ZEN_CONSOLE_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
}
else
{
- ZEN_ERROR("Info failed: {}", Result.error.message);
+ ZEN_CONSOLE_ERROR("Info failed: {}", Result.error.message);
}
return 1;
@@ -267,11 +267,11 @@ CacheStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv
if (Result.status_code)
{
- ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
+ ZEN_CONSOLE_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
}
else
{
- ZEN_ERROR("Info failed: {}", Result.error.message);
+ ZEN_CONSOLE_ERROR("Info failed: {}", Result.error.message);
}
return 1;
@@ -336,9 +336,7 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar
{
if (m_Namespace.empty() || m_Bucket.empty())
{
- ZEN_ERROR("Provide namespace and bucket name");
- ZEN_CONSOLE("{}", m_Options.help({""}).c_str());
- return 1;
+ throw OptionParseException("Provide namespace and bucket name");
}
Session.SetUrl({fmt::format("{}/z$/details$/{}/{}/{}", m_HostName, m_Namespace, m_Bucket, m_ValueKey)});
}
@@ -346,9 +344,7 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar
{
if (m_Namespace.empty())
{
- ZEN_ERROR("Provide namespace name");
- ZEN_CONSOLE("{}", m_Options.help({""}).c_str());
- return 1;
+ throw OptionParseException("Provide namespace name");
}
Session.SetUrl({fmt::format("{}/z$/details$/{}/{}", m_HostName, m_Namespace, m_Bucket)});
}
@@ -373,11 +369,11 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar
if (Result.status_code)
{
- ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
+ ZEN_CONSOLE_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
}
else
{
- ZEN_ERROR("Info failed: {}", Result.error.message);
+ ZEN_CONSOLE_ERROR("Info failed: {}", Result.error.message);
}
return 1;
diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp
index e86b6964c..4e54f27bb 100644
--- a/src/zen/cmds/copy_cmd.cpp
+++ b/src/zen/cmds/copy_cmd.cpp
@@ -147,7 +147,7 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
++FailedFileCount;
- ZEN_CONSOLE_ERROR("failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what());
+ ZEN_CONSOLE_ERROR("Failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what());
}
}
}
@@ -208,7 +208,7 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
catch (const std::exception& Ex)
{
- ZEN_CONSOLE_ERROR("Error: failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what());
+ ZEN_CONSOLE_ERROR("Failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what());
return 1;
}
diff --git a/src/zen/cmds/dedup_cmd.cpp b/src/zen/cmds/dedup_cmd.cpp
index 5f74305c7..033ac87c3 100644
--- a/src/zen/cmds/dedup_cmd.cpp
+++ b/src/zen/cmds/dedup_cmd.cpp
@@ -68,15 +68,13 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (!SourceGood)
{
- ZEN_ERROR("Source directory '{}' does not support deduplication", m_DedupSource);
-
+ ZEN_CONSOLE_ERROR("Source directory '{}' does not support deduplication", m_DedupSource);
return 0;
}
if (!TargetGood)
{
- ZEN_ERROR("Target directory '{}' does not support deduplication", m_DedupTarget);
-
+ ZEN_CONSOLE_ERROR("Target directory '{}' does not support deduplication", m_DedupTarget);
return 0;
}
diff --git a/src/zen/cmds/info_cmd.cpp b/src/zen/cmds/info_cmd.cpp
index 73a5d2e5d..8e0b3d663 100644
--- a/src/zen/cmds/info_cmd.cpp
+++ b/src/zen/cmds/info_cmd.cpp
@@ -44,6 +44,10 @@ InfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
ZEN_CONSOLE("{}", Result.ToText());
}
+ else
+ {
+ Result.ThrowError(fmt::format("Failed getting info from {}", m_HostName));
+ }
return 0;
}
diff --git a/src/zen/cmds/print_cmd.cpp b/src/zen/cmds/print_cmd.cpp
index 5bc8a8ed8..fd6d92f28 100644
--- a/src/zen/cmds/print_cmd.cpp
+++ b/src/zen/cmds/print_cmd.cpp
@@ -84,7 +84,7 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (Fc.ErrorCode)
{
- ZEN_ERROR("Failed to read file '{}': {}", m_Filename, Fc.ErrorCode.message());
+ ZEN_CONSOLE_ERROR("Failed to read file '{}': {}", m_Filename, Fc.ErrorCode.message());
return 1;
}
@@ -149,7 +149,7 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (FilteredResult == CbValidateError::None && FilteredResult != Result)
{
- ZEN_WARN(
+ ZEN_CONSOLE_WARN(
"Object in package message file '{}' does not appear to be an optimal compact binary format (validation error {:#x}: '{}')",
m_Filename,
uint32_t(Result),
@@ -158,10 +158,10 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (FilteredResult != CbValidateError::None)
{
- ZEN_ERROR("Object in package message file '{}' does not appear to be compact binary (validation error {:#x}: '{}')",
- m_Filename,
- uint32_t(FilteredResult),
- ToString(FilteredResult));
+ ZEN_CONSOLE_ERROR("Object in package message file '{}' does not appear to be compact binary (validation error {:#x}: '{}')",
+ m_Filename,
+ uint32_t(FilteredResult),
+ ToString(FilteredResult));
return 1;
}
else
@@ -177,18 +177,18 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (FilteredResult == CbValidateError::None && FilteredResult != Result)
{
- ZEN_WARN("Data in file '{}' does not appear to be an optimal compact binary format (validation error {:#x}: '{}')",
- m_Filename,
- uint32_t(Result),
- ToString(Result));
+ ZEN_CONSOLE_WARN("Data in file '{}' does not appear to be an optimal compact binary format (validation error {:#x}: '{}')",
+ m_Filename,
+ uint32_t(Result),
+ ToString(Result));
}
if (FilteredResult != CbValidateError::None)
{
- ZEN_ERROR("Data in file '{}' does not appear to be compact binary (validation error {:#x}: '{}')",
- m_Filename,
- uint32_t(FilteredResult),
- ToString(FilteredResult));
+ ZEN_CONSOLE_ERROR("Data in file '{}' does not appear to be compact binary (validation error {:#x}: '{}')",
+ m_Filename,
+ uint32_t(FilteredResult),
+ ToString(FilteredResult));
return 1;
}
else
diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp
index f919edc87..5f0f52f7c 100644
--- a/src/zen/cmds/projectstore_cmd.cpp
+++ b/src/zen/cmds/projectstore_cmd.cpp
@@ -449,7 +449,7 @@ namespace {
}
else
{
- ZEN_CONSOLE("Unable to resolve project name '{}', no projects found at {}", OptionalProjectName, Http.GetBaseUri());
+ ZEN_CONSOLE_WARN("Unable to resolve project name '{}', no projects found at {}", OptionalProjectName, Http.GetBaseUri());
}
return {};
}
@@ -470,10 +470,10 @@ namespace {
std::vector<std::string> MatchingProjectIds = MatchId(AvailableProjects, OptionalProjectName);
if (MatchingProjectIds.empty())
{
- ZEN_CONSOLE("Unable to match project name '{}' at {}, available projects: {}",
- OptionalProjectName,
- Http.GetBaseUri(),
- FmtProjectIdArray(AvailableProjects));
+ ZEN_CONSOLE_WARN("Unable to match project name '{}' at {}, available projects: {}",
+ OptionalProjectName,
+ Http.GetBaseUri(),
+ FmtProjectIdArray(AvailableProjects));
return {};
}
@@ -482,10 +482,10 @@ namespace {
return MatchingProjectIds.front();
}
- ZEN_CONSOLE("Project name is ambigous '{}' at {}: possible matches: {}",
- OptionalProjectName,
- Http.GetBaseUri(),
- FmtProjectIdArray(MatchingProjectIds));
+ ZEN_CONSOLE_WARN("Project name is ambigous '{}' at {}: possible matches: {}",
+ OptionalProjectName,
+ Http.GetBaseUri(),
+ FmtProjectIdArray(MatchingProjectIds));
return {};
}
@@ -500,10 +500,10 @@ namespace {
}
else
{
- ZEN_CONSOLE("Unable to resolve oplog name '{}' for project '{}', no oplogs found at {}",
- OptionalOplogName,
- ProjectName,
- Http.GetBaseUri());
+ ZEN_CONSOLE_WARN("Unable to resolve oplog name '{}' for project '{}', no oplogs found at {}",
+ OptionalOplogName,
+ ProjectName,
+ Http.GetBaseUri());
}
return {};
}
@@ -524,11 +524,11 @@ namespace {
std::vector<std::string> MatchingOplogIds = MatchId(AvailableOplogs, OptionalOplogName);
if (MatchingOplogIds.empty())
{
- ZEN_CONSOLE("Unable to match oplog name '{}' for project '{}', available oplogs at {}: {}",
- OptionalOplogName,
- ProjectName,
- Http.GetBaseUri(),
- FmtArray(AvailableOplogs));
+ ZEN_CONSOLE_WARN("Unable to match oplog name '{}' for project '{}', available oplogs at {}: {}",
+ OptionalOplogName,
+ ProjectName,
+ Http.GetBaseUri(),
+ FmtArray(AvailableOplogs));
return {};
}
@@ -537,11 +537,11 @@ namespace {
return MatchingOplogIds.front();
}
- ZEN_CONSOLE("Oplog name '{}' for project '{}' at {}, is ambigous, possible matches: {}",
- OptionalOplogName,
- ProjectName,
- Http.GetBaseUri(),
- FmtArray(MatchingOplogIds));
+ ZEN_CONSOLE_WARN("Oplog name '{}' for project '{}' at {}, is ambigous, possible matches: {}",
+ OptionalOplogName,
+ ProjectName,
+ Http.GetBaseUri(),
+ FmtArray(MatchingOplogIds));
return {};
}
@@ -766,8 +766,7 @@ CreateProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
if (m_ProjectId.empty())
{
- ZEN_ERROR("Project name must be given");
- return 1;
+ throw OptionParseException("Project name must be given");
}
HttpClient Http(m_HostName);
@@ -778,7 +777,7 @@ CreateProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
{
if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON)))
{
- ZEN_CONSOLE("Project already exists.\n{}", Result.ToText());
+ ZEN_CONSOLE_ERROR("Project already exists.\n{}", Result.ToText());
return 1;
}
}
@@ -1073,9 +1072,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
{
if (m_JupiterNamespace.empty() || m_JupiterBucket.empty())
{
- ZEN_ERROR("Options for cloud target are missing");
- ZEN_CONSOLE("{}", m_Options.help({"cloud"}).c_str());
- return 1;
+ throw OptionParseException("Options for cloud target are missing");
}
if (m_CloudKey.empty())
{
@@ -1088,23 +1085,21 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
if (!m_BuildsUrl.empty())
{
+ if (m_JupiterNamespace.empty())
+ {
+ throw OptionParseException("Namespace for builds target it missing");
+ }
if (m_JupiterNamespace.empty() || m_JupiterBucket.empty())
{
- ZEN_ERROR("Options for builds target are missing");
- ZEN_CONSOLE("{}", m_Options.help({"builds"}).c_str());
- return 1;
+ throw OptionParseException("Bucket for builds target it missing");
}
if (m_BuildsMetadataPath.empty() && m_BuildsMetadata.empty())
{
- ZEN_ERROR("Options for builds target are missing");
- ZEN_CONSOLE("{}", m_Options.help({"builds"}).c_str());
- return 1;
+ throw OptionParseException("Options for builds target metadata are missing");
}
if (!m_BuildsMetadataPath.empty() && !m_BuildsMetadata.empty())
{
- ZEN_ERROR("Conflicting options for builds target");
- ZEN_CONSOLE("{}", m_Options.help({"builds"}).c_str());
- return 1;
+ throw OptionParseException("Conflicting options for builds target metadata");
}
if (m_BuildsId.empty())
{
@@ -1118,12 +1113,12 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
if (m_ZenProjectName.empty())
{
m_ZenProjectName = m_ProjectName;
- ZEN_WARN("Using default zen target project id '{}'", m_ZenProjectName);
+ ZEN_CONSOLE_WARN("Using default zen target project id '{}'", m_ZenProjectName);
}
if (m_ZenOplogName.empty())
{
m_ZenOplogName = m_OplogName;
- ZEN_WARN("Using default zen target oplog id '{}'", m_ZenOplogName);
+ ZEN_CONSOLE_WARN("Using default zen target oplog id '{}'", m_ZenOplogName);
}
std::string TargetUrlBase = m_ZenUrl;
@@ -1141,7 +1136,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
{
if (m_ZenClean)
{
- ZEN_WARN("Deleting zen remote oplog '{}/{}'", m_ZenProjectName, m_ZenOplogName)
+ ZEN_CONSOLE_WARN("Deleting zen remote oplog '{}/{}'", m_ZenProjectName, m_ZenOplogName);
Result = TargetHttp.Delete(Url, HttpClient::Accept(ZenContentType::kJSON));
if (!Result)
{
@@ -1163,7 +1158,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
if (CreateOplog)
{
- ZEN_WARN("Creating zen remote oplog '{}/{}'", m_ZenProjectName, m_ZenOplogName);
+ ZEN_CONSOLE_WARN("Creating zen remote oplog '{}/{}'", m_ZenProjectName, m_ZenOplogName);
if (HttpClient::Response Result = TargetHttp.Post(Url); !Result)
{
Result.ThrowError("failed creating zen remote oplog"sv);
@@ -1177,7 +1172,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
if (m_FileName.empty())
{
m_FileName = m_OplogName;
- ZEN_WARN("Using default file name '{}'", m_FileName);
+ ZEN_CONSOLE_WARN("Using default file name '{}'", m_FileName);
}
}
@@ -1370,21 +1365,11 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
m_PlainProgress);
}
}
- catch (const HttpClientError& Ex)
- {
- ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what());
- return Ex.m_Error != 0 ? Ex.m_Error : (int)Ex.m_ResponseCode;
- }
catch (const AsyncJobError& Ex)
{
- ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what());
+ ZEN_CONSOLE_ERROR("Oplog export failed: '{}'", Ex.what());
return Ex.m_ReturnCode;
}
- catch (const std::exception& Ex)
- {
- ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what());
- return 1;
- }
return 0;
}
@@ -1498,14 +1483,12 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
if (m_ProjectName.empty())
{
- ZEN_ERROR("Project name must be given");
- return 1;
+ throw OptionParseException("Project name must be given");
}
if (m_OplogName.empty())
{
- ZEN_ERROR("Oplog name must be given");
- return 1;
+ throw OptionParseException("Oplog name must be given");
}
HttpClient Http(m_HostName);
@@ -1522,35 +1505,41 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
TargetCount += m_FileDirectoryPath.empty() ? 0 : 1;
if (TargetCount != 1)
{
- ZEN_ERROR("Provide one source only");
- ZEN_CONSOLE("{}", m_Options.help({""}).c_str());
- return 1;
+ throw OptionParseException("Provide one source only");
}
if (!m_CloudUrl.empty())
{
- if (m_JupiterNamespace.empty() || m_JupiterBucket.empty())
+ if (m_JupiterNamespace.empty())
{
- ZEN_ERROR("Options for cloud source are missing");
- ZEN_CONSOLE("{}", m_Options.help({"cloud"}).c_str());
- return 1;
+ throw OptionParseException("Namespace option for cloud source is missing");
+ }
+ if (m_JupiterBucket.empty())
+ {
+ throw OptionParseException("Bucket option for cloud source is missing");
}
if (m_CloudKey.empty())
{
std::string KeyString = fmt::format("{}/{}/{}/{}", m_ProjectName, m_OplogName, m_JupiterNamespace, m_JupiterBucket);
IoHash Key = IoHash::HashBuffer(KeyString.data(), KeyString.size());
m_CloudKey = Key.ToHexString();
- ZEN_WARN("Using auto generated cloud key '{}'", m_CloudKey);
+ ZEN_CONSOLE_WARN("Using auto generated cloud key '{}'", m_CloudKey);
}
}
if (!m_BuildsUrl.empty())
{
- if (m_JupiterNamespace.empty() || m_JupiterBucket.empty() || m_BuildsId.empty())
+ if (m_JupiterNamespace.empty())
{
- ZEN_ERROR("Options for builds source are missing");
- ZEN_CONSOLE("{}", m_Options.help({"builds"}).c_str());
- return 1;
+ throw OptionParseException("Namespace option for builds source is missing");
+ }
+ if (m_JupiterBucket.empty())
+ {
+ throw OptionParseException("Bucket option for builds source is missing");
+ }
+ if (m_BuildsId.empty())
+ {
+ throw OptionParseException("Build id option for builds source is missing");
}
}
@@ -1559,12 +1548,12 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
if (m_ZenProjectName.empty())
{
m_ZenProjectName = m_ProjectName;
- ZEN_WARN("Using default zen target project id '{}'", m_ZenProjectName);
+ ZEN_CONSOLE_WARN("Using default zen target project id '{}'", m_ZenProjectName);
}
if (m_ZenOplogName.empty())
{
m_ZenOplogName = m_OplogName;
- ZEN_WARN("Using default zen target oplog id '{}'", m_ZenOplogName);
+ ZEN_CONSOLE_WARN("Using default zen target oplog id '{}'", m_ZenOplogName);
}
}
@@ -1573,7 +1562,7 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
if (m_FileName.empty())
{
m_FileName = m_OplogName;
- ZEN_WARN("Using auto generated file name '{}'", m_FileName);
+ ZEN_CONSOLE_WARN("Using auto generated file name '{}'", m_FileName);
}
}
@@ -1598,7 +1587,7 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
{
OplogPayload = MakeCbObjectPayload([&](CbObjectWriter& Writer) { Writer.AddString("gcpath"sv, m_GcPath); });
}
- ZEN_WARN("Creating oplog '{}/{}'", m_ProjectName, m_OplogName);
+ ZEN_CONSOLE("Creating oplog '{}/{}'", m_ProjectName, m_OplogName);
if (HttpClient::Response Result = Http.Post(Url, OplogPayload); !Result)
{
Result.ThrowError("failed creating oplog"sv);
@@ -1723,21 +1712,11 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
m_PlainProgress);
}
}
- catch (const HttpClientError& Ex)
- {
- ZEN_CONSOLE("Oplog import failed: '{}'", Ex.what());
- return Ex.m_Error != 0 ? Ex.m_Error : (int)Ex.m_ResponseCode;
- }
catch (const AsyncJobError& Ex)
{
- ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what());
+ ZEN_CONSOLE_ERROR("Oplog export failed: '{}'", Ex.what());
return Ex.m_ReturnCode;
}
- catch (const std::exception& Ex)
- {
- ZEN_CONSOLE("Oplog import failed: '{}'", Ex.what());
- return 1;
- }
return 0;
}
@@ -1780,7 +1759,7 @@ SnapshotOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
if (m_ProjectName.empty())
{
- ZEN_ERROR("Project name must be given");
+ throw OptionParseException("Project name must be given");
return 1;
}
@@ -2147,7 +2126,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
catch (const std::exception& Ex)
{
AbortFlag.store(true);
- ZEN_CONSOLE("Failed writing file to '{}'. Reason: '{}'", TargetPath, Ex.what());
+ ZEN_CONSOLE_ERROR("Failed writing file to '{}'. Reason: '{}'", TargetPath, Ex.what());
}
}
});
@@ -2233,7 +2212,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
}
else
{
- ZEN_ERROR("unknown format response to oplog entries request");
+ ZEN_CONSOLE_ERROR("Unknown format response to oplog entries request");
}
}
else
diff --git a/src/zen/cmds/rpcreplay_cmd.cpp b/src/zen/cmds/rpcreplay_cmd.cpp
index 4fc38d92a..6c26eaf23 100644
--- a/src/zen/cmds/rpcreplay_cmd.cpp
+++ b/src/zen/cmds/rpcreplay_cmd.cpp
@@ -429,7 +429,7 @@ RpcReplayCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (Response.error || !(IsHttpSuccessCode(Response.status_code) ||
Response.status_code == gsl::narrow<long>(HttpResponseCode::NotFound)))
{
- ZEN_CONSOLE("{}", FormatHttpResponse(Response));
+ ZEN_CONSOLE_ERROR("{}", FormatHttpResponse(Response));
break;
}
BytesReceived.fetch_add(Response.downloaded_bytes);
diff --git a/src/zen/cmds/serve_cmd.cpp b/src/zen/cmds/serve_cmd.cpp
index 64039e4c9..cfaa12bc0 100644
--- a/src/zen/cmds/serve_cmd.cpp
+++ b/src/zen/cmds/serve_cmd.cpp
@@ -93,7 +93,7 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
catch (const std::exception& Ex)
{
- ZEN_CONSOLE("failed to spawn server on port {}: '{}'", ServerPort, Ex.what());
+ ZEN_CONSOLE_ERROR("Failed to spawn server on port {}: '{}'", ServerPort, Ex.what());
throw zen::OptionParseException("unable to resolve server specification (even after spawning server)");
}
@@ -111,7 +111,7 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (ServerInstance)
{
m_HostName = ServerInstance->GetBaseUri();
- ZEN_CONSOLE("base uri: {}", m_HostName);
+ ZEN_CONSOLE("Base uri: {}", m_HostName);
}
// Generate manifest for tree
@@ -159,7 +159,7 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
for (const FsVisitor::FileEntry& Entry : Visitor.Files)
{
- ZEN_CONSOLE("file: {}", Entry.FilePath);
+ ZEN_CONSOLE("File: {}", Entry.FilePath);
Cbo.BeginObject();
@@ -225,12 +225,10 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (!HttpResponse)
{
- ZEN_CONSOLE("error: failed to append manifest!");
-
- return 1;
+ HttpResponse.ThrowError("Failed to append manifest");
}
- ZEN_CONSOLE("ok serving files now");
+ ZEN_CONSOLE("Ok serving files now");
#if ZEN_PLATFORM_WINDOWS
_getch(); // TEMPORARY HACK
diff --git a/src/zen/cmds/status_cmd.cpp b/src/zen/cmds/status_cmd.cpp
index 2b507e43d..b5764af44 100644
--- a/src/zen/cmds/status_cmd.cpp
+++ b/src/zen/cmds/status_cmd.cpp
@@ -34,14 +34,14 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
if (!IsFile(m_DataDir / ".lock"))
{
- ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir);
+ ZEN_CONSOLE_ERROR("Lock file does not exist in directory '{}'", m_DataDir);
return 1;
}
LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock")));
std::string Reason;
if (!ValidateLockFileInfo(Info, Reason))
{
- ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
+ ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
return 1;
}
EffectivePort = Info.EffectiveListenPort;
@@ -50,8 +50,7 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZenServerState State;
if (!State.InitializeReadOnly())
{
- ZEN_CONSOLE("no Zen state found");
-
+ ZEN_CONSOLE("No Zen state found");
return 0;
}
diff --git a/src/zen/cmds/top_cmd.cpp b/src/zen/cmds/top_cmd.cpp
index 568ee76c9..9794dc1c0 100644
--- a/src/zen/cmds/top_cmd.cpp
+++ b/src/zen/cmds/top_cmd.cpp
@@ -27,7 +27,7 @@ TopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZenServerState State;
if (!State.InitializeReadOnly())
{
- ZEN_CONSOLE("no Zen state found");
+ ZEN_CONSOLE("No Zen state found");
return 0;
}
@@ -75,7 +75,7 @@ PsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZenServerState State;
if (!State.InitializeReadOnly())
{
- ZEN_CONSOLE("no Zen state found");
+ ZEN_CONSOLE("No Zen state found");
return 0;
}
diff --git a/src/zen/cmds/trace_cmd.cpp b/src/zen/cmds/trace_cmd.cpp
index dacc13ebf..85caf33b8 100644
--- a/src/zen/cmds/trace_cmd.cpp
+++ b/src/zen/cmds/trace_cmd.cpp
@@ -48,7 +48,7 @@ TraceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- ZEN_ERROR("trace stop failed: {}", Response.ToText());
+ ZEN_CONSOLE_ERROR("trace stop failed: {}", Response.ToText());
return 1;
}
}
@@ -72,7 +72,7 @@ TraceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- ZEN_ERROR("trace start failed: {}", Response.ToText());
+ ZEN_CONSOLE_ERROR("trace start failed: {}", Response.ToText());
return 1;
}
}
@@ -84,7 +84,7 @@ TraceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- ZEN_ERROR("trace status failed: {}", Response.ToText());
+ ZEN_CONSOLE_ERROR("trace status failed: {}", Response.ToText());
}
return 1;
diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp
index d0763701e..fd330f616 100644
--- a/src/zen/cmds/up_cmd.cpp
+++ b/src/zen/cmds/up_cmd.cpp
@@ -98,7 +98,7 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
if (Server.IsRunning())
{
- ZEN_ERROR("zen server launch failed (timed out), terminating");
+ ZEN_CONSOLE_ERROR("Zen server launch failed (timed out), terminating");
Server.Terminate();
if (!m_ShowConsole)
{
@@ -121,7 +121,7 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- ZEN_CONSOLE("zen server up");
+ ZEN_CONSOLE("Zen server up");
}
}
return 0;
@@ -157,14 +157,14 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
if (!IsFile(m_DataDir / ".lock"))
{
- ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir);
+ ZEN_CONSOLE("Lock file does not exist in directory '{}'", m_DataDir);
return 1;
}
LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock")));
std::string Reason;
if (!ValidateLockFileInfo(Info, Reason))
{
- ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
+ ZEN_CONSOLE("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
return 1;
}
Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort);
@@ -172,18 +172,18 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (!Entry)
{
- ZEN_WARN("no zen server instance to add sponsor process to");
+ ZEN_CONSOLE_ERROR("No zen server instance to add sponsor process to");
return 1;
}
// Sponsor processes are checked every second, so 2 second wait time should be enough
if (!Entry->AddSponsorProcess(m_OwnerPid, 2000))
{
- ZEN_WARN("unable to add sponsor process to running zen server instance");
+ ZEN_CONSOLE_ERROR("Unable to add sponsor process to running zen server instance");
return 1;
}
- ZEN_CONSOLE("added sponsor process {} to running instance {} on port {}", m_OwnerPid, Entry->Pid.load(), m_Port);
+ ZEN_CONSOLE("Added sponsor process {} to running instance {} on port {}", m_OwnerPid, Entry->Pid.load(), m_Port);
return 0;
}
@@ -224,14 +224,14 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
if (!IsFile(m_DataDir / ".lock"))
{
- ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir);
+ ZEN_CONSOLE_ERROR("Lock file does not exist in directory '{}'", m_DataDir);
return 1;
}
LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock")));
std::string Reason;
if (!ValidateLockFileInfo(Info, Reason))
{
- ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
+ ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
return 1;
}
Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort);
@@ -272,7 +272,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else if (Ec)
{
- ZEN_CONSOLE("requesting shutdown of server on port {} failed. Reason: '{}'", EntryPort, Ec.message());
+ ZEN_CONSOLE_WARN("Requesting shutdown of server on port {} failed. Reason: '{}'", EntryPort, Ec.message());
}
}
catch (const std::exception& Ex)
@@ -284,7 +284,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
// handle to determine when the server has shut down. Thus we signal that we would like
// a shutdown via the shutdown flag and the check if the entry is still running.
- ZEN_CONSOLE("requesting detached shutdown of server on port {}", EntryPort);
+ ZEN_CONSOLE("Requesting detached shutdown of server on port {}", EntryPort);
Entry->SignalShutdownRequest();
Stopwatch Timer;
@@ -294,12 +294,12 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
Entry = Instance.Lookup(EntryPort);
if (Entry == nullptr)
{
- ZEN_CONSOLE("shutdown complete");
+ ZEN_CONSOLE("Shutdown complete");
return 0;
}
if (Entry->Pid.load() != ServerProcessPid)
{
- ZEN_CONSOLE("shutdown complete");
+ ZEN_CONSOLE("Shutdown complete");
return 0;
}
Sleep(100);
@@ -313,35 +313,36 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ProcessHandle RunningProcess;
if (std::error_code Ec = FindProcess(ServerExePath, RunningProcess); !Ec, /*IncludeSelf*/ false)
{
- ZEN_WARN("attempting hard terminate of zen process with pid ({})", RunningProcess.Pid());
+ ZEN_CONSOLE_WARN("Attempting hard terminate of zen process with pid ({})", RunningProcess.Pid());
try
{
if (RunningProcess.Terminate(0))
{
- ZEN_CONSOLE("terminate complete");
+ ZEN_CONSOLE("Terminate complete");
return 0;
}
- ZEN_CONSOLE("failed to terminate server, still running");
+ ZEN_CONSOLE_ERROR("Failed to terminate server, still running");
return 1;
}
catch (const std::exception& Ex)
{
- ZEN_CONSOLE("failed to terminate server: '{}'", Ex.what());
+ ZEN_CONSOLE_ERROR("Failed to terminate server: '{}'", Ex.what());
return 1;
}
}
else
{
- ZEN_CONSOLE("Failed to find process '{}', reason: {}", ServerExePath.string(), Ec.message());
+ ZEN_CONSOLE_WARN("Failed to find process '{}', reason: {}", ServerExePath.string(), Ec.message());
}
}
else if (Entry)
{
- ZEN_CONSOLE("failed to shutdown of server on port {}, use --force to hard terminate process", Entry->DesiredListenPort.load());
+ ZEN_CONSOLE_ERROR("Failed to shutdown of server on port {}, use --force to hard terminate process",
+ Entry->DesiredListenPort.load());
return 1;
}
- ZEN_CONSOLE("no zen server to bring down");
+ ZEN_CONSOLE("No zen server to bring down");
return 0;
}
diff --git a/src/zen/cmds/version_cmd.cpp b/src/zen/cmds/version_cmd.cpp
index 7dfa125e4..1be02331d 100644
--- a/src/zen/cmds/version_cmd.cpp
+++ b/src/zen/cmds/version_cmd.cpp
@@ -72,7 +72,7 @@ VersionCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
HttpClient::Response Response = Client.Get(VersionRequest, {}, Parameters);
if (!Response.IsSuccess())
{
- ZEN_ERROR("{} failed: {}", VersionRequest, Response.ErrorMessage(""sv));
+ ZEN_CONSOLE_ERROR("{} failed: {}", VersionRequest, Response.ErrorMessage(""sv));
return 1;
}
Version = Response.AsText();
diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp
index fcc18df2b..5a9d0174e 100644
--- a/src/zen/cmds/wipe_cmd.cpp
+++ b/src/zen/cmds/wipe_cmd.cpp
@@ -479,7 +479,7 @@ namespace {
{
if (!Quiet)
{
- ZEN_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what());
+ ZEN_CONSOLE_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what());
}
CleanWipe = false;
FailedDeleteCount++;
@@ -587,18 +587,7 @@ WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
}
- try
- {
- CleanDirectory(m_Directory, {}, !m_KeepReadOnlyFiles, m_Dryrun);
- }
- catch (std::exception& Ex)
- {
- if (!m_Quiet)
- {
- ZEN_ERROR("{}", Ex.what());
- }
- return 3;
- }
+ CleanDirectory(m_Directory, {}, !m_KeepReadOnlyFiles, m_Dryrun);
return 0;
}
diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp
index 773734f12..3930e56b7 100644
--- a/src/zen/cmds/workspaces_cmd.cpp
+++ b/src/zen/cmds/workspaces_cmd.cpp
@@ -185,7 +185,7 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
HttpClient Http(m_HostName);
if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result)
{
- ZEN_CONSOLE("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv));
+ ZEN_CONSOLE_ERROR("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv));
}
}
ZEN_CONSOLE("Added/updated workspace {}", m_Id);
@@ -193,7 +193,7 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- ZEN_CONSOLE("Workspace {} already exists", m_Id);
+ ZEN_CONSOLE_WARN("Workspace {} already exists", m_Id);
return 0;
}
}
@@ -206,7 +206,7 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
static std::vector<Workspaces::WorkspaceConfiguration> Configs = Workspaces::ReadConfig(Log(), StatePath, Error);
if (!Error.empty())
{
- ZEN_CONSOLE("Failed to read workspaces state from '{}'. Reason: '{}'", StatePath, Error);
+ ZEN_CONSOLE_ERROR("Failed to read workspaces state from '{}'. Reason: '{}'", StatePath, Error);
}
else
{
@@ -233,7 +233,7 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- ZEN_CONSOLE("Workspace {} not found", m_Id);
+ ZEN_CONSOLE_WARN("Workspace {} not found", m_Id);
}
}
}
@@ -257,14 +257,14 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
HttpClient Http(m_HostName);
if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result)
{
- ZEN_CONSOLE("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv));
+ ZEN_CONSOLE_ERROR("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv));
}
}
ZEN_CONSOLE("Removed workspace {}", m_Id);
}
else
{
- ZEN_CONSOLE("Workspace {} does not exist", m_Id);
+ ZEN_CONSOLE_WARN("Workspace {} does not exist", m_Id);
}
return 0;
}
@@ -428,7 +428,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
Workspaces::WorkspaceConfiguration WorkspaceConfig = Workspaces::FindWorkspace(Log(), StatePath, WorkspaceId);
if (WorkspaceConfig.Id == Oid::Zero)
{
- ZEN_CONSOLE("Workspace {} does not exist", m_WorkspaceId);
+ ZEN_CONSOLE_ERROR("Workspace {} does not exist", m_WorkspaceId);
return 0;
}
m_WorkspaceRoot = WorkspaceConfig.RootPath;
@@ -481,7 +481,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
HttpClient Http(m_HostName);
if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result)
{
- ZEN_CONSOLE("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv));
+ ZEN_CONSOLE_ERROR("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv));
}
}
ZEN_CONSOLE("Created workspace share {}", m_ShareId);
@@ -602,7 +602,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
HttpClient Http(m_HostName);
if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result)
{
- ZEN_CONSOLE("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv));
+ ZEN_CONSOLE_ERROR("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv));
}
}
ZEN_CONSOLE("Removed workspace share {}", m_ShareId);
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index accd0d105..240ba2a81 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -1030,10 +1030,35 @@ main(int argc, char** argv)
}
catch (const OptionParseException& Ex)
{
- std::string help = VerbOptions.help();
-
- printf("Error parsing arguments for command '%s': %s\n\n%s", SubCommand.c_str(), Ex.what(), help.c_str());
-
+ ZEN_CONSOLE_ERROR("Invalid arguments for command '{}': {}\n\n{}", SubCommand, Ex.what(), VerbOptions.help());
+ exit(11);
+ }
+ catch (const std::system_error& Ex)
+ {
+ if (IsOOD(Ex))
+ {
+ ZEN_CONSOLE_ERROR("Operation failed due to out of disk space: {}", Ex.what());
+ exit(3);
+ }
+ else if (IsOOM(Ex))
+ {
+ ZEN_CONSOLE_ERROR("Operation failed due to out of memory: {}", Ex.what());
+ exit(3);
+ }
+ else
+ {
+ ZEN_CONSOLE_ERROR("Operation failed due to system error: {} ({})\n", Ex.what(), Ex.code() ? Ex.code().value() : 0);
+ exit(Ex.code() ? Ex.code().value() : 10);
+ }
+ }
+ catch (const HttpClientError& Ex)
+ {
+ ZEN_CONSOLE_ERROR("Operation failed due to a http error: {}", Ex.what());
+ exit(Ex.m_Error != 0 ? Ex.m_Error : (int)Ex.m_ResponseCode);
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_CONSOLE_ERROR("Operation failed due to: {}\n", Ex.what());
exit(11);
}
}
@@ -1051,14 +1076,30 @@ main(int argc, char** argv)
}
catch (const std::system_error& Ex)
{
- printf("System Error: %s\n", Ex.what());
-
- return Ex.code() ? Ex.code().value() : 10;
+ if (IsOOD(Ex))
+ {
+ printf("Operation failed due to out of disk space: %s", Ex.what());
+ return 3;
+ }
+ else if (IsOOM(Ex))
+ {
+ printf("Operation failed due to out of memory: %s", Ex.what());
+ return 3;
+ }
+ else
+ {
+ printf("Operation failed due to system error: %s (%d)\n", Ex.what(), Ex.code() ? Ex.code().value() : 0);
+ return Ex.code() ? Ex.code().value() : 10;
+ }
+ }
+ catch (const HttpClientError& Ex)
+ {
+ printf("Operation failed due to a http error: %s", Ex.what());
+ return Ex.m_Error != 0 ? Ex.m_Error : (int)Ex.m_ResponseCode;
}
catch (const std::exception& Ex)
{
- printf("Error: %s\n", Ex.what());
-
+ printf("Operation failed due to: %s\n", Ex.what());
return 11;
}
diff --git a/src/zencore/include/zencore/logging.h b/src/zencore/include/zencore/logging.h
index bffbe84b4..afbbbd3ee 100644
--- a/src/zencore/include/zencore/logging.h
+++ b/src/zencore/include/zencore/logging.h
@@ -59,7 +59,6 @@ struct LogCategory
zen::LoggerRef LoggerRef;
};
-void EmitConsoleLogMessage(int LogLevel, std::string_view Message);
void EmitConsoleLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args);
void EmitLogMessage(LoggerRef& Logger, int LogLevel, std::string_view Message);
void EmitLogMessage(LoggerRef& Logger, const SourceLocation& Location, int LogLevel, std::string_view Message);
diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp
index 5885587ad..685c79d82 100644
--- a/src/zencore/logging.cpp
+++ b/src/zencore/logging.cpp
@@ -122,11 +122,34 @@ EmitConsoleLogMessage(int LogLevel, const std::string_view Message)
ConsoleLog().SpdLogger->log(InLevel, Message);
}
+#define ZEN_COLOR_YELLOW "\033[0;33m"
+#define ZEN_COLOR_RED "\033[0;31m"
+#define ZEN_BRIGHT_COLOR_RED "\033[1;31m"
+#define ZEN_COLOR_RESET "\033[0m"
+
void
EmitConsoleLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args)
{
ZEN_MEMSCOPE(ELLMTag::Logging);
zen::logging::LoggingContext LogCtx;
+
+ // We are not using a format option for console which include log level since it would interfere with normal console output
+
+ const spdlog::level::level_enum InLevel = (spdlog::level::level_enum)LogLevel;
+ switch (InLevel)
+ {
+ case spdlog::level::level_enum::warn:
+ fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_COLOR_YELLOW "Warning: " ZEN_COLOR_RESET);
+ break;
+ case spdlog::level::level_enum::err:
+ fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_BRIGHT_COLOR_RED "Error: " ZEN_COLOR_RESET);
+ break;
+ case spdlog::level::level_enum::critical:
+ fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_COLOR_RED "Critical: " ZEN_COLOR_RESET);
+ break;
+ default:
+ break;
+ }
fmt::vformat_to(fmt::appender(LogCtx.MessageBuffer), Format, Args);
zen::logging::EmitConsoleLogMessage(LogLevel, LogCtx.Message());
}
diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index 19ac3a216..68f1c602e 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -85,7 +85,7 @@ namespace {
//////////////////////////////////////////////////////////////////////////
HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore,
- GetCidStoreFunc&& GetCidStore,
+ CidStore& InCidStore,
HttpStatsService& StatsService,
HttpStatusService& StatusService,
UpstreamCache& UpstreamCache,
@@ -95,10 +95,11 @@ HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCach
, m_CacheStore(InCacheStore)
, m_StatsService(StatsService)
, m_StatusService(StatusService)
+, m_CidStore(InCidStore)
, m_UpstreamCache(UpstreamCache)
, m_DiskWriteBlocker(InDiskWriteBlocker)
, m_OpenProcessCache(InOpenProcessCache)
-, m_RpcHandler(m_Log, m_CacheStats, UpstreamCache, InCacheStore, std::move(GetCidStore), InDiskWriteBlocker)
+, m_RpcHandler(m_Log, m_CacheStats, UpstreamCache, InCacheStore, InCidStore, InDiskWriteBlocker)
{
m_StatsService.RegisterHandler("z$", *this);
m_StatusService.RegisterHandler("z$", *this);
@@ -224,9 +225,6 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
for (const auto& NamespaceIt : ValueDetails.Namespaces)
{
const std::string& Namespace = NamespaceIt.first;
-
- CidStore& ChunkStore = m_RpcHandler.GetCidStore(Namespace);
-
for (const auto& BucketIt : NamespaceIt.second.Buckets)
{
const std::string& Bucket = BucketIt.first;
@@ -236,7 +234,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
{
for (const IoHash& Hash : ValueIt.second.Attachments)
{
- IoBuffer Payload = ChunkStore.FindChunkByCid(Hash);
+ IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
CSVWriter << "\r\n"
<< Namespace << "," << Bucket << "," << ValueIt.first.ToHexString() << ", " << Hash.ToHexString()
<< ", " << gsl::narrow<uint64_t>(Payload.GetSize());
@@ -254,7 +252,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
size_t AttachmentsSize = 0;
for (const IoHash& Hash : ValueIt.second.Attachments)
{
- IoBuffer Payload = ChunkStore.FindChunkByCid(Hash);
+ IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
AttachmentsSize += Payload.GetSize();
}
CSVWriter << ", " << gsl::narrow<uint64_t>(AttachmentsSize);
@@ -276,9 +274,6 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
for (const auto& NamespaceIt : ValueDetails.Namespaces)
{
const std::string& Namespace = NamespaceIt.first;
-
- CidStore& ChunkStore = m_RpcHandler.GetCidStore(Namespace);
-
Cbo.BeginObject();
{
Cbo.AddString("name", Namespace);
@@ -321,7 +316,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
{
Cbo.BeginObject();
Cbo.AddHash("cid", Hash);
- IoBuffer Payload = ChunkStore.FindChunkByCid(Hash);
+ IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
Cbo.AddInteger("size", gsl::narrow<uint64_t>(Payload.GetSize()));
Cbo.EndObject();
}
@@ -335,7 +330,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request)
size_t AttachmentsSize = 0;
for (const IoHash& Hash : ValueIt.second.Attachments)
{
- IoBuffer Payload = ChunkStore.FindChunkByCid(Hash);
+ IoBuffer Payload = m_CidStore.FindChunkByCid(Hash);
AttachmentsSize += Payload.GetSize();
}
Cbo.AddInteger("attachmentssize", gsl::narrow<uint64_t>(AttachmentsSize));
@@ -610,8 +605,6 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque
ResponseWriter.AddInteger("EntryCount", Info->DiskLayerInfo.EntryCount);
- CidStore& ChunkStore = m_RpcHandler.GetCidStore(NamespaceName);
-
if (auto Buckets = HttpServerRequest::Decode(Request.GetQueryParams().GetValue("bucketsizes")); !Buckets.empty())
{
ResponseWriter.BeginObject("BucketSizes");
@@ -670,7 +663,7 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque
uint64_t AttachmentsSize = 0;
- ChunkStore.IterateChunks(
+ m_CidStore.IterateChunks(
AllAttachments,
[&](size_t Index, const IoBuffer& Payload) {
ZEN_UNUSED(Index);
@@ -738,8 +731,6 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request,
ResponseWriter.AddInteger("DiskEntryCount", Info->DiskLayerInfo.EntryCount);
- CidStore& ChunkStore = m_RpcHandler.GetCidStore(NamespaceName);
-
if (auto GetBucketSize = Request.GetQueryParams().GetValue("bucketsize"); GetBucketSize == "true")
{
CacheContentStats ContentStats;
@@ -766,7 +757,7 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request,
WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background);
- ChunkStore.IterateChunks(
+ m_CidStore.IterateChunks(
ContentStats.Attachments,
[&](size_t Index, const IoBuffer& Payload) {
ZEN_UNUSED(Index);
@@ -841,8 +832,6 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()};
Stopwatch Timer;
- CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace);
-
if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal) &&
m_CacheStore.Get(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue))
{
@@ -857,17 +846,17 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
uint32_t MissingCount = 0;
CbObjectView CacheRecord(ClientResultValue.Value.Data());
- CacheRecord.IterateAttachments([this, &ChunkStore, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) {
+ CacheRecord.IterateAttachments([this, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) {
if (SkipData)
{
- if (!ChunkStore.ContainsChunk(AttachmentHash.AsHash()))
+ if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash()))
{
MissingCount++;
}
}
else
{
- if (IoBuffer Chunk = ChunkStore.FindChunkByCid(AttachmentHash.AsHash()))
+ if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash()))
{
CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk));
if (Compressed)
@@ -967,8 +956,6 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
{
Success = true;
- CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace);
-
ClientResultValue.Value = UpstreamResult.Value;
ClientResultValue.Value.SetContentType(AcceptType);
@@ -1019,7 +1006,6 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
CacheRecord.IterateAttachments([this,
&Package,
&Ref,
- &ChunkStore,
&WriteAttachmentBuffers,
&WriteRawHashes,
&ReferencedAttachments,
@@ -1054,12 +1040,12 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
{
if (SkipData)
{
- if (ChunkStore.ContainsChunk(Hash))
+ if (m_CidStore.ContainsChunk(Hash))
{
Count.Valid++;
}
}
- else if (IoBuffer Chunk = ChunkStore.FindChunkByCid(Hash))
+ else if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Hash))
{
CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk));
if (Compressed)
@@ -1101,7 +1087,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
if (!WriteAttachmentBuffers.empty())
{
std::vector<CidStore::InsertResult> InsertResults =
- ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes);
+ m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes);
for (const CidStore::InsertResult& Result : InsertResults)
{
if (Result.New)
@@ -1268,10 +1254,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
if (HasUpstream && EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote))
{
- CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace);
- m_UpstreamCache.EnqueueUpstream(
- {.Type = ContentType, .Namespace = Ref.Namespace, .Key = {Ref.BucketSegment, Ref.HashKey}},
- [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); });
+ m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Namespace = Ref.Namespace, .Key = {Ref.BucketSegment, Ref.HashKey}});
}
ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' in {}",
@@ -1305,18 +1288,15 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
std::vector<IoHash> ReferencedAttachments;
int32_t TotalCount = 0;
- CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace);
-
- CacheRecord.IterateAttachments(
- [this, &ChunkStore, &TotalCount, &ValidAttachments, &ReferencedAttachments](CbFieldView AttachmentHash) {
- const IoHash Hash = AttachmentHash.AsHash();
- ReferencedAttachments.push_back(Hash);
- if (ChunkStore.ContainsChunk(Hash))
- {
- ValidAttachments.emplace_back(Hash);
- }
- TotalCount++;
- });
+ CacheRecord.IterateAttachments([this, &TotalCount, &ValidAttachments, &ReferencedAttachments](CbFieldView AttachmentHash) {
+ const IoHash Hash = AttachmentHash.AsHash();
+ ReferencedAttachments.push_back(Hash);
+ if (m_CidStore.ContainsChunk(Hash))
+ {
+ ValidAttachments.emplace_back(Hash);
+ }
+ TotalCount++;
+ });
const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal);
@@ -1350,12 +1330,10 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
CachePolicy Policy = PolicyFromUrl;
if (HasUpstream && EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord)
{
- m_UpstreamCache.EnqueueUpstream(
- {.Type = ZenContentType::kCbObject,
- .Namespace = Ref.Namespace,
- .Key = {Ref.BucketSegment, Ref.HashKey},
- .ValueContentIds = std::move(ValidAttachments)},
- [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); });
+ m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbObject,
+ .Namespace = Ref.Namespace,
+ .Key = {Ref.BucketSegment, Ref.HashKey},
+ .ValueContentIds = std::move(ValidAttachments)});
}
Request.WriteResponse(HttpResponseCode::Created);
@@ -1388,46 +1366,38 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
WriteAttachmentBuffers.reserve(NumAttachments);
WriteRawHashes.reserve(NumAttachments);
- CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace);
-
- CacheRecord.IterateAttachments([this,
- &Ref,
- &Package,
- &ChunkStore,
- &WriteAttachmentBuffers,
- &WriteRawHashes,
- &ValidAttachments,
- &ReferencedAttachments,
- &Count](CbFieldView HashView) {
- const IoHash Hash = HashView.AsHash();
- ReferencedAttachments.push_back(Hash);
- if (const CbAttachment* Attachment = Package.FindAttachment(Hash))
- {
- if (Attachment->IsCompressedBinary())
+ CacheRecord.IterateAttachments(
+ [this, &Ref, &Package, &WriteAttachmentBuffers, &WriteRawHashes, &ValidAttachments, &ReferencedAttachments, &Count](
+ CbFieldView HashView) {
+ const IoHash Hash = HashView.AsHash();
+ ReferencedAttachments.push_back(Hash);
+ if (const CbAttachment* Attachment = Package.FindAttachment(Hash))
{
- WriteAttachmentBuffers.emplace_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer());
- WriteRawHashes.push_back(Hash);
- ValidAttachments.emplace_back(Hash);
- Count.Valid++;
+ if (Attachment->IsCompressedBinary())
+ {
+ WriteAttachmentBuffers.emplace_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer());
+ WriteRawHashes.push_back(Hash);
+ ValidAttachments.emplace_back(Hash);
+ Count.Valid++;
+ }
+ else
+ {
+ ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed",
+ Ref.Namespace,
+ Ref.BucketSegment,
+ Ref.HashKey,
+ ToString(HttpContentType::kCbPackage),
+ Hash);
+ Count.Invalid++;
+ }
}
- else
+ else if (m_CidStore.ContainsChunk(Hash))
{
- ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed",
- Ref.Namespace,
- Ref.BucketSegment,
- Ref.HashKey,
- ToString(HttpContentType::kCbPackage),
- Hash);
- Count.Invalid++;
+ ValidAttachments.emplace_back(Hash);
+ Count.Valid++;
}
- }
- else if (ChunkStore.ContainsChunk(Hash))
- {
- ValidAttachments.emplace_back(Hash);
- Count.Valid++;
- }
- Count.Total++;
- });
+ Count.Total++;
+ });
if (Count.Invalid > 0)
{
@@ -1451,7 +1421,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
if (!WriteAttachmentBuffers.empty())
{
- std::vector<CidStore::InsertResult> InsertResults = ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes);
+ std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes);
for (const CidStore::InsertResult& InsertResult : InsertResults)
{
if (InsertResult.New)
@@ -1478,12 +1448,10 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
if (HasUpstream && EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord)
{
- m_UpstreamCache.EnqueueUpstream(
- {.Type = ZenContentType::kCbPackage,
- .Namespace = Ref.Namespace,
- .Key = {Ref.BucketSegment, Ref.HashKey},
- .ValueContentIds = std::move(ValidAttachments)},
- [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); });
+ m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage,
+ .Namespace = Ref.Namespace,
+ .Key = {Ref.BucketSegment, Ref.HashKey},
+ .ValueContentIds = std::move(ValidAttachments)});
}
Request.WriteResponse(HttpResponseCode::Created);
@@ -1517,9 +1485,7 @@ HttpStructuredCacheService::HandleGetCacheChunk(HttpServerRequest& Request, cons
{
Stopwatch Timer;
- CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace);
-
- IoBuffer Value = ChunkStore.FindChunkByCid(Ref.ValueContentId);
+ IoBuffer Value = m_CidStore.FindChunkByCid(Ref.ValueContentId);
const UpstreamEndpointInfo* Source = nullptr;
CachePolicy Policy = PolicyFromUrl;
@@ -1541,7 +1507,7 @@ HttpStructuredCacheService::HandleGetCacheChunk(HttpServerRequest& Request, cons
{
if (AreDiskWritesAllowed())
{
- ChunkStore.AddChunk(UpstreamResult.Value, RawHash);
+ m_CidStore.AddChunk(UpstreamResult.Value, RawHash);
}
Source = UpstreamResult.Source;
}
@@ -1635,9 +1601,7 @@ HttpStructuredCacheService::HandlePutCacheChunk(HttpServerRequest& Request, cons
"ValueContentId does not match attachment hash"sv);
}
- CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace);
-
- CidStore::InsertResult Result = ChunkStore.AddChunk(Body, RawHash);
+ CidStore::InsertResult Result = m_CidStore.AddChunk(Body, RawHash);
ZEN_DEBUG("PUTCACHECHUNK - '{}/{}/{}/{}' {} '{}' ({}) in {}",
Ref.Namespace,
@@ -1852,51 +1816,16 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
EmitSnapshot("requests", m_HttpRequests, Cbo);
- const uint64_t HitCount = m_CacheStats.HitCount;
- const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount;
- const uint64_t MissCount = m_CacheStats.MissCount;
- const uint64_t WriteCount = m_CacheStats.WriteCount;
- const uint64_t BadRequestCount = m_CacheStats.BadRequestCount;
-
- uint64_t TotalChunkHitCount = 0;
- uint64_t TotalChunkMissCount = 0;
- uint64_t TotalChunkWriteCount = 0;
- CidStoreSize TotalCidSize;
-
- tsl::robin_map<CidStore*, std::string> UniqueStores;
- {
- std::vector<std::string> NamespaceNames = m_CacheStore.GetNamespaces();
-
- for (const std::string& NamespaceName : NamespaceNames)
- {
- CidStore* Store = &m_RpcHandler.GetCidStore(NamespaceName);
- if (auto It = UniqueStores.find(Store); It == UniqueStores.end())
- {
- UniqueStores.insert_or_assign(Store, NamespaceName);
- }
- else
- {
- UniqueStores.insert_or_assign(Store, std::string{});
- }
- }
-
- for (auto It : UniqueStores)
- {
- CidStore* ChunkStore = It.first;
-
- CidStoreStats StoreStats = ChunkStore->Stats();
- CidStoreSize StoreSize = ChunkStore->TotalSize();
-
- TotalChunkHitCount += StoreStats.HitCount;
- TotalChunkMissCount += StoreStats.MissCount;
- TotalChunkWriteCount += StoreStats.WriteCount;
-
- TotalCidSize.TinySize += StoreSize.TinySize;
- TotalCidSize.SmallSize += StoreSize.SmallSize;
- TotalCidSize.LargeSize += StoreSize.LargeSize;
- TotalCidSize.TotalSize += StoreSize.TotalSize;
- }
- }
+ const uint64_t HitCount = m_CacheStats.HitCount;
+ const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount;
+ const uint64_t MissCount = m_CacheStats.MissCount;
+ const uint64_t WriteCount = m_CacheStats.WriteCount;
+ const uint64_t BadRequestCount = m_CacheStats.BadRequestCount;
+ struct CidStoreStats StoreStats = m_CidStore.Stats();
+ const uint64_t ChunkHitCount = StoreStats.HitCount;
+ const uint64_t ChunkMissCount = StoreStats.MissCount;
+ const uint64_t ChunkWriteCount = StoreStats.WriteCount;
+ const uint64_t TotalCount = HitCount + MissCount;
const uint64_t RpcRequests = m_CacheStats.RpcRequests;
const uint64_t RpcRecordRequests = m_CacheStats.RpcRecordRequests;
@@ -1906,11 +1835,17 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
const uint64_t RpcChunkRequests = m_CacheStats.RpcChunkRequests;
const uint64_t RpcChunkBatchRequests = m_CacheStats.RpcChunkBatchRequests;
+ const CidStoreSize CidSize = m_CidStore.TotalSize();
const CacheStoreSize CacheSize = m_CacheStore.TotalSize();
bool ShowCidStoreStats = Request.GetQueryParams().GetValue("cidstorestats") == "true";
bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true";
+ CidStoreStats CidStoreStats = {};
+ if (ShowCidStoreStats)
+ {
+ CidStoreStats = m_CidStore.Stats();
+ }
ZenCacheStore::CacheStoreStats CacheStoreStats = {};
if (ShowCacheStoreStats)
{
@@ -1945,7 +1880,6 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
Cbo.EndObject();
Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount;
- const uint64_t TotalCount = HitCount + MissCount;
Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0);
if (m_UpstreamCache.IsActive())
@@ -1956,9 +1890,7 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0);
}
- Cbo << "cidhits" << TotalChunkHitCount << "cidmisses" << TotalChunkMissCount << "cidwrites" << TotalChunkWriteCount;
- const uint64_t TotalChunkCount = TotalChunkHitCount + TotalChunkMissCount;
- Cbo << "cidhit_ratio" << (TotalChunkHitCount ? (double(TotalChunkCount) / double(TotalChunkHitCount)) : 0.0);
+ Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount;
if (ShowCacheStoreStats)
{
@@ -2067,58 +1999,20 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request)
{
Cbo.BeginObject("size");
{
- Cbo << "tiny" << TotalCidSize.TinySize;
- Cbo << "small" << TotalCidSize.SmallSize;
- Cbo << "large" << TotalCidSize.LargeSize;
- Cbo << "total" << TotalCidSize.TotalSize;
+ Cbo << "tiny" << CidSize.TinySize;
+ Cbo << "small" << CidSize.SmallSize;
+ Cbo << "large" << CidSize.LargeSize;
+ Cbo << "total" << CidSize.TotalSize;
}
Cbo.EndObject();
if (ShowCidStoreStats)
{
Cbo.BeginObject("store");
-
- auto OutputStats = [&](CidStore& ChunkStore) {
- CidStoreStats StoreStats = ChunkStore.Stats();
- Cbo << "hits" << StoreStats.HitCount << "misses" << StoreStats.MissCount << "writes" << StoreStats.WriteCount;
- const uint64_t Count = StoreStats.HitCount + StoreStats.MissCount;
- Cbo << "hit_ratio" << (Count ? (double(StoreStats.HitCount) / double(Count)) : 0.0);
- EmitSnapshot("read", StoreStats.FindChunkOps, Cbo);
- EmitSnapshot("write", StoreStats.AddChunkOps, Cbo);
- };
-
- if (UniqueStores.size() > 1)
- {
- Cbo.BeginArray("namespaces");
- for (auto It : UniqueStores)
- {
- CidStore* ChunkStore = It.first;
- const std::string& Namespace = It.second;
- CidStoreSize ChunkStoreSize = ChunkStore->TotalSize();
- Cbo.BeginObject();
- {
- Cbo << "namespace" << Namespace;
- Cbo.BeginObject("stats");
- OutputStats(*ChunkStore);
- Cbo.EndObject();
-
- Cbo.BeginObject("size");
- {
- Cbo << "tiny" << ChunkStoreSize.TinySize;
- Cbo << "small" << ChunkStoreSize.SmallSize;
- Cbo << "large" << ChunkStoreSize.LargeSize;
- Cbo << "total" << ChunkStoreSize.TotalSize;
- }
- Cbo.EndObject();
- }
- Cbo.EndObject();
- }
- Cbo.EndArray(); // namespaces
- }
- else if (UniqueStores.size() != 0)
- {
- OutputStats(*UniqueStores.begin()->first);
- }
+ Cbo << "hits" << CidStoreStats.HitCount << "misses" << CidStoreStats.MissCount << "writes" << CidStoreStats.WriteCount;
+ EmitSnapshot("read", CidStoreStats.FindChunkOps, Cbo);
+ EmitSnapshot("write", CidStoreStats.AddChunkOps, Cbo);
+ // EmitSnapshot("exists", CidStoreStats.ContainChunkOps, Cbo);
Cbo.EndObject();
}
}
diff --git a/src/zenserver/cache/httpstructuredcache.h b/src/zenserver/cache/httpstructuredcache.h
index d46ca145d..cb822f117 100644
--- a/src/zenserver/cache/httpstructuredcache.h
+++ b/src/zenserver/cache/httpstructuredcache.h
@@ -70,10 +70,8 @@ namespace cache {
class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider, public IHttpStatusProvider
{
public:
- typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc;
-
HttpStructuredCacheService(ZenCacheStore& InCacheStore,
- GetCidStoreFunc&& GetCidStore,
+ CidStore& InCidStore,
HttpStatsService& StatsService,
HttpStatusService& StatusService,
UpstreamCache& UpstreamCache,
@@ -117,6 +115,7 @@ private:
ZenCacheStore& m_CacheStore;
HttpStatsService& m_StatsService;
HttpStatusService& m_StatusService;
+ CidStore& m_CidStore;
UpstreamCache& m_UpstreamCache;
metrics::OperationTiming m_HttpRequests;
metrics::OperationTiming m_UpstreamGetRequestTiming;
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 4049ae815..fb2d9b7f4 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -482,7 +482,8 @@ ParseConfigFile(const std::filesystem::path& Path,
LuaOptions.AddOption("server.pluginsconfigfile"sv, ServerOptions.PluginsConfigFile, "plugins-config"sv);
LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv);
LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv);
- LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "quiet"sv);
+ LuaOptions.AddOption("server.quiet"sv, ServerOptions.QuietConsole, "quiet"sv);
+ LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "noconsole"sv);
////// objectstore
LuaOptions.AddOption("server.objectstore.enabled"sv, ServerOptions.ObjectStoreEnabled, "objectstore-enabled"sv);
@@ -812,16 +813,17 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
// clang-format off
options.add_options("logging")
- ("abslog", "Path to log file", cxxopts::value<std::string>(AbsLogFile))
- ("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId))
- ("quiet", "Disable console logging", cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false"))
- ("log-trace", "Change selected loggers to level TRACE", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace]))
- ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug]))
- ("log-info", "Change selected loggers to level INFO", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info]))
- ("log-warn", "Change selected loggers to level WARN", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn]))
- ("log-error", "Change selected loggers to level ERROR", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err]))
- ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical]))
- ("log-off", "Change selected loggers to level OFF", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off]))
+ ("abslog", "Path to log file", cxxopts::value<std::string>(AbsLogFile))
+ ("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId))
+ ("quiet", "Configure console logger output to level WARN", cxxopts::value<bool>(ServerOptions.QuietConsole)->default_value("false"))
+ ("noconsole", "Disable console logging", cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false"))
+ ("log-trace", "Change selected loggers to level TRACE", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace]))
+ ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug]))
+ ("log-info", "Change selected loggers to level INFO", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info]))
+ ("log-warn", "Change selected loggers to level WARN", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn]))
+ ("log-error", "Change selected loggers to level ERROR", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err]))
+ ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical]))
+ ("log-off", "Change selected loggers to level OFF", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off]))
;
// clang-format on
@@ -1356,6 +1358,29 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
ZEN_TRACE_CPU("ConfigParse");
+ if (ServerOptions.QuietConsole)
+ {
+ bool HasExplicitConsoleLevel = false;
+ for (int i = 0; i < logging::level::LogLevelCount; ++i)
+ {
+ if (ServerOptions.Loggers[i].find("console") != std::string::npos)
+ {
+ HasExplicitConsoleLevel = true;
+ break;
+ }
+ }
+
+ if (!HasExplicitConsoleLevel)
+ {
+ std::string& WarnLoggers = ServerOptions.Loggers[logging::level::Warn];
+ if (!WarnLoggers.empty())
+ {
+ WarnLoggers += ",";
+ }
+ WarnLoggers += "console";
+ }
+ }
+
for (int i = 0; i < logging::level::LogLevelCount; ++i)
{
logging::ConfigureLogLevels(logging::level::LogLevel(i), ServerOptions.Loggers[i]);
@@ -1413,7 +1438,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
}
catch (const zen::OptionParseException& e)
{
- ZEN_CONSOLE_ERROR("Error parsing zenserver arguments: {}\n\n{}", e.what(), options.help());
+ ZEN_CONSOLE_ERROR("Invalid zenserver arguments: {}\n\n{}", e.what(), options.help());
throw;
}
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index 9e8787957..8380e72e7 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -206,6 +206,7 @@ struct ZenServerOptions
bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux)
bool ObjectStoreEnabled = false;
bool NoConsoleOutput = false; // Control default use of stdout for diagnostics
+ bool QuietConsole = false; // Configure console logger output to level WARN
std::string Loggers[zen::logging::level::LogLevelCount];
std::string ScrubOptions;
#if ZEN_WITH_TRACE
diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp
index f3d3377b0..34d9b05b7 100644
--- a/src/zenserver/diag/logging.cpp
+++ b/src/zenserver/diag/logging.cpp
@@ -27,6 +27,7 @@ InitializeServerLogging(const ZenServerOptions& InOptions)
.IsVerbose = false,
.IsTest = InOptions.IsTest,
.NoConsoleOutput = InOptions.NoConsoleOutput,
+ .QuietConsole = InOptions.QuietConsole,
.AbsLogFile = InOptions.AbsLogFile,
.LogId = InOptions.LogId};
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index d8922f885..d512d4894 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -139,21 +139,23 @@ ZenEntryPoint::Run()
{
if (Ec)
{
- ZEN_WARN(
- "Sponsor owner pid {} can not be checked for running state, reason: '{}'. Will not add sponsor to process "
- "listening to port {} (pid: {})",
- m_ServerOptions.OwnerPid,
- Ec.message(),
- m_ServerOptions.BasePort,
- Entry->Pid.load());
+ ZEN_WARN(ZEN_APP_NAME
+ " exiting, sponsor owner pid {} can not be checked for running state, reason: '{}'. Will not add sponsor "
+ "to process "
+ "listening to port {} (pid: {})",
+ m_ServerOptions.OwnerPid,
+ Ec.message(),
+ m_ServerOptions.BasePort,
+ Entry->Pid.load());
}
else
{
- ZEN_WARN(
- "Sponsor owner pid {} is no longer running, will not add sponsor to process listening to port {} (pid: {})",
- m_ServerOptions.OwnerPid,
- m_ServerOptions.BasePort,
- Entry->Pid.load());
+ ZEN_WARN(ZEN_APP_NAME
+ " exiting, sponsor owner pid {} is no longer running, will not add sponsor to process listening to port "
+ "{} (pid: {})",
+ m_ServerOptions.OwnerPid,
+ m_ServerOptions.BasePort,
+ Entry->Pid.load());
}
std::exit(1);
}
@@ -175,7 +177,7 @@ ZenEntryPoint::Run()
}
else
{
- ZEN_WARN("Failed to add sponsor owner pid {} to process listening to port {} (pid: {})",
+ ZEN_WARN(ZEN_APP_NAME " exiting, failed to add sponsor owner pid {} to process listening to port {} (pid: {})",
m_ServerOptions.OwnerPid,
m_ServerOptions.BasePort,
Entry->Pid.load());
@@ -184,7 +186,7 @@ ZenEntryPoint::Run()
}
else
{
- ZEN_WARN("Exiting since there is already a process listening to port {} (pid: {})",
+ ZEN_WARN(ZEN_APP_NAME " exiting, there is already a process listening to port {} (pid: {})",
m_ServerOptions.BasePort,
Entry->Pid.load());
std::exit(1);
@@ -208,14 +210,19 @@ ZenEntryPoint::Run()
if (Ec)
{
- ZEN_WARN("Unable to grab lock at '{}' (reason: '{}'), retrying", LockFilePath, Ec.message());
- Sleep(500);
+ ZEN_INFO(ZEN_APP_NAME " unable to grab lock at '{}' (reason: '{}'), retrying", LockFilePath, Ec.message());
+ Sleep(100);
m_LockFile.Create(LockFilePath, MakeLockData(false), Ec);
if (Ec)
{
- ZEN_WARN("ERROR: Unable to grab lock at '{}' (reason: '{}')", LockFilePath, Ec.message());
- std::exit(99);
+ ZEN_INFO(ZEN_APP_NAME " unable to grab lock at '{}' (reason: '{}'), retrying", LockFilePath, Ec.message());
+ Sleep(500);
+ if (Ec)
+ {
+ ZEN_WARN(ZEN_APP_NAME " exiting, unable to grab lock at '{}' (reason: '{}')", LockFilePath, Ec.message());
+ std::exit(99);
+ }
}
}
@@ -316,12 +323,14 @@ ZenEntryPoint::Run()
}
catch (const AssertException& AssertEx)
{
- ZEN_CRITICAL("Caught assert exception in main for process {}: {}", zen::GetCurrentProcessId(), AssertEx.FullDescription());
+ ZEN_CRITICAL(ZEN_APP_NAME " caught assert exception in main for process {}: {}",
+ zen::GetCurrentProcessId(),
+ AssertEx.FullDescription());
RequestApplicationExit(1);
}
catch (const std::system_error& e)
{
- ZEN_CRITICAL("Caught system error exception in main for process {}: {} ({})",
+ ZEN_CRITICAL(ZEN_APP_NAME " caught system error exception in main for process {}: {} ({})",
zen::GetCurrentProcessId(),
e.what(),
e.code().value());
@@ -329,7 +338,7 @@ ZenEntryPoint::Run()
}
catch (const std::exception& e)
{
- ZEN_CRITICAL("Caught exception in main for process {}: {}", zen::GetCurrentProcessId(), e.what());
+ ZEN_CRITICAL(ZEN_APP_NAME " caught exception in main for process {}: {}", zen::GetCurrentProcessId(), e.what());
RequestApplicationExit(1);
}
@@ -450,7 +459,7 @@ main(int argc, char* argv[])
{
if (IsDir(ServerOptions.DataDir))
{
- ZEN_CONSOLE_INFO("deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason);
+ ZEN_CONSOLE_INFO("Deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason);
DeleteDirectories(ServerOptions.DataDir);
}
}
@@ -463,7 +472,7 @@ main(int argc, char* argv[])
if (!ServerOptions.BaseSnapshotDir.empty())
{
- ZEN_CONSOLE_INFO("copying snapshot from '{}' into '{}", ServerOptions.BaseSnapshotDir, ServerOptions.DataDir);
+ ZEN_CONSOLE_INFO("Copying snapshot from '{}' into '{}", ServerOptions.BaseSnapshotDir, ServerOptions.DataDir);
CopyTree(ServerOptions.BaseSnapshotDir, ServerOptions.DataDir, {.EnableClone = true});
}
@@ -499,17 +508,17 @@ main(int argc, char* argv[])
catch (const OptionParseException& ParseEx)
{
// The parsing error already outputs all the details so no need to output the command line here
- fprintf(stderr, "ERROR: %s\n", ParseEx.what());
+ fprintf(stderr, ZEN_APP_NAME " ERROR: %s\n", ParseEx.what());
return 1;
}
catch (const AssertException& AssertEx)
{
- fprintf(stderr, "ERROR: Caught assert exception in main: '%s'", AssertEx.FullDescription().c_str());
+ fprintf(stderr, ZEN_APP_NAME " ERROR: Caught assert exception in main: '%s'", AssertEx.FullDescription().c_str());
return 1;
}
catch (const std::exception& Ex)
{
- fprintf(stderr, "ERROR: Caught exception in main: '%s'", Ex.what());
+ fprintf(stderr, ZEN_APP_NAME " ERROR: Caught exception in main: '%s'", Ex.what());
return 1;
}
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 9600133f3..317a419eb 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -235,11 +235,13 @@ namespace {
//////////////////////////////////////////////////////////////////////////
-HttpProjectService::HttpProjectService(ProjectStore* Projects,
+HttpProjectService::HttpProjectService(CidStore& Store,
+ ProjectStore* Projects,
HttpStatusService& StatusService,
HttpStatsService& StatsService,
AuthMgr& AuthMgr)
: m_Log(logging::Get("project"))
+, m_CidStore(Store)
, m_ProjectStore(Projects)
, m_StatusService(StatusService)
, m_StatsService(StatsService)
@@ -405,45 +407,8 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
{
ZEN_TRACE_CPU("ProjectService::Stats");
- bool ShowCidStoreStats = HttpReq.GetQueryParams().GetValue("cidstorestats") == "true";
-
- const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
- uint64_t TotalChunkHitCount = 0;
- uint64_t TotalChunkMissCount = 0;
- uint64_t TotalChunkWriteCount = 0;
- CidStoreSize TotalCidSize;
-
- tsl::robin_map<CidStore*, std::string> UniqueStores;
- {
- m_ProjectStore->IterateProjects([&UniqueStores](ProjectStore::Project& Project) {
- CidStore* Store = &Project.GetCidStore();
- if (auto It = UniqueStores.find(Store); It == UniqueStores.end())
- {
- UniqueStores.insert_or_assign(Store, Project.Identifier);
- }
- else
- {
- UniqueStores.insert_or_assign(Store, std::string{});
- }
- });
-
- for (auto It : UniqueStores)
- {
- CidStore* ChunkStore = It.first;
-
- CidStoreStats ChunkStoreStats = ChunkStore->Stats();
- CidStoreSize ChunkStoreSize = ChunkStore->TotalSize();
-
- TotalChunkHitCount += ChunkStoreStats.HitCount;
- TotalChunkMissCount += ChunkStoreStats.MissCount;
- TotalChunkWriteCount += ChunkStoreStats.WriteCount;
-
- TotalCidSize.TinySize += ChunkStoreSize.TinySize;
- TotalCidSize.SmallSize += ChunkStoreSize.SmallSize;
- TotalCidSize.LargeSize += ChunkStoreSize.LargeSize;
- TotalCidSize.TotalSize += ChunkStoreSize.TotalSize;
- }
- }
+ const GcStorageSize StoreSize = m_ProjectStore->StorageSize();
+ const CidStoreSize CidSize = m_CidStore.TotalSize();
CbObjectWriter Cbo;
@@ -495,66 +460,12 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
{
Cbo.BeginObject("size");
{
- Cbo << "tiny" << TotalCidSize.TinySize;
- Cbo << "small" << TotalCidSize.SmallSize;
- Cbo << "large" << TotalCidSize.LargeSize;
- Cbo << "total" << TotalCidSize.TotalSize;
+ Cbo << "tiny" << CidSize.TinySize;
+ Cbo << "small" << CidSize.SmallSize;
+ Cbo << "large" << CidSize.LargeSize;
+ Cbo << "total" << CidSize.TotalSize;
}
Cbo.EndObject();
-
- if (ShowCidStoreStats)
- {
- Cbo << "cidhits" << TotalChunkHitCount << "cidmisses" << TotalChunkMissCount << "cidwrites" << TotalChunkWriteCount;
- const uint64_t TotalChunkCount = TotalChunkHitCount + TotalChunkMissCount;
- Cbo << "cidhit_ratio" << (TotalChunkHitCount ? (double(TotalChunkCount) / double(TotalChunkHitCount)) : 0.0);
-
- Cbo.BeginObject("store");
-
- auto OutputStats = [&](CidStore& ChunkStore) {
- CidStoreStats StoreStats = ChunkStore.Stats();
- Cbo << "hits" << StoreStats.HitCount << "misses" << StoreStats.MissCount << "writes" << StoreStats.WriteCount;
- const uint64_t Count = StoreStats.HitCount + StoreStats.MissCount;
- Cbo << "hit_ratio" << (Count ? (double(StoreStats.HitCount) / double(Count)) : 0.0);
- EmitSnapshot("read", StoreStats.FindChunkOps, Cbo);
- EmitSnapshot("write", StoreStats.AddChunkOps, Cbo);
- };
-
- if (UniqueStores.size() > 1)
- {
- Cbo.BeginArray("projects");
- for (auto It : UniqueStores)
- {
- CidStore* ChunkStore = It.first;
- const std::string& ProjectId = It.second;
- CidStoreSize ChunkStoreSize = ChunkStore->TotalSize();
-
- Cbo.BeginObject();
- {
- Cbo << "project" << ProjectId;
- Cbo.BeginObject("stats");
- OutputStats(*ChunkStore);
- Cbo.EndObject();
-
- Cbo.BeginObject("size");
- {
- Cbo << "tiny" << ChunkStoreSize.TinySize;
- Cbo << "small" << ChunkStoreSize.SmallSize;
- Cbo << "large" << ChunkStoreSize.LargeSize;
- Cbo << "total" << ChunkStoreSize.TotalSize;
- }
- Cbo.EndObject();
- }
- Cbo.EndObject();
- }
- Cbo.EndArray(); // projects
- }
- else if (UniqueStores.size() != 0)
- {
- CidStore& ChunkStore = *UniqueStores.begin()->first;
- OutputStats(ChunkStore);
- }
- Cbo.EndObject();
- }
}
Cbo.EndObject();
@@ -1214,8 +1125,6 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
}
Project->TouchOplog(OplogId);
- CidStore& ChunkStore = Project->GetCidStore();
-
ProjectStore::Oplog& Oplog = *FoundLog;
IoBuffer Payload = HttpReq.ReadPayload();
@@ -1228,7 +1137,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
std::vector<IoHash> MissingChunks;
CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
- if (ChunkStore.ContainsChunk(Hash))
+ if (m_CidStore.ContainsChunk(Hash))
{
// Return null attachment as we already have it, no point in reading it and storing it again
return {};
@@ -1484,8 +1393,6 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
}
Project->TouchOplog(OplogId);
- CidStore& ChunkStore = Project->GetCidStore();
-
ProjectStore::Oplog& Oplog = *FoundLog;
if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString))
@@ -1500,7 +1407,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
Op.IterateAttachments([&](CbFieldView FieldView) {
const IoHash AttachmentHash = FieldView.AsAttachment();
- IoBuffer Payload = ChunkStore.FindChunkByCid(AttachmentHash);
+ IoBuffer Payload = m_CidStore.FindChunkByCid(AttachmentHash);
if (Payload)
{
switch (Payload.GetContentType())
@@ -2129,14 +2036,11 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req)
CSVHeader(Details, AttachmentDetails, CSVWriter);
m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
- CidStore& ChunkStore = Project.GetCidStore();
-
Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
- Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN,
- const Oid& Key,
- CbObjectView Op) {
- CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
- });
+ Oplog.IterateOplogWithKey(
+ [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
+ CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+ });
});
});
@@ -2150,9 +2054,8 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req)
m_ProjectStore->DiscoverProjects();
m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) {
- CidStore& ChunkStore = Project.GetCidStore();
- std::vector<std::string> OpLogs = Project.ScanForOplogs();
- CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+ std::vector<std::string> OpLogs = Project.ScanForOplogs();
+ CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
});
}
Cbo.EndArray();
@@ -2181,8 +2084,7 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
{
return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
- ProjectStore::Project& Project = *FoundProject.Get();
- CidStore& ChunkStore = Project.GetCidStore();
+ ProjectStore::Project& Project = *FoundProject.Get();
if (CSV)
{
@@ -2190,11 +2092,10 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
CSVHeader(Details, AttachmentDetails, CSVWriter);
FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) {
- Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN,
- const Oid& Key,
- CbObjectView Op) {
- CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
- });
+ Oplog.IterateOplogWithKey(
+ [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
+ CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+ });
});
HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
}
@@ -2204,7 +2105,7 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req)
std::vector<std::string> OpLogs = FoundProject->ScanForOplogs();
Cbo.BeginArray("projects");
{
- CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
+ CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo);
}
Cbo.EndArray();
HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
@@ -2240,17 +2141,16 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
return HttpReq.WriteResponse(HttpResponseCode::NotFound);
}
- ProjectStore::Project& Project = *FoundProject.Get();
- CidStore& ChunkStore = Project.GetCidStore();
- ProjectStore::Oplog& Oplog = *FoundLog;
+ ProjectStore::Project& Project = *FoundProject.Get();
+ ProjectStore::Oplog& Oplog = *FoundLog;
if (CSV)
{
ExtendableStringBuilder<4096> CSVWriter;
CSVHeader(Details, AttachmentDetails, CSVWriter);
Oplog.IterateOplogWithKey(
- [this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
- CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
+ [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) {
+ CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter);
});
HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
}
@@ -2259,7 +2159,7 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req)
CbObjectWriter Cbo;
Cbo.BeginArray("oplogs");
{
- CbWriteOplog(ChunkStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
+ CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo);
}
Cbo.EndArray();
HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
@@ -2304,10 +2204,9 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, OpId));
}
- const Oid ObjId = Oid::FromHexString(OpId);
- ProjectStore::Project& Project = *FoundProject.Get();
- CidStore& ChunkStore = Project.GetCidStore();
- ProjectStore::Oplog& Oplog = *FoundLog;
+ const Oid ObjId = Oid::FromHexString(OpId);
+ ProjectStore::Project& Project = *FoundProject.Get();
+ ProjectStore::Oplog& Oplog = *FoundLog;
std::optional<CbObject> Op = Oplog.GetOpByKey(ObjId);
if (!Op.has_value())
@@ -2325,7 +2224,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
ExtendableStringBuilder<4096> CSVWriter;
CSVHeader(Details, AttachmentDetails, CSVWriter);
- CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter);
+ CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter);
HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView());
}
else
@@ -2333,7 +2232,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req)
CbObjectWriter Cbo;
Cbo.BeginArray("ops");
{
- CbWriteOp(ChunkStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo);
+ CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo);
}
Cbo.EndArray();
HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save());
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
index 5782188e6..295defa5c 100644
--- a/src/zenserver/projectstore/httpprojectstore.h
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -35,7 +35,11 @@ class ProjectStore;
class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
{
public:
- HttpProjectService(ProjectStore* InProjectStore, HttpStatusService& StatusService, HttpStatsService& StatsService, AuthMgr& AuthMgr);
+ HttpProjectService(CidStore& Store,
+ ProjectStore* InProjectStore,
+ HttpStatusService& StatusService,
+ HttpStatsService& StatsService,
+ AuthMgr& AuthMgr);
~HttpProjectService();
virtual const char* BaseUri() const override;
@@ -88,6 +92,7 @@ private:
inline LoggerRef Log() { return m_Log; }
LoggerRef m_Log;
+ CidStore& m_CidStore;
HttpRequestRouter m_Router;
Ref<ProjectStore> m_ProjectStore;
HttpStatusService& m_StatusService;
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 7463e9caa..a5ab24cfb 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -3832,7 +3832,7 @@ ProjectStore::Project::LastOplogAccessTime(std::string_view Oplog) const
//////////////////////////////////////////////////////////////////////////
-ProjectStore::ProjectStore(GetCidStoreFunc&& GetCidStore,
+ProjectStore::ProjectStore(CidStore& Store,
std::filesystem::path BasePath,
GcManager& Gc,
JobQueue& JobQueue,
@@ -3840,7 +3840,7 @@ ProjectStore::ProjectStore(GetCidStoreFunc&& GetCidStore,
const Configuration& Config)
: m_Log(logging::Get("project"))
, m_Gc(Gc)
-, m_GetCidStore(std::move(GetCidStore))
+, m_CidStore(Store)
, m_JobQueue(JobQueue)
, m_OpenProcessCache(InOpenProcessCache)
, m_ProjectBasePath(BasePath)
@@ -4025,8 +4025,6 @@ ProjectStore::OpenProject(std::string_view ProjectId)
}
}
- CidStore& ChunkStore = m_GetCidStore(ProjectId);
-
RwLock::ExclusiveLockScope _(m_ProjectsLock);
if (auto ProjIt = m_Projects.find(std::string{ProjectId}); ProjIt != m_Projects.end())
{
@@ -4043,7 +4041,7 @@ ProjectStore::OpenProject(std::string_view ProjectId)
Ref<Project>& Prj =
m_Projects
- .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, ChunkStore, BasePath)))
+ .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath)))
.first->second;
Prj->Identifier = ProjectId;
Prj->Read();
@@ -4070,14 +4068,12 @@ ProjectStore::NewProject(const std::filesystem::path& BasePath,
ZEN_MEMSCOPE(GetProjectstoreTag());
ZEN_TRACE_CPU("Store::NewProject");
- CidStore& ChunkStore = m_GetCidStore(ProjectId);
-
RwLock::ExclusiveLockScope _(m_ProjectsLock);
ZEN_INFO("project '{}': creating project at '{}'", ProjectId, BasePath);
Ref<Project>& Prj =
- m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, ChunkStore, BasePath)))
+ m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath)))
.first->second;
Prj->Identifier = ProjectId;
Prj->RootDir = RootDir;
@@ -4806,7 +4802,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId,
}
const IoHash Hash = IoHash::FromHexString(Cid);
- OutChunk = Project->GetCidStore().FindChunkByCid(Hash);
+ OutChunk = m_CidStore.FindChunkByCid(Hash);
if (!OutChunk)
{
@@ -4869,7 +4865,7 @@ ProjectStore::PutChunk(const std::string_view ProjectId,
}
FoundLog->CaptureAddedAttachments(std::vector<IoHash>{Hash});
- CidStore::InsertResult Result = Project->GetCidStore().AddChunk(Chunk, Hash);
+ CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk, Hash);
return {Result.New ? HttpResponseCode::Created : HttpResponseCode::OK, {}};
}
@@ -4898,19 +4894,18 @@ ProjectStore::GetChunks(const std::string_view ProjectId,
}
Project->TouchOplog(OplogId);
- CidStore& ChunkStore = Project->GetCidStore();
-
if (RequestObject["chunks"sv].IsArray())
{
// Legacy full chunks only by rawhash
- CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView();
+ CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView();
+
CbObjectWriter ResponseWriter;
ResponseWriter.BeginArray("chunks"sv);
for (CbFieldView FieldView : ChunksArray)
{
IoHash RawHash = FieldView.AsHash();
- IoBuffer ChunkBuffer = ChunkStore.FindChunkByCid(RawHash);
+ IoBuffer ChunkBuffer = m_CidStore.FindChunkByCid(RawHash);
if (ChunkBuffer)
{
CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(ChunkBuffer));
@@ -5062,7 +5057,7 @@ ProjectStore::GetChunks(const std::string_view ProjectId,
if (ChunkRequest.Input.Id.index() == 0)
{
const IoHash& ChunkHash = std::get<IoHash>(ChunkRequest.Input.Id);
- IoBuffer Payload = ChunkStore.FindChunkByCid(ChunkHash);
+ IoBuffer Payload = m_CidStore.FindChunkByCid(ChunkHash);
if (Payload)
{
ChunkRequest.Output.Exists = true;
@@ -5249,7 +5244,7 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie
return {HttpResponseCode::BadRequest, "Invalid payload format"};
}
- CidStore& ChunkStore = Project->GetCidStore();
+ CidStore& ChunkStore = m_CidStore;
RwLock AttachmentsLock;
tsl::robin_set<IoHash, IoHash::Hasher> Attachments;
@@ -5355,7 +5350,7 @@ ProjectStore::ReadOplog(const std::string_view ProjectId,
}
}
- CidStore& ChunkStore = Project->GetCidStore();
+ CidStore& ChunkStore = m_CidStore;
RemoteProjectStore::LoadContainerResult ContainerResult = BuildContainer(
ChunkStore,
@@ -5467,8 +5462,6 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
}
Project->TouchOplog(OplogId);
- CidStore& ChunkStore = Project->GetCidStore();
-
if (Method == "import"sv)
{
if (!AreDiskWritesAllowed())
@@ -5550,7 +5543,7 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
}
Oplog->CaptureAddedAttachments(WriteRawHashes);
- ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly);
+ m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly);
}
HttpReq.WriteResponse(HttpResponseCode::OK);
return true;
@@ -5723,14 +5716,14 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
ResponseObj.EndArray();
}
- // Ops that have moved chunks to a compressed buffer for storage in ChunkStore have been rewritten with references to the new
- // chunk(s). Make sure we add the chunks to ChunkStore, and do it after we update the oplog so GC doesn't think we have
+ // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the new
+ // chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have
// unreferenced chunks.
for (auto It : AddedChunks)
{
const IoHash& RawHash = It.first;
AddedChunk& Chunk = It.second;
- CidStore::InsertResult Result = ChunkStore.AddChunk(Chunk.Buffer, RawHash);
+ CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk.Buffer, RawHash);
if (Result.New)
{
InlinedBytes += Chunk.RawSize;
@@ -5793,7 +5786,7 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
NiceBytes(MaxBlockSize),
NiceBytes(MaxChunkEmbedSize)));
- RemoteProjectStore::Result Result = SaveOplog(Project->GetCidStore(),
+ RemoteProjectStore::Result Result = SaveOplog(m_CidStore,
*ActualRemoteStore,
*Project.Get(),
*OplogPtr,
@@ -5838,12 +5831,11 @@ ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog,
}
std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store);
RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo();
- CidStore& ChunkStore = Project.GetCidStore();
JobId JobId = m_JobQueue.QueueJob(
fmt::format("Import oplog '{}/{}'", Project.Identifier, Oplog.OplogId()),
[this,
- ChunkStore = &ChunkStore,
+ ChunkStore = &m_CidStore,
ActualRemoteStore = std::move(RemoteStore),
OplogPtr = &Oplog,
Force,
@@ -5855,7 +5847,7 @@ ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog,
ActualRemoteStore->GetInfo().Description));
RemoteProjectStore::Result Result =
- LoadOplog(*ChunkStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, CleanOplog, &Context);
+ LoadOplog(m_CidStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, CleanOplog, &Context);
auto Response = ConvertResult(Result);
ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second);
if (!IsHttpSuccessCode(Response.first))
@@ -6894,11 +6886,6 @@ namespace testutils {
return BuildChunksRequest<IoHash>(SkipData, "RawHash", Chunks, Ranges, ModTags);
}
- ProjectStore::GetCidStoreFunc SingleChunkStore(CidStore& ChunkStore)
- {
- return [ChunkStore = &ChunkStore](std::string_view) -> CidStore& { return *ChunkStore; };
- }
-
} // namespace testutils
TEST_CASE("project.opkeys")
@@ -6961,18 +6948,13 @@ TEST_CASE("project.store.create")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore ChunkStore(Gc);
+ CidStore CidStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- ChunkStore.Initialize(CidConfig);
+ CidStore.Initialize(CidConfig);
std::string_view ProjectName("proj1"sv);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
- BasePath,
- Gc,
- *JobQueue,
- ProcessCache,
- ProjectStore::Configuration{});
+ ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
std::filesystem::path ProjectRootDir = TempDir.Path() / "game";
@@ -6997,17 +6979,12 @@ TEST_CASE("project.store.lifetimes")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore ChunkStore(Gc);
+ CidStore CidStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- ChunkStore.Initialize(CidConfig);
+ CidStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
- BasePath,
- Gc,
- *JobQueue,
- ProcessCache,
- ProjectStore::Configuration{});
+ ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
std::filesystem::path ProjectRootDir = TempDir.Path() / "game";
@@ -7065,17 +7042,12 @@ TEST_CASE_TEMPLATE("project.store.export",
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore ChunkStore(Gc);
+ CidStore CidStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- ChunkStore.Initialize(CidConfig);
+ CidStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
- BasePath,
- Gc,
- *JobQueue,
- ProcessCache,
- ProjectStore::Configuration{});
+ ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
std::filesystem::path ProjectRootDir = TempDir.Path() / "game";
@@ -7109,7 +7081,7 @@ TEST_CASE_TEMPLATE("project.store.export",
std::shared_ptr<RemoteProjectStore> RemoteStore = CreateFileRemoteStore(Options);
RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo();
- RemoteProjectStore::Result ExportResult = SaveOplog(ChunkStore,
+ RemoteProjectStore::Result ExportResult = SaveOplog(CidStore,
*RemoteStore,
*Project.Get(),
*Oplog,
@@ -7126,7 +7098,7 @@ TEST_CASE_TEMPLATE("project.store.export",
ProjectStore::Oplog* OplogImport = Project->NewOplog("oplog2", {});
CHECK(OplogImport != nullptr);
- RemoteProjectStore::Result ImportResult = LoadOplog(ChunkStore,
+ RemoteProjectStore::Result ImportResult = LoadOplog(CidStore,
*RemoteStore,
*OplogImport,
/*Force*/ false,
@@ -7135,7 +7107,7 @@ TEST_CASE_TEMPLATE("project.store.export",
nullptr);
CHECK(ImportResult.ErrorCode == 0);
- RemoteProjectStore::Result ImportForceResult = LoadOplog(ChunkStore,
+ RemoteProjectStore::Result ImportForceResult = LoadOplog(CidStore,
*RemoteStore,
*OplogImport,
/*Force*/ true,
@@ -7144,7 +7116,7 @@ TEST_CASE_TEMPLATE("project.store.export",
nullptr);
CHECK(ImportForceResult.ErrorCode == 0);
- RemoteProjectStore::Result ImportCleanResult = LoadOplog(ChunkStore,
+ RemoteProjectStore::Result ImportCleanResult = LoadOplog(CidStore,
*RemoteStore,
*OplogImport,
/*Force*/ false,
@@ -7153,7 +7125,7 @@ TEST_CASE_TEMPLATE("project.store.export",
nullptr);
CHECK(ImportCleanResult.ErrorCode == 0);
- RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(ChunkStore,
+ RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(CidStore,
*RemoteStore,
*OplogImport,
/*Force*/ true,
@@ -7173,17 +7145,12 @@ TEST_CASE("project.store.gc")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore ChunkStore(Gc);
+ CidStore CidStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- ChunkStore.Initialize(CidConfig);
+ CidStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
- BasePath,
- Gc,
- *JobQueue,
- ProcessCache,
- ProjectStore::Configuration{});
+ ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
@@ -7379,17 +7346,12 @@ TEST_CASE("project.store.gc.prep")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore ChunkStore(Gc);
+ CidStore CidStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- ChunkStore.Initialize(CidConfig);
+ CidStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore";
- ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
- BasePath,
- Gc,
- *JobQueue,
- ProcessCache,
- ProjectStore::Configuration{});
+ ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root";
std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
@@ -7432,7 +7394,7 @@ TEST_CASE("project.store.gc.prep")
// Equivalent of a `prep` existance check call
for (auto Attachment : OpAttachments)
{
- CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7446,7 +7408,7 @@ TEST_CASE("project.store.gc.prep")
// If a gc comes in between our prep and op write the chunks will be removed
for (auto Attachment : OpAttachments)
{
- CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7466,7 +7428,7 @@ TEST_CASE("project.store.gc.prep")
for (auto Attachment : OpAttachments)
{
- CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7480,7 +7442,7 @@ TEST_CASE("project.store.gc.prep")
// Attachments should now be retained
for (auto Attachment : OpAttachments)
{
- CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7494,7 +7456,7 @@ TEST_CASE("project.store.gc.prep")
// Attachments should now be retained across multiple GCs if retain time is still valud
for (auto Attachment : OpAttachments)
{
- CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7506,7 +7468,7 @@ TEST_CASE("project.store.gc.prep")
}
for (auto Attachment : OpAttachments)
{
- CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv);
@@ -7523,7 +7485,7 @@ TEST_CASE("project.store.gc.prep")
for (auto Attachment : OpAttachments)
{
- CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
{
@@ -7556,7 +7518,7 @@ TEST_CASE("project.store.gc.prep")
}
for (auto Attachment : OpAttachments)
{
- CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
Sleep(200);
@@ -7571,7 +7533,7 @@ TEST_CASE("project.store.gc.prep")
}
for (auto Attachment : OpAttachments)
{
- CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
// This pass the retention time has expired and the last GC pass cleared the entries
@@ -7585,7 +7547,7 @@ TEST_CASE("project.store.gc.prep")
for (auto Attachment : OpAttachments)
{
- CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash()));
+ CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash()));
}
}
@@ -7599,17 +7561,12 @@ TEST_CASE("project.store.rpc.getchunks")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore ChunkStore(Gc);
+ CidStore CidStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- ChunkStore.Initialize(CidConfig);
+ CidStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
- ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
- BasePath,
- Gc,
- *JobQueue,
- ProcessCache,
- ProjectStore::Configuration{});
+ ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root"sv;
std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv;
@@ -8526,17 +8483,12 @@ TEST_CASE("project.store.partial.read")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore ChunkStore(Gc);
+ CidStore CidStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- ChunkStore.Initialize(CidConfig);
+ CidStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
- ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
- BasePath,
- Gc,
- *JobQueue,
- ProcessCache,
- ProjectStore::Configuration{});
+ ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root"sv;
std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv;
@@ -8709,17 +8661,12 @@ TEST_CASE("project.store.iterateoplog")
auto JobQueue = MakeJobQueue(1, ""sv);
OpenProcessCache ProcessCache;
GcManager Gc;
- CidStore ChunkStore(Gc);
+ CidStore CidStore(Gc);
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
- ChunkStore.Initialize(CidConfig);
+ CidStore.Initialize(CidConfig);
std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
- ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore),
- BasePath,
- Gc,
- *JobQueue,
- ProcessCache,
- ProjectStore::Configuration{});
+ ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{});
std::filesystem::path RootDir = TempDir.Path() / "root"sv;
std::filesystem::path EngineRootDir = TempDir.Path() / "enginesv";
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index eb27665f9..2595d7198 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -68,9 +68,7 @@ public:
{
};
- typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc;
-
- ProjectStore(GetCidStoreFunc&& GetCidStore,
+ ProjectStore(CidStore& Store,
std::filesystem::path BasePath,
GcManager& Gc,
JobQueue& JobQueue,
@@ -328,8 +326,6 @@ public:
Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath);
virtual ~Project();
- CidStore& GetCidStore() { return m_CidStore; };
-
void Read();
void Write();
[[nodiscard]] static bool Exists(const std::filesystem::path& BasePath);
@@ -503,7 +499,7 @@ public:
private:
LoggerRef m_Log;
GcManager& m_Gc;
- GetCidStoreFunc m_GetCidStore;
+ CidStore& m_CidStore;
JobQueue& m_JobQueue;
OpenProcessCache& m_OpenProcessCache;
std::filesystem::path m_ProjectBasePath;
diff --git a/src/zenserver/upstream/upstreamcache.cpp b/src/zenserver/upstream/upstreamcache.cpp
index a1c460bc0..744b861dd 100644
--- a/src/zenserver/upstream/upstreamcache.cpp
+++ b/src/zenserver/upstream/upstreamcache.cpp
@@ -1475,17 +1475,12 @@ namespace detail {
class UpstreamCacheImpl final : public UpstreamCache
{
- struct EnqueuedRequest
- {
- UpstreamCacheRecord Record;
- std::function<IoBuffer(const IoHash& ChunkHash)> GetValueFunc;
- };
-
public:
- UpstreamCacheImpl(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore)
+ UpstreamCacheImpl(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore)
: m_Log(logging::Get("upstream"))
, m_Options(Options)
, m_CacheStore(CacheStore)
+ , m_CidStore(CidStore)
{
}
@@ -1841,17 +1836,17 @@ public:
}
}
- virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord, std::function<IoBuffer(const IoHash&)>&& GetValueFunc) override
+ virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord) override
{
if (m_RunState.IsRunning && m_Options.WriteUpstream && m_Endpoints.size() > 0)
{
if (!m_UpstreamThreads.empty())
{
- m_UpstreamQueue.Enqueue(EnqueuedRequest{.Record = std::move(CacheRecord), .GetValueFunc = GetValueFunc});
+ m_UpstreamQueue.Enqueue(std::move(CacheRecord));
}
else
{
- ProcessCacheRecord(std::move(CacheRecord), std::move(GetValueFunc));
+ ProcessCacheRecord(std::move(CacheRecord));
}
}
}
@@ -1905,7 +1900,7 @@ public:
}
private:
- void ProcessCacheRecord(const UpstreamCacheRecord& CacheRecord, std::function<IoBuffer(const IoHash& ChunkHash)>&& GetValueFunc)
+ void ProcessCacheRecord(UpstreamCacheRecord CacheRecord)
{
ZEN_TRACE_CPU("Upstream::ProcessCacheRecord");
@@ -1923,7 +1918,7 @@ private:
for (const IoHash& ValueContentId : CacheRecord.ValueContentIds)
{
- if (IoBuffer Payload = GetValueFunc(ValueContentId))
+ if (IoBuffer Payload = m_CidStore.FindChunkByCid(ValueContentId))
{
Payloads.push_back(Payload);
}
@@ -1975,19 +1970,19 @@ private:
for (;;)
{
- EnqueuedRequest Request;
- if (m_UpstreamQueue.WaitAndDequeue(Request))
+ UpstreamCacheRecord CacheRecord;
+ if (m_UpstreamQueue.WaitAndDequeue(CacheRecord))
{
try
{
- ProcessCacheRecord(Request.Record, std::move(Request.GetValueFunc));
+ ProcessCacheRecord(std::move(CacheRecord));
}
catch (const std::exception& Err)
{
ZEN_ERROR("upload cache record '{}/{}/{}' FAILED, reason '{}'",
- Request.Record.Namespace,
- Request.Record.Key.Bucket,
- Request.Record.Key.Hash,
+ CacheRecord.Namespace,
+ CacheRecord.Key.Bucket,
+ CacheRecord.Key.Hash,
Err.what());
}
}
@@ -2081,7 +2076,7 @@ private:
LoggerRef Log() { return m_Log; }
- using UpstreamQueue = BlockingQueue<EnqueuedRequest>;
+ using UpstreamQueue = BlockingQueue<UpstreamCacheRecord>;
struct RunState
{
@@ -2107,6 +2102,7 @@ private:
LoggerRef m_Log;
UpstreamCacheOptions m_Options;
ZenCacheStore& m_CacheStore;
+ CidStore& m_CidStore;
UpstreamQueue m_UpstreamQueue;
std::shared_mutex m_EndpointsMutex;
std::vector<std::unique_ptr<UpstreamEndpoint>> m_Endpoints;
@@ -2130,9 +2126,9 @@ UpstreamEndpoint::CreateJupiterEndpoint(const JupiterClientOptions& Options, con
}
std::unique_ptr<UpstreamCache>
-CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore)
+CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore)
{
- return std::make_unique<UpstreamCacheImpl>(Options, CacheStore);
+ return std::make_unique<UpstreamCacheImpl>(Options, CacheStore, CidStore);
}
} // namespace zen
diff --git a/src/zenserver/upstream/upstreamcache.h b/src/zenserver/upstream/upstreamcache.h
index e4b9a73ad..26e5decac 100644
--- a/src/zenserver/upstream/upstreamcache.h
+++ b/src/zenserver/upstream/upstreamcache.h
@@ -24,6 +24,7 @@ class AuthMgr;
class CbObjectView;
class CbPackage;
class CbObjectWriter;
+class CidStore;
class ZenCacheStore;
struct JupiterClientOptions;
class JupiterAccessTokenProvider;
@@ -161,6 +162,6 @@ struct UpstreamCacheOptions
bool WriteUpstream = true;
};
-std::unique_ptr<UpstreamCache> CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore);
+std::unique_ptr<UpstreamCache> CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore);
} // namespace zen
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 71b52817c..44c25368c 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -187,7 +187,14 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
m_Http = CreateHttpServer(ServerOptions.HttpServerConfig);
int EffectiveBasePort = m_Http->Initialize(ServerOptions.BasePort, ServerOptions.DataDir);
- ZEN_ASSERT(EffectiveBasePort > 0);
+ if (EffectiveBasePort == 0)
+ {
+ ZEN_WARN("Failed to initialize http service '{}' using base port {} and data dir {}",
+ ServerOptions.HttpServerConfig.ServerClass,
+ ServerOptions.BasePort,
+ ServerOptions.DataDir);
+ return -1;
+ }
// Setup authentication manager
{
@@ -198,7 +205,10 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
{
EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456";
- ZEN_WARN("using default encryption key");
+ if (ServerOptions.IsDedicated)
+ {
+ ZEN_WARN("Using default encryption key for authentication state");
+ }
}
std::string EncryptionIV = ServerOptions.EncryptionIV;
@@ -207,7 +217,10 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
{
EncryptionIV = "0123456789abcdef";
- ZEN_WARN("using default encryption initialization vector");
+ if (ServerOptions.IsDedicated)
+ {
+ ZEN_WARN("Using default encryption initialization vector for authentication state");
+ }
}
m_AuthMgr = AuthMgr::Create({.RootDirectory = m_DataRoot / "auth",
@@ -242,18 +255,18 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
CidStoreConfiguration Config;
Config.RootDirectory = m_DataRoot / "cas";
- m_CidStores.insert_or_assign({}, std::make_unique<CidStore>(m_GcManager));
- m_CidStores.at({})->Initialize(Config);
+ m_CidStore = std::make_unique<CidStore>(m_GcManager);
+ m_CidStore->Initialize(Config);
ZEN_INFO("instantiating project service");
- m_ProjectStore = new ProjectStore([this](std::string_view) -> CidStore& { return *m_CidStores.at({}).get(); },
+ m_ProjectStore = new ProjectStore(*m_CidStore,
m_DataRoot / "projects",
m_GcManager,
*m_JobQueue,
*m_OpenProcessCache,
ProjectStore::Configuration{});
- m_HttpProjectService.reset(new HttpProjectService(m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr));
+ m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr});
if (ServerOptions.WorksSpacesConfig.Enabled)
{
@@ -600,7 +613,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
UpstreamOptions.ThreadCount = static_cast<uint32_t>(UpstreamConfig.UpstreamThreadCount);
}
- m_UpstreamCache = CreateUpstreamCache(UpstreamOptions, *m_CacheStore);
+ m_UpstreamCache = CreateUpstreamCache(UpstreamOptions, *m_CacheStore, *m_CidStore);
m_UpstreamService = std::make_unique<HttpUpstreamService>(*m_UpstreamCache, *m_AuthMgr);
m_UpstreamCache->Initialize();
@@ -664,23 +677,19 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
}
}
- m_StructuredCacheService = std::make_unique<HttpStructuredCacheService>(
- *m_CacheStore,
- [this](std::string_view) -> CidStore& { return *m_CidStores.at({}).get(); },
- m_StatsService,
- m_StatusService,
- *m_UpstreamCache,
- m_GcManager.GetDiskWriteBlocker(),
- *m_OpenProcessCache);
+ m_StructuredCacheService = std::make_unique<HttpStructuredCacheService>(*m_CacheStore,
+ *m_CidStore,
+ m_StatsService,
+ m_StatusService,
+ *m_UpstreamCache,
+ m_GcManager.GetDiskWriteBlocker(),
+ *m_OpenProcessCache);
m_Http->RegisterService(*m_StructuredCacheService);
m_Http->RegisterService(*m_UpstreamService);
m_StatsReporter.AddProvider(m_CacheStore.Get());
- for (const auto& It : m_CidStores)
- {
- m_StatsReporter.AddProvider(It.second.get());
- }
+ m_StatsReporter.AddProvider(m_CidStore.get());
m_StatsReporter.AddProvider(m_BuildCidStore.get());
}
@@ -865,7 +874,7 @@ ZenServer::Cleanup()
m_Workspaces.reset();
m_HttpProjectService.reset();
m_ProjectStore = {};
- m_CidStores.clear();
+ m_CidStore.reset();
m_AuthService.reset();
m_AuthMgr.reset();
m_Http = {};
@@ -1053,10 +1062,8 @@ ZenServer::Flush()
{
ZEN_TRACE_CPU("ZenServer::Flush");
- for (auto& It : m_CidStores)
- {
- It.second->Flush();
- }
+ if (m_CidStore)
+ m_CidStore->Flush();
if (m_StructuredCacheService)
m_StructuredCacheService->Flush();
diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h
index 0a17446ae..bcb02d336 100644
--- a/src/zenserver/zenserver.h
+++ b/src/zenserver/zenserver.h
@@ -115,20 +115,20 @@ private:
inline void SetNewState(ServerState NewState) { m_CurrentState = NewState; }
static std::string_view ToString(ServerState Value);
- StatsReporter m_StatsReporter;
- Ref<HttpServer> m_Http;
- std::unique_ptr<AuthMgr> m_AuthMgr;
- std::unique_ptr<HttpAuthService> m_AuthService;
- HttpStatusService m_StatusService;
- HttpStatsService m_StatsService;
- GcManager m_GcManager;
- GcScheduler m_GcScheduler{m_GcManager};
- tsl::robin_map<std::string, std::unique_ptr<CidStore>> m_CidStores;
- Ref<ZenCacheStore> m_CacheStore;
- std::unique_ptr<OpenProcessCache> m_OpenProcessCache;
- HttpTestService m_TestService;
- std::unique_ptr<CidStore> m_BuildCidStore;
- std::unique_ptr<BuildStore> m_BuildStore;
+ StatsReporter m_StatsReporter;
+ Ref<HttpServer> m_Http;
+ std::unique_ptr<AuthMgr> m_AuthMgr;
+ std::unique_ptr<HttpAuthService> m_AuthService;
+ HttpStatusService m_StatusService;
+ HttpStatsService m_StatsService;
+ GcManager m_GcManager;
+ GcScheduler m_GcScheduler{m_GcManager};
+ std::unique_ptr<CidStore> m_CidStore;
+ Ref<ZenCacheStore> m_CacheStore;
+ std::unique_ptr<OpenProcessCache> m_OpenProcessCache;
+ HttpTestService m_TestService;
+ std::unique_ptr<CidStore> m_BuildCidStore;
+ std::unique_ptr<BuildStore> m_BuildStore;
#if ZEN_WITH_TESTS
HttpTestingService m_TestingService;
diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp
index ff21d1ede..5d9a68919 100644
--- a/src/zenstore/cache/cacherpc.cpp
+++ b/src/zenstore/cache/cacherpc.cpp
@@ -153,13 +153,13 @@ CacheRpcHandler::CacheRpcHandler(LoggerRef InLog,
CacheStats& InCacheStats,
UpstreamCacheClient& InUpstreamCache,
ZenCacheStore& InCacheStore,
- GetCidStoreFunc&& InGetCidStore,
+ CidStore& InCidStore,
const DiskWriteBlocker* InDiskWriteBlocker)
: m_Log(InLog)
, m_CacheStats(InCacheStats)
, m_UpstreamCache(InUpstreamCache)
, m_CacheStore(InCacheStore)
-, m_GetCidStore(std::move(InGetCidStore))
+, m_CidStore(InCidStore)
, m_DiskWriteBlocker(InDiskWriteBlocker)
{
}
@@ -174,12 +174,6 @@ CacheRpcHandler::AreDiskWritesAllowed() const
return (m_DiskWriteBlocker == nullptr || m_DiskWriteBlocker->AreDiskWritesAllowed());
}
-CidStore&
-CacheRpcHandler::GetCidStore(std::string_view Namespace)
-{
- return m_GetCidStore(Namespace);
-}
-
CacheRpcHandler::RpcResponseCode
CacheRpcHandler::HandleRpcRequest(const CacheRequestContext& Context,
std::string_view UriNamespace,
@@ -387,12 +381,9 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag
Stopwatch Timer;
- CidStore& ChunkStore = m_GetCidStore(Request.Namespace);
-
Request.RecordObject.IterateAttachments([this,
&Request,
Package,
- &ChunkStore,
&WriteAttachmentBuffers,
&WriteRawHashes,
&ValidAttachments,
@@ -421,7 +412,7 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag
Count.Invalid++;
}
}
- else if (ChunkStore.ContainsChunk(ValueHash))
+ else if (m_CidStore.ContainsChunk(ValueHash))
{
ValidAttachments.emplace_back(ValueHash);
Count.Valid++;
@@ -457,7 +448,7 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag
if (!WriteAttachmentBuffers.empty())
{
- std::vector<CidStore::InsertResult> InsertResults = ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes);
+ std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes);
for (size_t Index = 0; Index < InsertResults.size(); Index++)
{
if (InsertResults[Index].New)
@@ -484,12 +475,10 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag
if (HasUpstream && EnumHasAllFlags(Request.Policy.GetRecordPolicy(), CachePolicy::StoreRemote) && !IsPartialRecord)
{
- m_UpstreamCache.EnqueueUpstream(
- {.Type = ZenContentType::kCbPackage,
- .Namespace = Request.Namespace,
- .Key = Request.Key,
- .ValueContentIds = std::move(ValidAttachments)},
- [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); });
+ m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage,
+ .Namespace = Request.Namespace,
+ .Key = Request.Key,
+ .ValueContentIds = std::move(ValidAttachments)});
}
return PutStatus::Success;
}
@@ -532,8 +521,6 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
return CbPackage{};
}
- CidStore& ChunkStore = m_GetCidStore(Namespace.value());
-
const bool HasUpstream = m_UpstreamCache.IsActive();
eastl::fixed_vector<RecordRequestData, 16> Requests;
@@ -633,7 +620,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
}
else if (EnumHasAllFlags(ValuePolicy, CachePolicy::SkipData))
{
- if (ChunkStore.ContainsChunk(Value.ContentId))
+ if (m_CidStore.ContainsChunk(Value.ContentId))
{
Value.Exists = true;
}
@@ -654,7 +641,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
}
else
{
- if (IoBuffer Chunk = ChunkStore.FindChunkByCid(Value.ContentId))
+ if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Value.ContentId))
{
if (Chunk.GetSize() > 0)
{
@@ -678,7 +665,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
}
if (!RequestValueIndexes.empty())
{
- ChunkStore.IterateChunks(
+ m_CidStore.IterateChunks(
CidHashes,
[this, &Request, &RequestValueIndexes](size_t Index, const IoBuffer& Payload) -> bool {
try
@@ -771,7 +758,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
}
}
- const auto OnCacheRecordGetComplete = [this, Namespace, &ChunkStore, &ParseValues, Context](CacheRecordGetCompleteParams&& Params) {
+ const auto OnCacheRecordGetComplete = [this, Namespace, &ParseValues, Context](CacheRecordGetCompleteParams&& Params) {
if (!Params.Record)
{
return;
@@ -840,7 +827,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
Value.Exists = true;
if (StoreLocal)
{
- ChunkStore.AddChunk(Compressed.GetCompressed().Flatten().AsIoBuffer(), Attachment->GetHash());
+ m_CidStore.AddChunk(Compressed.GetCompressed().Flatten().AsIoBuffer(), Attachment->GetHash());
}
if (!EnumHasAllFlags(ValuePolicy, CachePolicy::SkipData))
{
@@ -957,8 +944,6 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con
const bool HasUpstream = m_UpstreamCache.IsActive();
CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
- CidStore& ChunkStore = m_GetCidStore(Namespace.value());
-
std::vector<ZenCacheStore::PutResult> BatchResults;
eastl::fixed_vector<size_t, 32> BatchResultIndexes;
eastl::fixed_vector<ZenCacheStore::PutResult, 32> Results;
@@ -1109,8 +1094,7 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con
if ((Results[Index].Status == zen::PutStatus::Success) && UpstreamCacheKeys[Index] != CacheKey::Empty)
{
m_UpstreamCache.EnqueueUpstream(
- {.Type = ZenContentType::kCompressedBinary, .Namespace = *Namespace, .Key = UpstreamCacheKeys[Index]},
- [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); });
+ {.Type = ZenContentType::kCompressedBinary, .Namespace = *Namespace, .Key = UpstreamCacheKeys[Index]});
}
}
{
@@ -1562,8 +1546,6 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context,
using namespace cache::detail;
const bool HasUpstream = m_UpstreamCache.IsActive();
- CidStore& ChunkStore = m_GetCidStore(Namespace);
-
// TODO: BatchGet records?
std::vector<CacheKeyRequest*> UpstreamRecordRequests;
for (size_t RecordIndex = 0; RecordIndex < Records.size(); ++RecordIndex)
@@ -1702,12 +1684,12 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context,
{
if (EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData) && Request->RawSizeKnown)
{
- if (ChunkStore.ContainsChunk(Request->Key->ChunkId))
+ if (m_CidStore.ContainsChunk(Request->Key->ChunkId))
{
Request->Exists = true;
}
}
- else if (IoBuffer Payload = ChunkStore.FindChunkByCid(Request->Key->ChunkId))
+ else if (IoBuffer Payload = m_CidStore.FindChunkByCid(Request->Key->ChunkId))
{
if (!EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData))
{
@@ -1840,8 +1822,6 @@ CacheRpcHandler::GetUpstreamCacheChunks(const CacheRequestContext& Context,
return;
}
- CidStore& ChunkStore = m_GetCidStore(Namespace);
-
CacheChunkRequest& Key = Params.Request;
size_t RequestIndex = std::distance(RequestKeys.data(), &Key);
ChunkRequest& Request = Requests[RequestIndex];
@@ -1861,7 +1841,7 @@ CacheRpcHandler::GetUpstreamCacheChunks(const CacheRequestContext& Context,
bool Overwrite = !EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::QueryLocal);
if (Request.IsRecordRequest)
{
- ChunkStore.AddChunk(Params.Value, Params.RawHash);
+ m_CidStore.AddChunk(Params.Value, Params.RawHash);
}
else
{
diff --git a/src/zenstore/include/zenstore/cache/cacherpc.h b/src/zenstore/include/zenstore/cache/cacherpc.h
index 80340d72c..104746aba 100644
--- a/src/zenstore/include/zenstore/cache/cacherpc.h
+++ b/src/zenstore/include/zenstore/cache/cacherpc.h
@@ -70,13 +70,11 @@ IsCompressedBinary(ZenContentType Type)
struct CacheRpcHandler
{
- typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc;
-
CacheRpcHandler(LoggerRef InLog,
CacheStats& InCacheStats,
UpstreamCacheClient& InUpstreamCache,
ZenCacheStore& InCacheStore,
- GetCidStoreFunc&& InGetCidStore,
+ CidStore& InCidStore,
const DiskWriteBlocker* InDiskWriteBlocker);
~CacheRpcHandler();
@@ -96,8 +94,6 @@ struct CacheRpcHandler
int& OutTargetProcessId,
CbPackage& OutPackage);
- CidStore& GetCidStore(std::string_view Namespace);
-
private:
CbPackage HandleRpcPutCacheRecords(const CacheRequestContext& Context, const CbPackage& BatchRequest);
CbPackage HandleRpcGetCacheRecords(const CacheRequestContext& Context, CbObjectView BatchRequest);
@@ -146,7 +142,7 @@ private:
CacheStats& m_CacheStats;
UpstreamCacheClient& m_UpstreamCache;
ZenCacheStore& m_CacheStore;
- GetCidStoreFunc m_GetCidStore;
+ CidStore& m_CidStore;
const DiskWriteBlocker* m_DiskWriteBlocker = nullptr;
bool AreDiskWritesAllowed() const;
diff --git a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h
index c3993c028..152031c3a 100644
--- a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h
+++ b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h
@@ -113,7 +113,7 @@ public:
std::span<CacheChunkRequest*> CacheChunkRequests,
OnCacheChunksGetComplete&& OnComplete) = 0;
- virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord, std::function<IoBuffer(const IoHash&)>&& GetValueFunc) = 0;
+ virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord) = 0;
};
} // namespace zen
diff --git a/src/zenutil/include/zenutil/logging.h b/src/zenutil/include/zenutil/logging.h
index d64eef207..85ddc86cd 100644
--- a/src/zenutil/include/zenutil/logging.h
+++ b/src/zenutil/include/zenutil/logging.h
@@ -34,6 +34,7 @@ struct LoggingOptions
bool IsTest = false;
bool AllowAsync = true;
bool NoConsoleOutput = false;
+ bool QuietConsole = false;
std::filesystem::path AbsLogFile; // Absolute path to main log file
std::string LogId;
};
diff --git a/src/zenutil/logging.cpp b/src/zenutil/logging.cpp
index cb0fd6679..8ff58ee73 100644
--- a/src/zenutil/logging.cpp
+++ b/src/zenutil/logging.cpp
@@ -121,6 +121,10 @@ BeginInitializeLogging(const LoggingOptions& LogOptions)
else
{
auto ConsoleSink = std::make_shared<spdlog::sinks::ansicolor_stdout_sink_mt>();
+ if (LogOptions.QuietConsole)
+ {
+ ConsoleSink->set_level(spdlog::level::warn);
+ }
Sinks.push_back(ConsoleSink);
}