diff options
| author | Stefan Boberg <[email protected]> | 2025-08-26 14:53:31 +0200 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2025-08-26 14:53:31 +0200 |
| commit | 71b9f80cc72fae4cf0476bfca1646ec8311c9272 (patch) | |
| tree | 59c1e0b8911814445e6b78005a43207593393323 /src | |
| parent | default Sentry to off in debug (diff) | |
| parent | Merge branch 'main' into sb/zen-master (diff) | |
| download | zen-71b9f80cc72fae4cf0476bfca1646ec8311c9272.tar.xz zen-71b9f80cc72fae4cf0476bfca1646ec8311c9272.zip | |
Merge branch 'sb/zen-master' of https://github.ol.epicgames.net/ue-foundation/zen into sb/zen-master
Diffstat (limited to 'src')
40 files changed, 1400 insertions, 1668 deletions
diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp index fe2bbbdc7..dca6da6c4 100644 --- a/src/zen/cmds/admin_cmd.cpp +++ b/src/zen/cmds/admin_cmd.cpp @@ -51,17 +51,17 @@ ScrubCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (HttpClient::Response Response = Http.Post("/admin/scrub"sv, /* headers */ HttpClient::KeyValueMap{}, Params)) { - ZEN_CONSOLE("scrub started OK: {}", Response.ToText()); + ZEN_CONSOLE("Scrub started OK: {}", Response.ToText()); return 0; } else if (int StatusCode = (int)Response.StatusCode) { - ZEN_ERROR("scrub start failed: {}: {} ({})", StatusCode, ReasonStringForHttpResultCode(StatusCode), Response.ToText()); + ZEN_CONSOLE_ERROR("Scrub start failed: {}: {} ({})", StatusCode, ReasonStringForHttpResultCode(StatusCode), Response.ToText()); } else { - ZEN_ERROR("scrub start failed: {}", Response.ToText()); + ZEN_CONSOLE_ERROR("Scrub start failed: {}", Response.ToText()); } return 1; @@ -259,11 +259,11 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (Result.status_code) { - ZEN_ERROR("GC start failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); + ZEN_CONSOLE_ERROR("GC start failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); } else { - ZEN_ERROR("GC start failed: {}", Result.error.message); + ZEN_CONSOLE_ERROR("GC start failed: {}", Result.error.message); } return 1; @@ -315,11 +315,11 @@ GcStatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (Result.status_code) { - ZEN_ERROR("GC status failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); + ZEN_CONSOLE_ERROR("GC status failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); } else { - ZEN_ERROR("GC status failed: {}", Result.error.message); + ZEN_CONSOLE_ERROR("GC status failed: {}", Result.error.message); } return 1; @@ -369,11 +369,11 @@ GcStopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (Result.status_code) { - ZEN_ERROR("GC status failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); + ZEN_CONSOLE_ERROR("GC status failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); } else { - ZEN_ERROR("GC status failed: {}", Result.error.message); + ZEN_CONSOLE_ERROR("GC status failed: {}", Result.error.message); } return 1; @@ -416,7 +416,7 @@ JobCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (m_JobId == 0) { - ZEN_ERROR("Job id must be given"); + throw OptionParseException("Job id must be given"); return 1; } } @@ -514,7 +514,7 @@ LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - ZEN_ERROR("Invalid value for parameter 'cache-write-log'. Use 'enable' or 'disable'"); + ZEN_CONSOLE_ERROR("Invalid value for parameter 'cache-write-log'. Use 'enable' or 'disable'"); return 1; } } @@ -531,7 +531,7 @@ LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - ZEN_ERROR("Invalid value for parameter 'cache-access-log'. Use 'enable' or 'disable'"); + ZEN_CONSOLE_ERROR("Invalid value for parameter 'cache-access-log'. Use 'enable' or 'disable'"); return 1; } } @@ -551,12 +551,12 @@ LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) auto CopyLog = [](std::string_view SourceName, std::string_view SourcePath, std::string_view TargetPath) -> bool { if (SourcePath.empty()) { - ZEN_ERROR("Failed to retrieve {} log path", SourceName); + ZEN_CONSOLE_ERROR("Failed to retrieve {} log path", SourceName); return false; } if (!CopyFile(SourcePath, TargetPath, {})) { - ZEN_ERROR("Failed to copy {} log file {} to output file '{}'", SourceName, SourcePath, TargetPath); + ZEN_CONSOLE_ERROR("Failed to copy {} log file {} to output file '{}'", SourceName, SourcePath, TargetPath); return false; } return true; @@ -642,11 +642,11 @@ FlushCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (int StatusCode = (int)Response.StatusCode) { - ZEN_ERROR("flush failed: {}: {} ({})", StatusCode, ReasonStringForHttpResultCode(StatusCode), Response.ToText()); + ZEN_CONSOLE_ERROR("Flush failed: {}: {} ({})", StatusCode, ReasonStringForHttpResultCode(StatusCode), Response.ToText()); } else { - ZEN_ERROR("flush failed: {}", Response.ToText()); + ZEN_CONSOLE_ERROR("Flush failed: {}", Response.ToText()); } return 1; diff --git a/src/zen/cmds/bench_cmd.cpp b/src/zen/cmds/bench_cmd.cpp index 86b82d838..d904a51f6 100644 --- a/src/zen/cmds/bench_cmd.cpp +++ b/src/zen/cmds/bench_cmd.cpp @@ -50,11 +50,11 @@ BenchCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } catch (const zen::bench::util::elevation_required_exception&) { - ZEN_CONSOLE("purging standby lists requires elevation. Will try launch as elevated process"); + ZEN_CONSOLE_WARN("Purging standby lists requires elevation. Will try launch as elevated process"); } catch (const std::exception& Ex) { - ZEN_CONSOLE("ERROR: {}", Ex.what()); + ZEN_CONSOLE_ERROR("{}", Ex.what()); } if (!Ok && !m_SingleProcess) @@ -79,20 +79,20 @@ BenchCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - ZEN_CONSOLE("ERROR: Elevated child process failed with return code {}", ExitCode); + ZEN_CONSOLE_ERROR("Elevated child process failed with return code {}", ExitCode); } } } catch (const std::exception& Ex) { - ZEN_CONSOLE("ERROR: {}", Ex.what()); + ZEN_CONSOLE_ERROR("{}", Ex.what()); } } if (Ok) { // TODO: could also add reporting on just how much memory was purged - ZEN_CONSOLE("purged standby lists! (took {})", zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + ZEN_CONSOLE("Purged standby lists! (took {})", zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } } #endif diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 8692e5941..2c6b8e50e 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -258,7 +258,7 @@ namespace { } catch (const std::exception& Ex) { - ZEN_ERROR("ZenState::~ZenState threw exception: {}", Ex.what()); + ZEN_CONSOLE_ERROR("ZenState::~ZenState threw exception: {}", Ex.what()); } } @@ -469,7 +469,7 @@ namespace { ZEN_ASSERT_SLOW(IsFile(SourcePath)); if (Retries > 5) { - ZEN_CONSOLE("Unable to overwrite file {} ({}: {}), retrying...", TargetPath, Ec.value(), Ec.message()); + ZEN_CONSOLE_WARN("Unable to overwrite file {} ({}: {}), retrying...", TargetPath, Ec.value(), Ec.message()); } Sleep(50 + int(Retries * 150)); Ec.clear(); @@ -701,7 +701,7 @@ namespace { } catch (const std::exception& Ex) { - ZEN_WARN("Failed removing file {}. Reason: {}", FilePath, Ex.what()); + ZEN_CONSOLE_WARN("Failed removing file {}. Reason: {}", FilePath, Ex.what()); CleanWipe = false; } } @@ -784,7 +784,7 @@ namespace { } catch (const std::exception& Ex) { - ZEN_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what()); + ZEN_CONSOLE_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what()); CleanWipe = false; } @@ -1569,7 +1569,7 @@ namespace { } catch (const std::exception& Ex) { - ZEN_CONSOLE("Unable to read local state: ", Ex.what()); + ZEN_CONSOLE_WARN("Unable to read local state: ", Ex.what()); return false; } } @@ -1707,7 +1707,7 @@ namespace { } else { - ZEN_WARN("Invalid download state file at {}. '{}'", EntryPath, JsonError); + ZEN_CONSOLE_WARN("Invalid download state file at {}. '{}'", EntryPath, JsonError); DeleteEntry = true; } } @@ -2927,8 +2927,8 @@ namespace { } else { - ZEN_CONSOLE( - "Warning: Build blob {} was reported as needed for upload but it was reported as existing at the start of the " + ZEN_CONSOLE_WARN( + "Build blob {} was reported as needed for upload but it was reported as existing at the start of the " "operation. Treating it as a transient inconsistent issue and will attempt to retry finalization", RawHash); } @@ -3570,8 +3570,8 @@ namespace { } else if (AllowMultiparts) { - ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", - NiceBytes(Result.PreferredMultipartChunkSize)); + ZEN_CONSOLE_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", + NiceBytes(Result.PreferredMultipartChunkSize)); } } @@ -4765,7 +4765,7 @@ namespace { for (const std::string& Error : Errors) { - ZEN_CONSOLE("{}", Error); + ZEN_CONSOLE_ERROR("{}", Error); } if (!Errors.empty()) { @@ -5671,7 +5671,7 @@ namespace { } catch (const std::exception& Ex) { - ZEN_CONSOLE("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what()); + ZEN_CONSOLE_WARN("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what()); } return HasLocalState; } @@ -8194,7 +8194,7 @@ namespace { // Clean target folder if (!CleanDirectory(Path, DefaultExcludeFolders)) { - ZEN_WARN("Some files in {} could not be removed", Path); + ZEN_CONSOLE_WARN("Some files in {} could not be removed", Path); } RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs(); } @@ -8750,7 +8750,7 @@ namespace { ChunkBlockDescription Description = ParseChunkBlockDescription(CacheBlockMetadata); if (Description.BlockHash == IoHash::Zero) { - ZEN_WARN("Unexpected/invalid block metadata received from remote cache, skipping block"); + ZEN_CONSOLE_WARN("Unexpected/invalid block metadata received from remote cache, skipping block"); } else { @@ -8787,7 +8787,7 @@ namespace { if (Description.BlockHash == IoHash::Zero) { - ZEN_WARN("Unexpected/invalid block metadata received from remote store, skipping block"); + ZEN_CONSOLE_WARN("Unexpected/invalid block metadata received from remote store, skipping block"); } else { @@ -8843,7 +8843,7 @@ namespace { BlockRawHashes.size() - OutBlockDescriptions.size()); if (AttemptFallback) { - ZEN_CONSOLE("{} Attemping fallback options.", ErrorDescription); + ZEN_CONSOLE_WARN("{} Attemping fallback options.", ErrorDescription); std::vector<ChunkBlockDescription> AugmentedBlockDescriptions; AugmentedBlockDescriptions.reserve(BlockRawHashes.size()); std::vector<ChunkBlockDescription> FoundBlocks = @@ -9465,7 +9465,7 @@ namespace { { if (!ChunkController && !IsQuiet) { - ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default"); + ZEN_CONSOLE_WARN("Unspecified chunking algorith, using default"); ChunkController = CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{}); } @@ -10576,7 +10576,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; if (!IsQuiet) { - ZEN_CONSOLE("Warning: Using default encryption key"); + ZEN_CONSOLE_WARN("Using default encryption key"); } } @@ -10585,7 +10585,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_EncryptionIV = "0123456789abcdef"; if (!IsQuiet) { - ZEN_CONSOLE("Warning: Using default encryption initialization vector"); + ZEN_CONSOLE_WARN("Using default encryption initialization vector"); } } @@ -10897,7 +10897,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } if (m_ZenCacheHost.empty() && !IsQuiet) { - ZEN_CONSOLE("Warning: Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host); + ZEN_CONSOLE_WARN("Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host); } } } @@ -10917,7 +10917,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - ZEN_WARN("Unable to reach cache host {}. Reason: {}", m_ZenCacheHost, TestResult.second); + ZEN_CONSOLE_WARN("Unable to reach cache host {}. Reason: {}", m_ZenCacheHost, TestResult.second); } } } @@ -11163,163 +11163,73 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BoostWorkerThreads = m_BoostWorkerThreads; UseSparseFiles = m_UseSparseFiles; - try + if (SubOption == &m_ListNamespacesOptions) { - if (SubOption == &m_ListNamespacesOptions) + if (!m_ListResultPath.empty()) { - if (!m_ListResultPath.empty()) + if (!IsQuiet) { - if (!IsQuiet) - { - ZEN_CONSOLE("Running {}: {} (pid {})", - GetRunningExecutablePath(), - ZEN_CFG_VERSION_BUILD_STRING_FULL, - GetCurrentProcessId()); - } + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); } + } - BuildStorage::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() - ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName - : MakeSafeAbsolutePath(m_ZenFolderPath); - CreateDirectories(ZenFolderPath); - auto _ = MakeGuard([ZenFolderPath]() { - if (CleanDirectory(ZenFolderPath, {})) - { - std::error_code DummyEc; - RemoveDir(ZenFolderPath, DummyEc); - } - }); - - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(ZenFolderPath), - /*RequriesNamespace*/ false, - /*RequireBucket*/ false); + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive); - ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); - if (m_ListResultPath.empty()) + const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() + ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName + : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - ZEN_CONSOLE("{}", SB.ToView()); - } - else - { - std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath); - if (ToLower(ListResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } + std::error_code DummyEc; + RemoveDir(ZenFolderPath, DummyEc); } + }); - return 0; - } + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(ZenFolderPath), + /*RequriesNamespace*/ false, + /*RequireBucket*/ false); - if (SubOption == &m_ListOptions) + CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive); + ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); + if (m_ListResultPath.empty()) { - MakeSafeAbsolutePathÍnPlace(m_ListQueryPath); - MakeSafeAbsolutePathÍnPlace(m_ListResultPath); - - if (!m_ListResultPath.empty()) - { - if (!IsQuiet) - { - ZEN_CONSOLE("Running {}: {} (pid {})", - GetRunningExecutablePath(), - ZEN_CFG_VERSION_BUILD_STRING_FULL, - GetCurrentProcessId()); - } - } - CbObject QueryObject; - if (m_ListQueryPath.empty()) + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + ZEN_CONSOLE("{}", SB.ToView()); + } + else + { + std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath); + if (ToLower(ListResultPath.extension().string()) == ".cbo") { - CbObjectWriter QueryWriter; - QueryWriter.BeginObject("query"); - QueryWriter.EndObject(); // query - QueryObject = QueryWriter.Save(); + MemoryView ResponseView = Response.GetView(); + WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); } else { - if (ToLower(m_ListQueryPath.extension().string()) == ".cbo") - { - QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_ListQueryPath)); - } - else - { - IoBuffer MetaDataJson = ReadFile(m_ListQueryPath).Flatten(); - std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); - std::string JsonError; - QueryObject = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); - if (!JsonError.empty()) - { - throw std::runtime_error( - fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_ListQueryPath, JsonError)); - } - } - } - - BuildStorage::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; - } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - - CreateDirectories(m_ZenFolderPath); - auto _ = MakeGuard([this]() { - if (CleanDirectory(m_ZenFolderPath, {})) - { - std::error_code DummyEc; - RemoveDir(m_ZenFolderPath, DummyEc); - } - }); - - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ false); - - CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject); - ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); - if (m_ListResultPath.empty()) - { ExtendableStringBuilder<1024> SB; CompactBinaryToJson(Response.GetView(), SB); - ZEN_CONSOLE("{}", SB.ToView()); + WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); } - else - { - if (ToLower(m_ListResultPath.extension().string()) == ".cbo") - { - MemoryView ResponseView = Response.GetView(); - WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); - } - else - { - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); - } - } - - return 0; } - if (SubOption == &m_UploadOptions) + return 0; + } + + if (SubOption == &m_ListOptions) + { + MakeSafeAbsolutePathÍnPlace(m_ListQueryPath); + MakeSafeAbsolutePathÍnPlace(m_ListResultPath); + + if (!m_ListResultPath.empty()) { if (!IsQuiet) { @@ -11328,544 +11238,439 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); } - - ZenState InstanceState; - - ParsePath(); - - BuildStorage::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - if (m_ZenFolderPath.empty()) + } + CbObject QueryObject; + if (m_ListQueryPath.empty()) + { + CbObjectWriter QueryWriter; + QueryWriter.BeginObject("query"); + QueryWriter.EndObject(); // query + QueryObject = QueryWriter.Save(); + } + else + { + if (ToLower(m_ListQueryPath.extension().string()) == ".cbo") { - m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_ListQueryPath)); } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - - CreateDirectories(m_ZenFolderPath); - auto _ = MakeGuard([this]() { - if (CleanDirectory(m_ZenFolderPath, {})) + else + { + IoBuffer MetaDataJson = ReadFile(m_ListQueryPath).Flatten(); + std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); + std::string JsonError; + QueryObject = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); + if (!JsonError.empty()) { - std::error_code DummyEc; - RemoveDir(m_ZenFolderPath, DummyEc); + throw std::runtime_error( + fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_ListQueryPath, JsonError)); } - }); + } + } - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true); + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - if (m_BuildPartName.empty()) - { - m_BuildPartName = m_Path.filename().string(); - } + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId(); - if (m_BuildId.empty()) - { - m_BuildId = BuildId.ToString(); - } - const Oid BuildPartId = m_BuildPartId.empty() ? Oid::NewOid() : ParseBuildPartId(); - if (m_BuildPartId.empty()) + CreateDirectories(m_ZenFolderPath); + auto _ = MakeGuard([this]() { + if (CleanDirectory(m_ZenFolderPath, {})) { - m_BuildPartId = BuildPartId.ToString(); + std::error_code DummyEc; + RemoveDir(m_ZenFolderPath, DummyEc); } + }); - CbObject MetaData = ParseBuildMetadata(); - - const std::filesystem::path TempDir = ZenTempFolderPath(m_ZenFolderPath); - - m_ManifestPath = RemoveQuotes(m_ManifestPath); + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ false); - UploadFolder(Storage, - BuildId, - BuildPartId, - m_BuildPartName, - m_Path, - TempDir, - m_ManifestPath, - m_FindBlockMaxCount, - m_BlockReuseMinPercentLimit, - m_AllowMultiparts, - MetaData, - m_CreateBuild, - m_Clean, - m_PostUploadVerify); - - if (false) - { - if (!IsQuiet) - { - ZEN_CONSOLE( - "{}:\n" - "Read: {}\n" - "Write: {}\n" - "Requests: {}\n" - "Avg Request Time: {}\n" - "Avg I/O Time: {}", - Storage.StorageName, - NiceBytes(StorageStats.TotalBytesRead.load()), - NiceBytes(StorageStats.TotalBytesWritten.load()), - StorageStats.TotalRequestCount.load(), - StorageStats.TotalExecutionTimeUs.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0, - StorageStats.TotalRequestCount.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0); - } - } - return AbortFlag ? 11 : 0; + CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject); + ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); + if (m_ListResultPath.empty()) + { + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + ZEN_CONSOLE("{}", SB.ToView()); } - - if (SubOption == &m_DownloadOptions) + else { - if (!IsQuiet) + if (ToLower(m_ListResultPath.extension().string()) == ".cbo") { - ZEN_CONSOLE("Running {}: {} (pid {})", - GetRunningExecutablePath(), - ZEN_CFG_VERSION_BUILD_STRING_FULL, - GetCurrentProcessId()); + MemoryView ResponseView = Response.GetView(); + WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); } - - ZenState InstanceState; - - ParsePath(); - ParseFileFilters(); - - if (m_ZenFolderPath.empty()) + else { - m_ZenFolderPath = m_Path / ZenFolderName; + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + } - BuildStorage::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + return 0; + } - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true); + if (SubOption == &m_UploadOptions) + { + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + } - const Oid BuildId = ParseBuildId(); + ZenState InstanceState; - if (m_PostDownloadVerify && m_PrimeCacheOnly) - { - throw zen::OptionParseException( - fmt::format("'cache-prime-only' option is not compatible with 'verify' option\n{}", SubOption->help())); - } + ParsePath(); - if (m_Clean && m_PrimeCacheOnly) - { - ZEN_WARN("ignoring 'clean' option when 'cache-prime-only' is enabled"); - } + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; + + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - if (m_AllowPartialBlockRequests && m_PrimeCacheOnly) + CreateDirectories(m_ZenFolderPath); + auto _ = MakeGuard([this]() { + if (CleanDirectory(m_ZenFolderPath, {})) { - ZEN_WARN("ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled"); + std::error_code DummyEc; + RemoveDir(m_ZenFolderPath, DummyEc); } + }); - std::vector<Oid> BuildPartIds = ParseBuildPartIds(); - std::vector<std::string> BuildPartNames = ParseBuildPartNames(); + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); - // TODO: Add file filters - DownloadFolder(Storage, - BuildId, - BuildPartIds, - BuildPartNames, - m_Path, - m_ZenFolderPath, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests && !m_PrimeCacheOnly, - m_Clean, - m_PostDownloadVerify, - m_PrimeCacheOnly, - m_EnableScavenging, - m_IncludeWildcard, - m_ExcludeWildcard); + if (m_BuildPartName.empty()) + { + m_BuildPartName = m_Path.filename().string(); + } - return AbortFlag ? 11 : 0; + const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId(); + if (m_BuildId.empty()) + { + m_BuildId = BuildId.ToString(); } + const Oid BuildPartId = m_BuildPartId.empty() ? Oid::NewOid() : ParseBuildPartId(); + if (m_BuildPartId.empty()) + { + m_BuildPartId = BuildPartId.ToString(); + } + + CbObject MetaData = ParseBuildMetadata(); + + const std::filesystem::path TempDir = ZenTempFolderPath(m_ZenFolderPath); - if (SubOption == &m_LsOptions) + m_ManifestPath = RemoveQuotes(m_ManifestPath); + + UploadFolder(Storage, + BuildId, + BuildPartId, + m_BuildPartName, + m_Path, + TempDir, + m_ManifestPath, + m_FindBlockMaxCount, + m_BlockReuseMinPercentLimit, + m_AllowMultiparts, + MetaData, + m_CreateBuild, + m_Clean, + m_PostUploadVerify); + + if (false) { if (!IsQuiet) { - ZEN_CONSOLE("Running {}: {} (pid {})", - GetRunningExecutablePath(), - ZEN_CFG_VERSION_BUILD_STRING_FULL, - GetCurrentProcessId()); - } - - ZenState InstanceState; + ZEN_CONSOLE( + "{}:\n" + "Read: {}\n" + "Write: {}\n" + "Requests: {}\n" + "Avg Request Time: {}\n" + "Avg I/O Time: {}", + Storage.StorageName, + NiceBytes(StorageStats.TotalBytesRead.load()), + NiceBytes(StorageStats.TotalBytesWritten.load()), + StorageStats.TotalRequestCount.load(), + StorageStats.TotalExecutionTimeUs.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0, + StorageStats.TotalRequestCount.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0); + } + } + return AbortFlag ? 11 : 0; + } - ParseFileFilters(); + if (SubOption == &m_DownloadOptions) + { + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + } - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = m_Path / ZenFolderName; - } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + ZenState InstanceState; - BuildStorage::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + ParsePath(); + ParseFileFilters(); - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true); + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = m_Path / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - const Oid BuildId = ParseBuildId(); + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - std::vector<Oid> BuildPartIds = ParseBuildPartIds(); - std::vector<std::string> BuildPartNames = ParseBuildPartNames(); + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); - ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, m_IncludeWildcard, m_ExcludeWildcard); + const Oid BuildId = ParseBuildId(); - return AbortFlag ? 11 : 0; + if (m_PostDownloadVerify && m_PrimeCacheOnly) + { + throw zen::OptionParseException( + fmt::format("'cache-prime-only' option is not compatible with 'verify' option\n{}", SubOption->help())); } - if (SubOption == &m_DiffOptions) + if (m_Clean && m_PrimeCacheOnly) { - ParsePath(); - ParseDiffPath(); - - DiffFolders(m_Path, m_DiffPath, m_OnlyChunked); - return AbortFlag ? 11 : 0; + ZEN_CONSOLE_WARN("Ignoring 'clean' option when 'cache-prime-only' is enabled"); } - if (SubOption == &m_FetchBlobOptions) + if (m_AllowPartialBlockRequests && m_PrimeCacheOnly) { - if (!IsQuiet) - { - ZEN_CONSOLE("Running {}: {} (pid {})", - GetRunningExecutablePath(), - ZEN_CFG_VERSION_BUILD_STRING_FULL, - GetCurrentProcessId()); - } + ZEN_CONSOLE_WARN("Ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled"); + } - BuildStorage::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + std::vector<Oid> BuildPartIds = ParseBuildPartIds(); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(); - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; - } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + // TODO: Add file filters + DownloadFolder(Storage, + BuildId, + BuildPartIds, + BuildPartNames, + m_Path, + m_ZenFolderPath, + m_SystemRootDir, + m_AllowMultiparts, + m_AllowPartialBlockRequests && !m_PrimeCacheOnly, + m_Clean, + m_PostDownloadVerify, + m_PrimeCacheOnly, + m_EnableScavenging, + m_IncludeWildcard, + m_ExcludeWildcard); - CreateDirectories(m_ZenFolderPath); - auto _ = MakeGuard([this]() { - if (CleanDirectory(m_ZenFolderPath, {})) - { - std::error_code DummyEc; - RemoveDir(m_ZenFolderPath, DummyEc); - } - }); - - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true); + return AbortFlag ? 11 : 0; + } - IoHash BlobHash = ParseBlobHash(); + if (SubOption == &m_LsOptions) + { + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + } - const Oid BuildId = Oid::FromHexString(m_BuildId); + ZenState InstanceState; - uint64_t CompressedSize; - uint64_t DecompressedSize; - ValidateBlob(*Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); - if (AbortFlag) - { - return 11; - } - if (!IsQuiet) - { - ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", - BlobHash, - CompressedSize, - DecompressedSize); - } - return 0; - } + ParseFileFilters(); - if (SubOption == &m_ValidateBuildPartOptions) + if (m_ZenFolderPath.empty()) { - if (!IsQuiet) - { - ZEN_CONSOLE("Running {}: {} (pid {})", - GetRunningExecutablePath(), - ZEN_CFG_VERSION_BUILD_STRING_FULL, - GetCurrentProcessId()); - } + m_ZenFolderPath = m_Path / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - ZenState InstanceState; + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - BuildStorage::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; - } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - - CreateDirectories(m_ZenFolderPath); - auto _ = MakeGuard([this]() { - if (CleanDirectory(m_ZenFolderPath, {})) - { - std::error_code DummyEc; - RemoveDir(m_ZenFolderPath, DummyEc); - } - }); + const Oid BuildId = ParseBuildId(); - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true); + std::vector<Oid> BuildPartIds = ParseBuildPartIds(); + std::vector<std::string> BuildPartNames = ParseBuildPartNames(); - Oid BuildId = ParseBuildId(); + ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, m_IncludeWildcard, m_ExcludeWildcard); - if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) - { - throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", SubOption->help())); - } + return AbortFlag ? 11 : 0; + } - const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(); + if (SubOption == &m_DiffOptions) + { + ParsePath(); + ParseDiffPath(); - ValidateStatistics ValidateStats; - DownloadStatistics DownloadStats; - ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats); + DiffFolders(m_Path, m_DiffPath, m_OnlyChunked); + return AbortFlag ? 11 : 0; + } - return AbortFlag ? 13 : 0; + if (SubOption == &m_FetchBlobOptions) + { + if (!IsQuiet) + { + ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); } - if (SubOption == &m_MultiTestDownloadOptions) - { - m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(m_SystemRootDir); - CleanDirectory(m_SystemRootDir, {}); - auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); }); + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - ParsePath(); + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - if (m_ZenFolderPath.empty()) + CreateDirectories(m_ZenFolderPath); + auto _ = MakeGuard([this]() { + if (CleanDirectory(m_ZenFolderPath, {})) { - m_ZenFolderPath = m_Path / ZenFolderName; + std::error_code DummyEc; + RemoveDir(m_ZenFolderPath, DummyEc); } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + }); - BuildStorage::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true); + IoHash BlobHash = ParseBlobHash(); - Stopwatch Timer; - for (const std::string& BuildIdString : m_BuildIds) - { - Oid BuildId = Oid::FromHexString(RemoveQuotes(BuildIdString)); - if (BuildId == Oid::Zero) - { - throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, SubOption->help())); - } - DownloadFolder(Storage, - BuildId, - {}, - {}, - m_Path, - m_ZenFolderPath, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - BuildIdString == m_BuildIds.front(), - true, - false, - m_EnableScavenging, - ""sv, - ""sv); - if (AbortFlag) - { - ZEN_CONSOLE("Download cancelled"); - return 11; - } - if (!IsQuiet) - { - ZEN_CONSOLE("\n"); - } - } - if (!IsQuiet) - { - ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - } - return 0; - } + const Oid BuildId = Oid::FromHexString(m_BuildId); - auto ParseZenProcessId = [&]() { - if (m_ZenProcessId == -1) - { - const std::filesystem::path RunningExecutablePath = GetRunningExecutablePath(); - ProcessHandle RunningProcess; - std::error_code Ec = FindProcess(RunningExecutablePath, RunningProcess, /*IncludeSelf*/ false); - if (Ec) - { - throw std::runtime_error( - fmt::format("Failed finding process running '{}', reason: '{}'", RunningExecutablePath, Ec.message())); - } - if (!RunningProcess.IsValid()) - { - throw zen::OptionParseException( - fmt::format("Unable to find a running instance of the zen executable '{}'", RunningExecutablePath)); - } - m_ZenProcessId = RunningProcess.Pid(); - } - }; - - if (SubOption == &m_PauseOptions) + uint64_t CompressedSize; + uint64_t DecompressedSize; + ValidateBlob(*Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); + if (AbortFlag) { - ParseZenProcessId(); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Pause.store(true); - return 0; + return 11; } - - if (SubOption == &m_ResumeOptions) + if (!IsQuiet) { - ParseZenProcessId(); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Pause.store(false); - return 0; + ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", + BlobHash, + CompressedSize, + DecompressedSize); } + return 0; + } - if (SubOption == &m_AbortOptions) + if (SubOption == &m_ValidateBuildPartOptions) + { + if (!IsQuiet) { - ParseZenProcessId(); - ZenState RunningState(m_ZenProcessId); - RunningState.StateData().Abort.store(true); - return 0; + ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); } - if (SubOption == &m_TestOptions) - { - m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(m_SystemRootDir); - CleanDirectory(m_SystemRootDir, {}); - auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); }); + ZenState InstanceState; - ParsePath(); + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - if (m_OverrideHost.empty() && m_StoragePath.empty()) + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + + CreateDirectories(m_ZenFolderPath); + auto _ = MakeGuard([this]() { + if (CleanDirectory(m_ZenFolderPath, {})) { - m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); - CreateDirectories(m_StoragePath); - CleanDirectory(m_StoragePath, {}); - m_StoragePath = m_StoragePath.generic_string(); + std::error_code DummyEc; + RemoveDir(m_ZenFolderPath, DummyEc); } + }); - auto __ = MakeGuard([&]() { - if (m_OverrideHost.empty() && m_StoragePath.empty()) - { - DeleteDirectories(m_StoragePath); - } - }); + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); - BuildStorage::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; + Oid BuildId = ParseBuildId(); - const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_test"); - const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2"); + if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) + { + throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", SubOption->help())); + } - auto ___ = MakeGuard([DownloadPath, DownloadPath2]() { - CleanDirectory(DownloadPath, true); - DeleteDirectories(DownloadPath); - CleanDirectory(DownloadPath2, true); - DeleteDirectories(DownloadPath2); - }); + const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(); - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = m_Path / ZenFolderName; - } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + ValidateStatistics ValidateStats; + DownloadStatistics DownloadStats; + ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats); - StorageInstance Storage = CreateBuildStorage(StorageStats, - StorageCacheStats, - ZenTempFolderPath(m_ZenFolderPath), - /*RequriesNamespace*/ true, - /*RequireBucket*/ true); + return AbortFlag ? 13 : 0; + } - m_BuildId = Oid::NewOid().ToString(); - m_BuildPartName = m_Path.filename().string(); - m_BuildPartId = Oid::NewOid().ToString(); - m_CreateBuild = true; + if (SubOption == &m_MultiTestDownloadOptions) + { + m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_SystemRootDir); + CleanDirectory(m_SystemRootDir, {}); + auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); }); - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + ParsePath(); - auto MakeMetaData = [](const Oid& BuildId) -> CbObject { - CbObjectWriter BuildMetaDataWriter; - { - const uint32_t CL = BuildId.OidBits[2]; - BuildMetaDataWriter.AddString("name", fmt::format("++Test+Main-CL-{}", CL)); - BuildMetaDataWriter.AddString("branch", "ZenTestBuild"); - BuildMetaDataWriter.AddString("baselineBranch", "ZenTestBuild"); - BuildMetaDataWriter.AddString("platform", "Windows"); - BuildMetaDataWriter.AddString("project", "Test"); - BuildMetaDataWriter.AddInteger("changelist", CL); - BuildMetaDataWriter.AddString("buildType", "test-folder"); - } - return BuildMetaDataWriter.Save(); - }; - CbObject MetaData = MakeMetaData(Oid::TryFromHexString(m_BuildId)); - { - ExtendableStringBuilder<256> SB; - CompactBinaryToJson(MetaData, SB); - ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView()); - } + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = m_Path / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path); - // std::filesystem::path UploadTempDir = m_ZenFolderPath / "upload_tmp"; + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - UploadFolder(Storage, - BuildId, - BuildPartId, - m_BuildPartName, - m_Path, - UploadTempDir, - {}, - m_FindBlockMaxCount, - m_BlockReuseMinPercentLimit, - m_AllowMultiparts, - MetaData, - true, - false, - true); - if (AbortFlag) + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); + + Stopwatch Timer; + for (const std::string& BuildIdString : m_BuildIds) + { + Oid BuildId = Oid::FromHexString(RemoveQuotes(BuildIdString)); + if (BuildId == Oid::Zero) { - ZEN_CONSOLE("Upload failed."); - return 11; + throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, SubOption->help())); } - - ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); DownloadFolder(Storage, BuildId, - {BuildPartId}, {}, - DownloadPath, - DownloadPath / ZenFolderName, + {}, + m_Path, + m_ZenFolderPath, m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, - true, + BuildIdString == m_BuildIds.front(), true, false, m_EnableScavenging, @@ -11873,306 +11678,445 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ""sv); if (AbortFlag) { - ZEN_CONSOLE("Download failed."); + ZEN_CONSOLE("Download cancelled"); return 11; } + if (!IsQuiet) + { + ZEN_CONSOLE("\n"); + } + } + if (!IsQuiet) + { + ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } + return 0; + } - ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", - BuildId, - BuildPartId, - m_BuildPartName, - DownloadPath); - DownloadFolder(Storage, - BuildId, - {BuildPartId}, - {}, - DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging, - ""sv, - ""sv); - if (AbortFlag) + auto ParseZenProcessId = [&]() { + if (m_ZenProcessId == -1) + { + const std::filesystem::path RunningExecutablePath = GetRunningExecutablePath(); + ProcessHandle RunningProcess; + std::error_code Ec = FindProcess(RunningExecutablePath, RunningProcess, /*IncludeSelf*/ false); + if (Ec) { - ZEN_CONSOLE("Re-download failed. (identical target)"); - return 11; + throw std::runtime_error( + fmt::format("Failed finding process running '{}', reason: '{}'", RunningExecutablePath, Ec.message())); + } + if (!RunningProcess.IsValid()) + { + throw zen::OptionParseException( + fmt::format("Unable to find a running instance of the zen executable '{}'", RunningExecutablePath)); } + m_ZenProcessId = RunningProcess.Pid(); + } + }; - auto ScrambleDir = [](const std::filesystem::path& Path) { - ZEN_CONSOLE("\nScrambling '{}'", Path); - Stopwatch Timer; - DirectoryContent DownloadContent; - GetDirectoryContent( - Path, - DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, - DownloadContent); - auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders, Path](const std::filesystem::path& AbsolutePath) -> bool { - std::string RelativePath = std::filesystem::relative(AbsolutePath, Path).generic_string(); - for (const std::string_view& ExcludeFolder : ExcludeFolders) - { - if (RelativePath.starts_with(ExcludeFolder)) - { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } - } - } - return true; - }; + if (SubOption == &m_PauseOptions) + { + ParseZenProcessId(); + ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(true); + return 0; + } - ParallelWork Work(AbortFlag, PauseFlag); + if (SubOption == &m_ResumeOptions) + { + ParseZenProcessId(); + ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(false); + return 0; + } - uint32_t Randomizer = 0; - auto FileSizeIt = DownloadContent.FileSizes.begin(); - for (const std::filesystem::path& FilePath : DownloadContent.Files) - { - if (IsAcceptedFolder(FilePath)) - { - uint32_t Case = (Randomizer++) % 7; - switch (Case) - { - case 0: - { - uint64_t SourceSize = *FileSizeIt; - if (SourceSize > 256) - { - Work.ScheduleWork( - GetIOWorkerPool(), - [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) { - if (!AbortFlag) - { - bool IsReadOnly = SetFileReadOnlyWithRetry(FilePath, false); - { - BasicFile Source(FilePath, BasicFile::Mode::kWrite); - uint64_t RangeSize = Min(SourceSize / 3, 512u * 1024u); - IoBuffer TempBuffer1(RangeSize); - IoBuffer TempBuffer2(RangeSize); - IoBuffer TempBuffer3(RangeSize); - Source.Read(TempBuffer1.GetMutableView().GetData(), RangeSize, 0); - Source.Read(TempBuffer2.GetMutableView().GetData(), RangeSize, SourceSize / 2); - Source.Read(TempBuffer3.GetMutableView().GetData(), - RangeSize, - SourceSize - RangeSize); - Source.Write(TempBuffer1, SourceSize / 2); - Source.Write(TempBuffer2, SourceSize - RangeSize); - Source.Write(TempBuffer3, SourceSize - 0); - } - if (IsReadOnly) - { - SetFileReadOnly(FilePath, true); - } - } - }); - } - } - break; - case 1: - { - (void)SetFileReadOnlyWithRetry(FilePath, false); - RemoveFileWithRetry(FilePath); - } - break; - default: - break; - } - } - FileSizeIt++; - } - Work.Wait(5000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, IsPaused); - ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork); - }); - ZEN_ASSERT(!AbortFlag.load()); - ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }; + if (SubOption == &m_AbortOptions) + { + ParseZenProcessId(); + ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Abort.store(true); + return 0; + } - ScrambleDir(DownloadPath); - ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", - BuildId, - BuildPartId, - m_BuildPartName, - DownloadPath); - DownloadFolder(Storage, - BuildId, - {BuildPartId}, - {}, - DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging, - ""sv, - ""sv); - if (AbortFlag) - { - ZEN_CONSOLE("Re-download failed. (scrambled target)"); - return 11; - } + if (SubOption == &m_TestOptions) + { + m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_SystemRootDir); + CleanDirectory(m_SystemRootDir, {}); + auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); }); - ScrambleDir(DownloadPath); + ParsePath(); - Oid BuildId2 = Oid::NewOid(); - Oid BuildPartId2 = Oid::NewOid(); + if (m_OverrideHost.empty() && m_StoragePath.empty()) + { + m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); + CreateDirectories(m_StoragePath); + CleanDirectory(m_StoragePath, {}); + m_StoragePath = m_StoragePath.generic_string(); + } - CbObject MetaData2 = MakeMetaData(BuildId2); + auto __ = MakeGuard([&]() { + if (m_OverrideHost.empty() && m_StoragePath.empty()) { - ExtendableStringBuilder<256> SB; - CompactBinaryToJson(MetaData, SB); - ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView()); + DeleteDirectories(m_StoragePath); } + }); - UploadFolder(Storage, - BuildId2, - BuildPartId2, - m_BuildPartName, - DownloadPath, - UploadTempDir, - {}, - m_FindBlockMaxCount, - m_BlockReuseMinPercentLimit, - m_AllowMultiparts, - MetaData2, - true, - false, - true); - if (AbortFlag) - { - ZEN_CONSOLE("Upload of scrambled failed."); - return 11; - } + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, - BuildId, - {BuildPartId}, - {}, - DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging, - ""sv, - ""sv); - if (AbortFlag) - { - ZEN_CONSOLE("Re-download failed."); - return 11; - } + const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_test"); + const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2"); - ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, - BuildId2, - {BuildPartId2}, - {}, - DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging, - ""sv, - ""sv); - if (AbortFlag) - { - ZEN_CONSOLE("Re-download failed."); - return 11; - } + auto ___ = MakeGuard([DownloadPath, DownloadPath2]() { + CleanDirectory(DownloadPath, true); + DeleteDirectories(DownloadPath); + CleanDirectory(DownloadPath2, true); + DeleteDirectories(DownloadPath2); + }); - ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, - BuildId2, - {BuildPartId2}, - {}, - DownloadPath, - DownloadPath / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging, - ""sv, - ""sv); - if (AbortFlag) + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = m_Path / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); + + m_BuildId = Oid::NewOid().ToString(); + m_BuildPartName = m_Path.filename().string(); + m_BuildPartId = Oid::NewOid().ToString(); + m_CreateBuild = true; + + const Oid BuildId = Oid::FromHexString(m_BuildId); + const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + + auto MakeMetaData = [](const Oid& BuildId) -> CbObject { + CbObjectWriter BuildMetaDataWriter; { - ZEN_CONSOLE("Re-download failed."); - return 11; + const uint32_t CL = BuildId.OidBits[2]; + BuildMetaDataWriter.AddString("name", fmt::format("++Test+Main-CL-{}", CL)); + BuildMetaDataWriter.AddString("branch", "ZenTestBuild"); + BuildMetaDataWriter.AddString("baselineBranch", "ZenTestBuild"); + BuildMetaDataWriter.AddString("platform", "Windows"); + BuildMetaDataWriter.AddString("project", "Test"); + BuildMetaDataWriter.AddInteger("changelist", CL); + BuildMetaDataWriter.AddString("buildType", "test-folder"); } + return BuildMetaDataWriter.Save(); + }; + CbObject MetaData = MakeMetaData(Oid::TryFromHexString(m_BuildId)); + { + ExtendableStringBuilder<256> SB; + CompactBinaryToJson(MetaData, SB); + ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView()); + } - ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2); - DownloadFolder(Storage, - BuildId, - {BuildPartId}, - {}, - DownloadPath2, - DownloadPath2 / ZenFolderName, - m_SystemRootDir, - m_AllowMultiparts, - m_AllowPartialBlockRequests, - false, - true, - false, - m_EnableScavenging, - ""sv, - ""sv); - if (AbortFlag) + const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path); + // std::filesystem::path UploadTempDir = m_ZenFolderPath / "upload_tmp"; + + UploadFolder(Storage, + BuildId, + BuildPartId, + m_BuildPartName, + m_Path, + UploadTempDir, + {}, + m_FindBlockMaxCount, + m_BlockReuseMinPercentLimit, + m_AllowMultiparts, + MetaData, + true, + false, + true); + if (AbortFlag) + { + ZEN_CONSOLE_ERROR("Upload failed."); + return 11; + } + + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + DownloadPath / ZenFolderName, + m_SystemRootDir, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + true, + true, + false, + m_EnableScavenging, + ""sv, + ""sv); + if (AbortFlag) + { + ZEN_CONSOLE_ERROR("Download failed."); + return 11; + } + + ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + DownloadPath / ZenFolderName, + m_SystemRootDir, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false, + m_EnableScavenging, + ""sv, + ""sv); + if (AbortFlag) + { + ZEN_CONSOLE_ERROR("Re-download failed. (identical target)"); + return 11; + } + + auto ScrambleDir = [](const std::filesystem::path& Path) { + ZEN_CONSOLE("\nScrambling '{}'", Path); + Stopwatch Timer; + DirectoryContent DownloadContent; + GetDirectoryContent( + Path, + DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, + DownloadContent); + auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders, Path](const std::filesystem::path& AbsolutePath) -> bool { + std::string RelativePath = std::filesystem::relative(AbsolutePath, Path).generic_string(); + for (const std::string_view& ExcludeFolder : ExcludeFolders) + { + if (RelativePath.starts_with(ExcludeFolder)) + { + if (RelativePath.length() == ExcludeFolder.length()) + { + return false; + } + else if (RelativePath[ExcludeFolder.length()] == '/') + { + return false; + } + } + } + return true; + }; + + ParallelWork Work(AbortFlag, PauseFlag); + + uint32_t Randomizer = 0; + auto FileSizeIt = DownloadContent.FileSizes.begin(); + for (const std::filesystem::path& FilePath : DownloadContent.Files) { - ZEN_CONSOLE("Re-download failed."); - return 11; + if (IsAcceptedFolder(FilePath)) + { + uint32_t Case = (Randomizer++) % 7; + switch (Case) + { + case 0: + { + uint64_t SourceSize = *FileSizeIt; + if (SourceSize > 256) + { + Work.ScheduleWork( + GetIOWorkerPool(), + [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) { + if (!AbortFlag) + { + bool IsReadOnly = SetFileReadOnlyWithRetry(FilePath, false); + { + BasicFile Source(FilePath, BasicFile::Mode::kWrite); + uint64_t RangeSize = Min(SourceSize / 3, 512u * 1024u); + IoBuffer TempBuffer1(RangeSize); + IoBuffer TempBuffer2(RangeSize); + IoBuffer TempBuffer3(RangeSize); + Source.Read(TempBuffer1.GetMutableView().GetData(), RangeSize, 0); + Source.Read(TempBuffer2.GetMutableView().GetData(), RangeSize, SourceSize / 2); + Source.Read(TempBuffer3.GetMutableView().GetData(), RangeSize, SourceSize - RangeSize); + Source.Write(TempBuffer1, SourceSize / 2); + Source.Write(TempBuffer2, SourceSize - RangeSize); + Source.Write(TempBuffer3, SourceSize - 0); + } + if (IsReadOnly) + { + SetFileReadOnly(FilePath, true); + } + } + }); + } + } + break; + case 1: + { + (void)SetFileReadOnlyWithRetry(FilePath, false); + RemoveFileWithRetry(FilePath); + } + break; + default: + break; + } + } + FileSizeIt++; } + Work.Wait(5000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, IsPaused); + ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork); + }); + ZEN_ASSERT(!AbortFlag.load()); + ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }; - return 0; - } - } - catch (const std::system_error& SysErr) - { - if (IsOOD(SysErr)) + ScrambleDir(DownloadPath); + ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + DownloadPath / ZenFolderName, + m_SystemRootDir, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false, + m_EnableScavenging, + ""sv, + ""sv); + if (AbortFlag) { - ZEN_CONSOLE("Operation failed due to out of disk space: {}", SysErr.what()); - return 3; + ZEN_CONSOLE_ERROR("Re-download failed. (scrambled target)"); + return 11; } - else if (IsOOM(SysErr)) + + ScrambleDir(DownloadPath); + + Oid BuildId2 = Oid::NewOid(); + Oid BuildPartId2 = Oid::NewOid(); + + CbObject MetaData2 = MakeMetaData(BuildId2); { - ZEN_CONSOLE("Operation failed due to out of memory: {}", SysErr.what()); - return 3; + ExtendableStringBuilder<256> SB; + CompactBinaryToJson(MetaData, SB); + ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView()); } - else + + UploadFolder(Storage, + BuildId2, + BuildPartId2, + m_BuildPartName, + DownloadPath, + UploadTempDir, + {}, + m_FindBlockMaxCount, + m_BlockReuseMinPercentLimit, + m_AllowMultiparts, + MetaData2, + true, + false, + true); + if (AbortFlag) + { + ZEN_CONSOLE_ERROR("Upload of scrambled failed."); + return 11; + } + + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + DownloadPath / ZenFolderName, + m_SystemRootDir, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false, + m_EnableScavenging, + ""sv, + ""sv); + if (AbortFlag) + { + ZEN_CONSOLE_ERROR("Re-download failed."); + return 11; + } + + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); + DownloadFolder(Storage, + BuildId2, + {BuildPartId2}, + {}, + DownloadPath, + DownloadPath / ZenFolderName, + m_SystemRootDir, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false, + m_EnableScavenging, + ""sv, + ""sv); + if (AbortFlag) + { + ZEN_CONSOLE_ERROR("Re-download failed."); + return 11; + } + + ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); + DownloadFolder(Storage, + BuildId2, + {BuildPartId2}, + {}, + DownloadPath, + DownloadPath / ZenFolderName, + m_SystemRootDir, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false, + m_EnableScavenging, + ""sv, + ""sv); + if (AbortFlag) { - ZEN_ERROR("{}", SysErr.what()); - return 3; + ZEN_CONSOLE_ERROR("Re-download failed."); + return 11; + } + + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath2, + DownloadPath2 / ZenFolderName, + m_SystemRootDir, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false, + m_EnableScavenging, + ""sv, + ""sv); + if (AbortFlag) + { + ZEN_CONSOLE_ERROR("Re-download failed."); + return 11; } - } - catch (const HttpClientError& HttpEx) - { - ZEN_CONSOLE("Operation failed: {}", HttpEx.what()); - return HttpEx.m_Error != 0 ? HttpEx.m_Error : (int)HttpEx.m_ResponseCode; - } - catch (const std::exception& Ex) - { - ZEN_ERROR("{}", Ex.what()); - return 3; + + return 0; } ZEN_ASSERT(false); } diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp index 4412eaf34..90f428b29 100644 --- a/src/zen/cmds/cache_cmd.cpp +++ b/src/zen/cmds/cache_cmd.cpp @@ -115,11 +115,11 @@ DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (Result.status_code) { - ZEN_ERROR("Drop failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); + ZEN_CONSOLE_ERROR("Drop failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); } else { - ZEN_ERROR("Drop failed: {}", Result.error.message); + ZEN_CONSOLE_ERROR("Drop failed: {}", Result.error.message); } return 1; @@ -217,11 +217,11 @@ CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (Result.status_code) { - ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); + ZEN_CONSOLE_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); } else { - ZEN_ERROR("Info failed: {}", Result.error.message); + ZEN_CONSOLE_ERROR("Info failed: {}", Result.error.message); } return 1; @@ -267,11 +267,11 @@ CacheStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv if (Result.status_code) { - ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); + ZEN_CONSOLE_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); } else { - ZEN_ERROR("Info failed: {}", Result.error.message); + ZEN_CONSOLE_ERROR("Info failed: {}", Result.error.message); } return 1; @@ -336,9 +336,7 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar { if (m_Namespace.empty() || m_Bucket.empty()) { - ZEN_ERROR("Provide namespace and bucket name"); - ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); - return 1; + throw OptionParseException("Provide namespace and bucket name"); } Session.SetUrl({fmt::format("{}/z$/details$/{}/{}/{}", m_HostName, m_Namespace, m_Bucket, m_ValueKey)}); } @@ -346,9 +344,7 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar { if (m_Namespace.empty()) { - ZEN_ERROR("Provide namespace name"); - ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); - return 1; + throw OptionParseException("Provide namespace name"); } Session.SetUrl({fmt::format("{}/z$/details$/{}/{}", m_HostName, m_Namespace, m_Bucket)}); } @@ -373,11 +369,11 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar if (Result.status_code) { - ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); + ZEN_CONSOLE_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); } else { - ZEN_ERROR("Info failed: {}", Result.error.message); + ZEN_CONSOLE_ERROR("Info failed: {}", Result.error.message); } return 1; diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp index e86b6964c..4e54f27bb 100644 --- a/src/zen/cmds/copy_cmd.cpp +++ b/src/zen/cmds/copy_cmd.cpp @@ -147,7 +147,7 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ++FailedFileCount; - ZEN_CONSOLE_ERROR("failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what()); + ZEN_CONSOLE_ERROR("Failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what()); } } } @@ -208,7 +208,7 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } catch (const std::exception& Ex) { - ZEN_CONSOLE_ERROR("Error: failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what()); + ZEN_CONSOLE_ERROR("Failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what()); return 1; } diff --git a/src/zen/cmds/dedup_cmd.cpp b/src/zen/cmds/dedup_cmd.cpp index 5f74305c7..033ac87c3 100644 --- a/src/zen/cmds/dedup_cmd.cpp +++ b/src/zen/cmds/dedup_cmd.cpp @@ -68,15 +68,13 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!SourceGood) { - ZEN_ERROR("Source directory '{}' does not support deduplication", m_DedupSource); - + ZEN_CONSOLE_ERROR("Source directory '{}' does not support deduplication", m_DedupSource); return 0; } if (!TargetGood) { - ZEN_ERROR("Target directory '{}' does not support deduplication", m_DedupTarget); - + ZEN_CONSOLE_ERROR("Target directory '{}' does not support deduplication", m_DedupTarget); return 0; } diff --git a/src/zen/cmds/info_cmd.cpp b/src/zen/cmds/info_cmd.cpp index 73a5d2e5d..8e0b3d663 100644 --- a/src/zen/cmds/info_cmd.cpp +++ b/src/zen/cmds/info_cmd.cpp @@ -44,6 +44,10 @@ InfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("{}", Result.ToText()); } + else + { + Result.ThrowError(fmt::format("Failed getting info from {}", m_HostName)); + } return 0; } diff --git a/src/zen/cmds/print_cmd.cpp b/src/zen/cmds/print_cmd.cpp index 5bc8a8ed8..fd6d92f28 100644 --- a/src/zen/cmds/print_cmd.cpp +++ b/src/zen/cmds/print_cmd.cpp @@ -84,7 +84,7 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (Fc.ErrorCode) { - ZEN_ERROR("Failed to read file '{}': {}", m_Filename, Fc.ErrorCode.message()); + ZEN_CONSOLE_ERROR("Failed to read file '{}': {}", m_Filename, Fc.ErrorCode.message()); return 1; } @@ -149,7 +149,7 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (FilteredResult == CbValidateError::None && FilteredResult != Result) { - ZEN_WARN( + ZEN_CONSOLE_WARN( "Object in package message file '{}' does not appear to be an optimal compact binary format (validation error {:#x}: '{}')", m_Filename, uint32_t(Result), @@ -158,10 +158,10 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (FilteredResult != CbValidateError::None) { - ZEN_ERROR("Object in package message file '{}' does not appear to be compact binary (validation error {:#x}: '{}')", - m_Filename, - uint32_t(FilteredResult), - ToString(FilteredResult)); + ZEN_CONSOLE_ERROR("Object in package message file '{}' does not appear to be compact binary (validation error {:#x}: '{}')", + m_Filename, + uint32_t(FilteredResult), + ToString(FilteredResult)); return 1; } else @@ -177,18 +177,18 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (FilteredResult == CbValidateError::None && FilteredResult != Result) { - ZEN_WARN("Data in file '{}' does not appear to be an optimal compact binary format (validation error {:#x}: '{}')", - m_Filename, - uint32_t(Result), - ToString(Result)); + ZEN_CONSOLE_WARN("Data in file '{}' does not appear to be an optimal compact binary format (validation error {:#x}: '{}')", + m_Filename, + uint32_t(Result), + ToString(Result)); } if (FilteredResult != CbValidateError::None) { - ZEN_ERROR("Data in file '{}' does not appear to be compact binary (validation error {:#x}: '{}')", - m_Filename, - uint32_t(FilteredResult), - ToString(FilteredResult)); + ZEN_CONSOLE_ERROR("Data in file '{}' does not appear to be compact binary (validation error {:#x}: '{}')", + m_Filename, + uint32_t(FilteredResult), + ToString(FilteredResult)); return 1; } else diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index f919edc87..5f0f52f7c 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -449,7 +449,7 @@ namespace { } else { - ZEN_CONSOLE("Unable to resolve project name '{}', no projects found at {}", OptionalProjectName, Http.GetBaseUri()); + ZEN_CONSOLE_WARN("Unable to resolve project name '{}', no projects found at {}", OptionalProjectName, Http.GetBaseUri()); } return {}; } @@ -470,10 +470,10 @@ namespace { std::vector<std::string> MatchingProjectIds = MatchId(AvailableProjects, OptionalProjectName); if (MatchingProjectIds.empty()) { - ZEN_CONSOLE("Unable to match project name '{}' at {}, available projects: {}", - OptionalProjectName, - Http.GetBaseUri(), - FmtProjectIdArray(AvailableProjects)); + ZEN_CONSOLE_WARN("Unable to match project name '{}' at {}, available projects: {}", + OptionalProjectName, + Http.GetBaseUri(), + FmtProjectIdArray(AvailableProjects)); return {}; } @@ -482,10 +482,10 @@ namespace { return MatchingProjectIds.front(); } - ZEN_CONSOLE("Project name is ambigous '{}' at {}: possible matches: {}", - OptionalProjectName, - Http.GetBaseUri(), - FmtProjectIdArray(MatchingProjectIds)); + ZEN_CONSOLE_WARN("Project name is ambigous '{}' at {}: possible matches: {}", + OptionalProjectName, + Http.GetBaseUri(), + FmtProjectIdArray(MatchingProjectIds)); return {}; } @@ -500,10 +500,10 @@ namespace { } else { - ZEN_CONSOLE("Unable to resolve oplog name '{}' for project '{}', no oplogs found at {}", - OptionalOplogName, - ProjectName, - Http.GetBaseUri()); + ZEN_CONSOLE_WARN("Unable to resolve oplog name '{}' for project '{}', no oplogs found at {}", + OptionalOplogName, + ProjectName, + Http.GetBaseUri()); } return {}; } @@ -524,11 +524,11 @@ namespace { std::vector<std::string> MatchingOplogIds = MatchId(AvailableOplogs, OptionalOplogName); if (MatchingOplogIds.empty()) { - ZEN_CONSOLE("Unable to match oplog name '{}' for project '{}', available oplogs at {}: {}", - OptionalOplogName, - ProjectName, - Http.GetBaseUri(), - FmtArray(AvailableOplogs)); + ZEN_CONSOLE_WARN("Unable to match oplog name '{}' for project '{}', available oplogs at {}: {}", + OptionalOplogName, + ProjectName, + Http.GetBaseUri(), + FmtArray(AvailableOplogs)); return {}; } @@ -537,11 +537,11 @@ namespace { return MatchingOplogIds.front(); } - ZEN_CONSOLE("Oplog name '{}' for project '{}' at {}, is ambigous, possible matches: {}", - OptionalOplogName, - ProjectName, - Http.GetBaseUri(), - FmtArray(MatchingOplogIds)); + ZEN_CONSOLE_WARN("Oplog name '{}' for project '{}' at {}, is ambigous, possible matches: {}", + OptionalOplogName, + ProjectName, + Http.GetBaseUri(), + FmtArray(MatchingOplogIds)); return {}; } @@ -766,8 +766,7 @@ CreateProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a if (m_ProjectId.empty()) { - ZEN_ERROR("Project name must be given"); - return 1; + throw OptionParseException("Project name must be given"); } HttpClient Http(m_HostName); @@ -778,7 +777,7 @@ CreateProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a { if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) { - ZEN_CONSOLE("Project already exists.\n{}", Result.ToText()); + ZEN_CONSOLE_ERROR("Project already exists.\n{}", Result.ToText()); return 1; } } @@ -1073,9 +1072,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg { if (m_JupiterNamespace.empty() || m_JupiterBucket.empty()) { - ZEN_ERROR("Options for cloud target are missing"); - ZEN_CONSOLE("{}", m_Options.help({"cloud"}).c_str()); - return 1; + throw OptionParseException("Options for cloud target are missing"); } if (m_CloudKey.empty()) { @@ -1088,23 +1085,21 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg if (!m_BuildsUrl.empty()) { + if (m_JupiterNamespace.empty()) + { + throw OptionParseException("Namespace for builds target it missing"); + } if (m_JupiterNamespace.empty() || m_JupiterBucket.empty()) { - ZEN_ERROR("Options for builds target are missing"); - ZEN_CONSOLE("{}", m_Options.help({"builds"}).c_str()); - return 1; + throw OptionParseException("Bucket for builds target it missing"); } if (m_BuildsMetadataPath.empty() && m_BuildsMetadata.empty()) { - ZEN_ERROR("Options for builds target are missing"); - ZEN_CONSOLE("{}", m_Options.help({"builds"}).c_str()); - return 1; + throw OptionParseException("Options for builds target metadata are missing"); } if (!m_BuildsMetadataPath.empty() && !m_BuildsMetadata.empty()) { - ZEN_ERROR("Conflicting options for builds target"); - ZEN_CONSOLE("{}", m_Options.help({"builds"}).c_str()); - return 1; + throw OptionParseException("Conflicting options for builds target metadata"); } if (m_BuildsId.empty()) { @@ -1118,12 +1113,12 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg if (m_ZenProjectName.empty()) { m_ZenProjectName = m_ProjectName; - ZEN_WARN("Using default zen target project id '{}'", m_ZenProjectName); + ZEN_CONSOLE_WARN("Using default zen target project id '{}'", m_ZenProjectName); } if (m_ZenOplogName.empty()) { m_ZenOplogName = m_OplogName; - ZEN_WARN("Using default zen target oplog id '{}'", m_ZenOplogName); + ZEN_CONSOLE_WARN("Using default zen target oplog id '{}'", m_ZenOplogName); } std::string TargetUrlBase = m_ZenUrl; @@ -1141,7 +1136,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg { if (m_ZenClean) { - ZEN_WARN("Deleting zen remote oplog '{}/{}'", m_ZenProjectName, m_ZenOplogName) + ZEN_CONSOLE_WARN("Deleting zen remote oplog '{}/{}'", m_ZenProjectName, m_ZenOplogName); Result = TargetHttp.Delete(Url, HttpClient::Accept(ZenContentType::kJSON)); if (!Result) { @@ -1163,7 +1158,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg if (CreateOplog) { - ZEN_WARN("Creating zen remote oplog '{}/{}'", m_ZenProjectName, m_ZenOplogName); + ZEN_CONSOLE_WARN("Creating zen remote oplog '{}/{}'", m_ZenProjectName, m_ZenOplogName); if (HttpClient::Response Result = TargetHttp.Post(Url); !Result) { Result.ThrowError("failed creating zen remote oplog"sv); @@ -1177,7 +1172,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg if (m_FileName.empty()) { m_FileName = m_OplogName; - ZEN_WARN("Using default file name '{}'", m_FileName); + ZEN_CONSOLE_WARN("Using default file name '{}'", m_FileName); } } @@ -1370,21 +1365,11 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg m_PlainProgress); } } - catch (const HttpClientError& Ex) - { - ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what()); - return Ex.m_Error != 0 ? Ex.m_Error : (int)Ex.m_ResponseCode; - } catch (const AsyncJobError& Ex) { - ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what()); + ZEN_CONSOLE_ERROR("Oplog export failed: '{}'", Ex.what()); return Ex.m_ReturnCode; } - catch (const std::exception& Ex) - { - ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what()); - return 1; - } return 0; } @@ -1498,14 +1483,12 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg if (m_ProjectName.empty()) { - ZEN_ERROR("Project name must be given"); - return 1; + throw OptionParseException("Project name must be given"); } if (m_OplogName.empty()) { - ZEN_ERROR("Oplog name must be given"); - return 1; + throw OptionParseException("Oplog name must be given"); } HttpClient Http(m_HostName); @@ -1522,35 +1505,41 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg TargetCount += m_FileDirectoryPath.empty() ? 0 : 1; if (TargetCount != 1) { - ZEN_ERROR("Provide one source only"); - ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); - return 1; + throw OptionParseException("Provide one source only"); } if (!m_CloudUrl.empty()) { - if (m_JupiterNamespace.empty() || m_JupiterBucket.empty()) + if (m_JupiterNamespace.empty()) { - ZEN_ERROR("Options for cloud source are missing"); - ZEN_CONSOLE("{}", m_Options.help({"cloud"}).c_str()); - return 1; + throw OptionParseException("Namespace option for cloud source is missing"); + } + if (m_JupiterBucket.empty()) + { + throw OptionParseException("Bucket option for cloud source is missing"); } if (m_CloudKey.empty()) { std::string KeyString = fmt::format("{}/{}/{}/{}", m_ProjectName, m_OplogName, m_JupiterNamespace, m_JupiterBucket); IoHash Key = IoHash::HashBuffer(KeyString.data(), KeyString.size()); m_CloudKey = Key.ToHexString(); - ZEN_WARN("Using auto generated cloud key '{}'", m_CloudKey); + ZEN_CONSOLE_WARN("Using auto generated cloud key '{}'", m_CloudKey); } } if (!m_BuildsUrl.empty()) { - if (m_JupiterNamespace.empty() || m_JupiterBucket.empty() || m_BuildsId.empty()) + if (m_JupiterNamespace.empty()) { - ZEN_ERROR("Options for builds source are missing"); - ZEN_CONSOLE("{}", m_Options.help({"builds"}).c_str()); - return 1; + throw OptionParseException("Namespace option for builds source is missing"); + } + if (m_JupiterBucket.empty()) + { + throw OptionParseException("Bucket option for builds source is missing"); + } + if (m_BuildsId.empty()) + { + throw OptionParseException("Build id option for builds source is missing"); } } @@ -1559,12 +1548,12 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg if (m_ZenProjectName.empty()) { m_ZenProjectName = m_ProjectName; - ZEN_WARN("Using default zen target project id '{}'", m_ZenProjectName); + ZEN_CONSOLE_WARN("Using default zen target project id '{}'", m_ZenProjectName); } if (m_ZenOplogName.empty()) { m_ZenOplogName = m_OplogName; - ZEN_WARN("Using default zen target oplog id '{}'", m_ZenOplogName); + ZEN_CONSOLE_WARN("Using default zen target oplog id '{}'", m_ZenOplogName); } } @@ -1573,7 +1562,7 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg if (m_FileName.empty()) { m_FileName = m_OplogName; - ZEN_WARN("Using auto generated file name '{}'", m_FileName); + ZEN_CONSOLE_WARN("Using auto generated file name '{}'", m_FileName); } } @@ -1598,7 +1587,7 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg { OplogPayload = MakeCbObjectPayload([&](CbObjectWriter& Writer) { Writer.AddString("gcpath"sv, m_GcPath); }); } - ZEN_WARN("Creating oplog '{}/{}'", m_ProjectName, m_OplogName); + ZEN_CONSOLE("Creating oplog '{}/{}'", m_ProjectName, m_OplogName); if (HttpClient::Response Result = Http.Post(Url, OplogPayload); !Result) { Result.ThrowError("failed creating oplog"sv); @@ -1723,21 +1712,11 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg m_PlainProgress); } } - catch (const HttpClientError& Ex) - { - ZEN_CONSOLE("Oplog import failed: '{}'", Ex.what()); - return Ex.m_Error != 0 ? Ex.m_Error : (int)Ex.m_ResponseCode; - } catch (const AsyncJobError& Ex) { - ZEN_CONSOLE("Oplog export failed: '{}'", Ex.what()); + ZEN_CONSOLE_ERROR("Oplog export failed: '{}'", Ex.what()); return Ex.m_ReturnCode; } - catch (const std::exception& Ex) - { - ZEN_CONSOLE("Oplog import failed: '{}'", Ex.what()); - return 1; - } return 0; } @@ -1780,7 +1759,7 @@ SnapshotOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a if (m_ProjectName.empty()) { - ZEN_ERROR("Project name must be given"); + throw OptionParseException("Project name must be given"); return 1; } @@ -2147,7 +2126,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg catch (const std::exception& Ex) { AbortFlag.store(true); - ZEN_CONSOLE("Failed writing file to '{}'. Reason: '{}'", TargetPath, Ex.what()); + ZEN_CONSOLE_ERROR("Failed writing file to '{}'. Reason: '{}'", TargetPath, Ex.what()); } } }); @@ -2233,7 +2212,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg } else { - ZEN_ERROR("unknown format response to oplog entries request"); + ZEN_CONSOLE_ERROR("Unknown format response to oplog entries request"); } } else diff --git a/src/zen/cmds/rpcreplay_cmd.cpp b/src/zen/cmds/rpcreplay_cmd.cpp index 4fc38d92a..6c26eaf23 100644 --- a/src/zen/cmds/rpcreplay_cmd.cpp +++ b/src/zen/cmds/rpcreplay_cmd.cpp @@ -429,7 +429,7 @@ RpcReplayCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (Response.error || !(IsHttpSuccessCode(Response.status_code) || Response.status_code == gsl::narrow<long>(HttpResponseCode::NotFound))) { - ZEN_CONSOLE("{}", FormatHttpResponse(Response)); + ZEN_CONSOLE_ERROR("{}", FormatHttpResponse(Response)); break; } BytesReceived.fetch_add(Response.downloaded_bytes); diff --git a/src/zen/cmds/serve_cmd.cpp b/src/zen/cmds/serve_cmd.cpp index 64039e4c9..cfaa12bc0 100644 --- a/src/zen/cmds/serve_cmd.cpp +++ b/src/zen/cmds/serve_cmd.cpp @@ -93,7 +93,7 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } catch (const std::exception& Ex) { - ZEN_CONSOLE("failed to spawn server on port {}: '{}'", ServerPort, Ex.what()); + ZEN_CONSOLE_ERROR("Failed to spawn server on port {}: '{}'", ServerPort, Ex.what()); throw zen::OptionParseException("unable to resolve server specification (even after spawning server)"); } @@ -111,7 +111,7 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (ServerInstance) { m_HostName = ServerInstance->GetBaseUri(); - ZEN_CONSOLE("base uri: {}", m_HostName); + ZEN_CONSOLE("Base uri: {}", m_HostName); } // Generate manifest for tree @@ -159,7 +159,7 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) for (const FsVisitor::FileEntry& Entry : Visitor.Files) { - ZEN_CONSOLE("file: {}", Entry.FilePath); + ZEN_CONSOLE("File: {}", Entry.FilePath); Cbo.BeginObject(); @@ -225,12 +225,10 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!HttpResponse) { - ZEN_CONSOLE("error: failed to append manifest!"); - - return 1; + HttpResponse.ThrowError("Failed to append manifest"); } - ZEN_CONSOLE("ok serving files now"); + ZEN_CONSOLE("Ok serving files now"); #if ZEN_PLATFORM_WINDOWS _getch(); // TEMPORARY HACK diff --git a/src/zen/cmds/status_cmd.cpp b/src/zen/cmds/status_cmd.cpp index 2b507e43d..b5764af44 100644 --- a/src/zen/cmds/status_cmd.cpp +++ b/src/zen/cmds/status_cmd.cpp @@ -34,14 +34,14 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (!IsFile(m_DataDir / ".lock")) { - ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir); + ZEN_CONSOLE_ERROR("Lock file does not exist in directory '{}'", m_DataDir); return 1; } LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); std::string Reason; if (!ValidateLockFileInfo(Info, Reason)) { - ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); + ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); return 1; } EffectivePort = Info.EffectiveListenPort; @@ -50,8 +50,7 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZenServerState State; if (!State.InitializeReadOnly()) { - ZEN_CONSOLE("no Zen state found"); - + ZEN_CONSOLE("No Zen state found"); return 0; } diff --git a/src/zen/cmds/top_cmd.cpp b/src/zen/cmds/top_cmd.cpp index 568ee76c9..9794dc1c0 100644 --- a/src/zen/cmds/top_cmd.cpp +++ b/src/zen/cmds/top_cmd.cpp @@ -27,7 +27,7 @@ TopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZenServerState State; if (!State.InitializeReadOnly()) { - ZEN_CONSOLE("no Zen state found"); + ZEN_CONSOLE("No Zen state found"); return 0; } @@ -75,7 +75,7 @@ PsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZenServerState State; if (!State.InitializeReadOnly()) { - ZEN_CONSOLE("no Zen state found"); + ZEN_CONSOLE("No Zen state found"); return 0; } diff --git a/src/zen/cmds/trace_cmd.cpp b/src/zen/cmds/trace_cmd.cpp index dacc13ebf..85caf33b8 100644 --- a/src/zen/cmds/trace_cmd.cpp +++ b/src/zen/cmds/trace_cmd.cpp @@ -48,7 +48,7 @@ TraceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - ZEN_ERROR("trace stop failed: {}", Response.ToText()); + ZEN_CONSOLE_ERROR("trace stop failed: {}", Response.ToText()); return 1; } } @@ -72,7 +72,7 @@ TraceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - ZEN_ERROR("trace start failed: {}", Response.ToText()); + ZEN_CONSOLE_ERROR("trace start failed: {}", Response.ToText()); return 1; } } @@ -84,7 +84,7 @@ TraceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - ZEN_ERROR("trace status failed: {}", Response.ToText()); + ZEN_CONSOLE_ERROR("trace status failed: {}", Response.ToText()); } return 1; diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp index d0763701e..fd330f616 100644 --- a/src/zen/cmds/up_cmd.cpp +++ b/src/zen/cmds/up_cmd.cpp @@ -98,7 +98,7 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (Server.IsRunning()) { - ZEN_ERROR("zen server launch failed (timed out), terminating"); + ZEN_CONSOLE_ERROR("Zen server launch failed (timed out), terminating"); Server.Terminate(); if (!m_ShowConsole) { @@ -121,7 +121,7 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - ZEN_CONSOLE("zen server up"); + ZEN_CONSOLE("Zen server up"); } } return 0; @@ -157,14 +157,14 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (!IsFile(m_DataDir / ".lock")) { - ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir); + ZEN_CONSOLE("Lock file does not exist in directory '{}'", m_DataDir); return 1; } LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); std::string Reason; if (!ValidateLockFileInfo(Info, Reason)) { - ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); + ZEN_CONSOLE("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); return 1; } Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort); @@ -172,18 +172,18 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!Entry) { - ZEN_WARN("no zen server instance to add sponsor process to"); + ZEN_CONSOLE_ERROR("No zen server instance to add sponsor process to"); return 1; } // Sponsor processes are checked every second, so 2 second wait time should be enough if (!Entry->AddSponsorProcess(m_OwnerPid, 2000)) { - ZEN_WARN("unable to add sponsor process to running zen server instance"); + ZEN_CONSOLE_ERROR("Unable to add sponsor process to running zen server instance"); return 1; } - ZEN_CONSOLE("added sponsor process {} to running instance {} on port {}", m_OwnerPid, Entry->Pid.load(), m_Port); + ZEN_CONSOLE("Added sponsor process {} to running instance {} on port {}", m_OwnerPid, Entry->Pid.load(), m_Port); return 0; } @@ -224,14 +224,14 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (!IsFile(m_DataDir / ".lock")) { - ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir); + ZEN_CONSOLE_ERROR("Lock file does not exist in directory '{}'", m_DataDir); return 1; } LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); std::string Reason; if (!ValidateLockFileInfo(Info, Reason)) { - ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); + ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); return 1; } Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort); @@ -272,7 +272,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (Ec) { - ZEN_CONSOLE("requesting shutdown of server on port {} failed. Reason: '{}'", EntryPort, Ec.message()); + ZEN_CONSOLE_WARN("Requesting shutdown of server on port {} failed. Reason: '{}'", EntryPort, Ec.message()); } } catch (const std::exception& Ex) @@ -284,7 +284,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) // handle to determine when the server has shut down. Thus we signal that we would like // a shutdown via the shutdown flag and the check if the entry is still running. - ZEN_CONSOLE("requesting detached shutdown of server on port {}", EntryPort); + ZEN_CONSOLE("Requesting detached shutdown of server on port {}", EntryPort); Entry->SignalShutdownRequest(); Stopwatch Timer; @@ -294,12 +294,12 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Entry = Instance.Lookup(EntryPort); if (Entry == nullptr) { - ZEN_CONSOLE("shutdown complete"); + ZEN_CONSOLE("Shutdown complete"); return 0; } if (Entry->Pid.load() != ServerProcessPid) { - ZEN_CONSOLE("shutdown complete"); + ZEN_CONSOLE("Shutdown complete"); return 0; } Sleep(100); @@ -313,35 +313,36 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ProcessHandle RunningProcess; if (std::error_code Ec = FindProcess(ServerExePath, RunningProcess); !Ec, /*IncludeSelf*/ false) { - ZEN_WARN("attempting hard terminate of zen process with pid ({})", RunningProcess.Pid()); + ZEN_CONSOLE_WARN("Attempting hard terminate of zen process with pid ({})", RunningProcess.Pid()); try { if (RunningProcess.Terminate(0)) { - ZEN_CONSOLE("terminate complete"); + ZEN_CONSOLE("Terminate complete"); return 0; } - ZEN_CONSOLE("failed to terminate server, still running"); + ZEN_CONSOLE_ERROR("Failed to terminate server, still running"); return 1; } catch (const std::exception& Ex) { - ZEN_CONSOLE("failed to terminate server: '{}'", Ex.what()); + ZEN_CONSOLE_ERROR("Failed to terminate server: '{}'", Ex.what()); return 1; } } else { - ZEN_CONSOLE("Failed to find process '{}', reason: {}", ServerExePath.string(), Ec.message()); + ZEN_CONSOLE_WARN("Failed to find process '{}', reason: {}", ServerExePath.string(), Ec.message()); } } else if (Entry) { - ZEN_CONSOLE("failed to shutdown of server on port {}, use --force to hard terminate process", Entry->DesiredListenPort.load()); + ZEN_CONSOLE_ERROR("Failed to shutdown of server on port {}, use --force to hard terminate process", + Entry->DesiredListenPort.load()); return 1; } - ZEN_CONSOLE("no zen server to bring down"); + ZEN_CONSOLE("No zen server to bring down"); return 0; } diff --git a/src/zen/cmds/version_cmd.cpp b/src/zen/cmds/version_cmd.cpp index 7dfa125e4..1be02331d 100644 --- a/src/zen/cmds/version_cmd.cpp +++ b/src/zen/cmds/version_cmd.cpp @@ -72,7 +72,7 @@ VersionCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) HttpClient::Response Response = Client.Get(VersionRequest, {}, Parameters); if (!Response.IsSuccess()) { - ZEN_ERROR("{} failed: {}", VersionRequest, Response.ErrorMessage(""sv)); + ZEN_CONSOLE_ERROR("{} failed: {}", VersionRequest, Response.ErrorMessage(""sv)); return 1; } Version = Response.AsText(); diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp index fcc18df2b..5a9d0174e 100644 --- a/src/zen/cmds/wipe_cmd.cpp +++ b/src/zen/cmds/wipe_cmd.cpp @@ -479,7 +479,7 @@ namespace { { if (!Quiet) { - ZEN_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what()); + ZEN_CONSOLE_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what()); } CleanWipe = false; FailedDeleteCount++; @@ -587,18 +587,7 @@ WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - try - { - CleanDirectory(m_Directory, {}, !m_KeepReadOnlyFiles, m_Dryrun); - } - catch (std::exception& Ex) - { - if (!m_Quiet) - { - ZEN_ERROR("{}", Ex.what()); - } - return 3; - } + CleanDirectory(m_Directory, {}, !m_KeepReadOnlyFiles, m_Dryrun); return 0; } diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp index 773734f12..3930e56b7 100644 --- a/src/zen/cmds/workspaces_cmd.cpp +++ b/src/zen/cmds/workspaces_cmd.cpp @@ -185,7 +185,7 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) HttpClient Http(m_HostName); if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result) { - ZEN_CONSOLE("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); + ZEN_CONSOLE_ERROR("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); } } ZEN_CONSOLE("Added/updated workspace {}", m_Id); @@ -193,7 +193,7 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - ZEN_CONSOLE("Workspace {} already exists", m_Id); + ZEN_CONSOLE_WARN("Workspace {} already exists", m_Id); return 0; } } @@ -206,7 +206,7 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) static std::vector<Workspaces::WorkspaceConfiguration> Configs = Workspaces::ReadConfig(Log(), StatePath, Error); if (!Error.empty()) { - ZEN_CONSOLE("Failed to read workspaces state from '{}'. Reason: '{}'", StatePath, Error); + ZEN_CONSOLE_ERROR("Failed to read workspaces state from '{}'. Reason: '{}'", StatePath, Error); } else { @@ -233,7 +233,7 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - ZEN_CONSOLE("Workspace {} not found", m_Id); + ZEN_CONSOLE_WARN("Workspace {} not found", m_Id); } } } @@ -257,14 +257,14 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) HttpClient Http(m_HostName); if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result) { - ZEN_CONSOLE("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); + ZEN_CONSOLE_ERROR("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); } } ZEN_CONSOLE("Removed workspace {}", m_Id); } else { - ZEN_CONSOLE("Workspace {} does not exist", m_Id); + ZEN_CONSOLE_WARN("Workspace {} does not exist", m_Id); } return 0; } @@ -428,7 +428,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** Workspaces::WorkspaceConfiguration WorkspaceConfig = Workspaces::FindWorkspace(Log(), StatePath, WorkspaceId); if (WorkspaceConfig.Id == Oid::Zero) { - ZEN_CONSOLE("Workspace {} does not exist", m_WorkspaceId); + ZEN_CONSOLE_ERROR("Workspace {} does not exist", m_WorkspaceId); return 0; } m_WorkspaceRoot = WorkspaceConfig.RootPath; @@ -481,7 +481,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** HttpClient Http(m_HostName); if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result) { - ZEN_CONSOLE("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); + ZEN_CONSOLE_ERROR("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); } } ZEN_CONSOLE("Created workspace share {}", m_ShareId); @@ -602,7 +602,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** HttpClient Http(m_HostName); if (HttpClient::Response Result = Http.Get("/ws/refresh"); !Result) { - ZEN_CONSOLE("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); + ZEN_CONSOLE_ERROR("Failed to refresh workspaces for host {}. Reason: '{}'", m_HostName, Result.ErrorMessage(""sv)); } } ZEN_CONSOLE("Removed workspace share {}", m_ShareId); diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index accd0d105..240ba2a81 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -1030,10 +1030,35 @@ main(int argc, char** argv) } catch (const OptionParseException& Ex) { - std::string help = VerbOptions.help(); - - printf("Error parsing arguments for command '%s': %s\n\n%s", SubCommand.c_str(), Ex.what(), help.c_str()); - + ZEN_CONSOLE_ERROR("Invalid arguments for command '{}': {}\n\n{}", SubCommand, Ex.what(), VerbOptions.help()); + exit(11); + } + catch (const std::system_error& Ex) + { + if (IsOOD(Ex)) + { + ZEN_CONSOLE_ERROR("Operation failed due to out of disk space: {}", Ex.what()); + exit(3); + } + else if (IsOOM(Ex)) + { + ZEN_CONSOLE_ERROR("Operation failed due to out of memory: {}", Ex.what()); + exit(3); + } + else + { + ZEN_CONSOLE_ERROR("Operation failed due to system error: {} ({})\n", Ex.what(), Ex.code() ? Ex.code().value() : 0); + exit(Ex.code() ? Ex.code().value() : 10); + } + } + catch (const HttpClientError& Ex) + { + ZEN_CONSOLE_ERROR("Operation failed due to a http error: {}", Ex.what()); + exit(Ex.m_Error != 0 ? Ex.m_Error : (int)Ex.m_ResponseCode); + } + catch (const std::exception& Ex) + { + ZEN_CONSOLE_ERROR("Operation failed due to: {}\n", Ex.what()); exit(11); } } @@ -1051,14 +1076,30 @@ main(int argc, char** argv) } catch (const std::system_error& Ex) { - printf("System Error: %s\n", Ex.what()); - - return Ex.code() ? Ex.code().value() : 10; + if (IsOOD(Ex)) + { + printf("Operation failed due to out of disk space: %s", Ex.what()); + return 3; + } + else if (IsOOM(Ex)) + { + printf("Operation failed due to out of memory: %s", Ex.what()); + return 3; + } + else + { + printf("Operation failed due to system error: %s (%d)\n", Ex.what(), Ex.code() ? Ex.code().value() : 0); + return Ex.code() ? Ex.code().value() : 10; + } + } + catch (const HttpClientError& Ex) + { + printf("Operation failed due to a http error: %s", Ex.what()); + return Ex.m_Error != 0 ? Ex.m_Error : (int)Ex.m_ResponseCode; } catch (const std::exception& Ex) { - printf("Error: %s\n", Ex.what()); - + printf("Operation failed due to: %s\n", Ex.what()); return 11; } diff --git a/src/zencore/include/zencore/logging.h b/src/zencore/include/zencore/logging.h index bffbe84b4..afbbbd3ee 100644 --- a/src/zencore/include/zencore/logging.h +++ b/src/zencore/include/zencore/logging.h @@ -59,7 +59,6 @@ struct LogCategory zen::LoggerRef LoggerRef; }; -void EmitConsoleLogMessage(int LogLevel, std::string_view Message); void EmitConsoleLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args); void EmitLogMessage(LoggerRef& Logger, int LogLevel, std::string_view Message); void EmitLogMessage(LoggerRef& Logger, const SourceLocation& Location, int LogLevel, std::string_view Message); diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp index 5885587ad..685c79d82 100644 --- a/src/zencore/logging.cpp +++ b/src/zencore/logging.cpp @@ -122,11 +122,34 @@ EmitConsoleLogMessage(int LogLevel, const std::string_view Message) ConsoleLog().SpdLogger->log(InLevel, Message); } +#define ZEN_COLOR_YELLOW "\033[0;33m" +#define ZEN_COLOR_RED "\033[0;31m" +#define ZEN_BRIGHT_COLOR_RED "\033[1;31m" +#define ZEN_COLOR_RESET "\033[0m" + void EmitConsoleLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) { ZEN_MEMSCOPE(ELLMTag::Logging); zen::logging::LoggingContext LogCtx; + + // We are not using a format option for console which include log level since it would interfere with normal console output + + const spdlog::level::level_enum InLevel = (spdlog::level::level_enum)LogLevel; + switch (InLevel) + { + case spdlog::level::level_enum::warn: + fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_COLOR_YELLOW "Warning: " ZEN_COLOR_RESET); + break; + case spdlog::level::level_enum::err: + fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_BRIGHT_COLOR_RED "Error: " ZEN_COLOR_RESET); + break; + case spdlog::level::level_enum::critical: + fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_COLOR_RED "Critical: " ZEN_COLOR_RESET); + break; + default: + break; + } fmt::vformat_to(fmt::appender(LogCtx.MessageBuffer), Format, Args); zen::logging::EmitConsoleLogMessage(LogLevel, LogCtx.Message()); } diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp index 19ac3a216..68f1c602e 100644 --- a/src/zenserver/cache/httpstructuredcache.cpp +++ b/src/zenserver/cache/httpstructuredcache.cpp @@ -85,7 +85,7 @@ namespace { ////////////////////////////////////////////////////////////////////////// HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, - GetCidStoreFunc&& GetCidStore, + CidStore& InCidStore, HttpStatsService& StatsService, HttpStatusService& StatusService, UpstreamCache& UpstreamCache, @@ -95,10 +95,11 @@ HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCach , m_CacheStore(InCacheStore) , m_StatsService(StatsService) , m_StatusService(StatusService) +, m_CidStore(InCidStore) , m_UpstreamCache(UpstreamCache) , m_DiskWriteBlocker(InDiskWriteBlocker) , m_OpenProcessCache(InOpenProcessCache) -, m_RpcHandler(m_Log, m_CacheStats, UpstreamCache, InCacheStore, std::move(GetCidStore), InDiskWriteBlocker) +, m_RpcHandler(m_Log, m_CacheStats, UpstreamCache, InCacheStore, InCidStore, InDiskWriteBlocker) { m_StatsService.RegisterHandler("z$", *this); m_StatusService.RegisterHandler("z$", *this); @@ -224,9 +225,6 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) for (const auto& NamespaceIt : ValueDetails.Namespaces) { const std::string& Namespace = NamespaceIt.first; - - CidStore& ChunkStore = m_RpcHandler.GetCidStore(Namespace); - for (const auto& BucketIt : NamespaceIt.second.Buckets) { const std::string& Bucket = BucketIt.first; @@ -236,7 +234,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) { for (const IoHash& Hash : ValueIt.second.Attachments) { - IoBuffer Payload = ChunkStore.FindChunkByCid(Hash); + IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); CSVWriter << "\r\n" << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString() << ", " << Hash.ToHexString() << ", " << gsl::narrow<uint64_t>(Payload.GetSize()); @@ -254,7 +252,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) size_t AttachmentsSize = 0; for (const IoHash& Hash : ValueIt.second.Attachments) { - IoBuffer Payload = ChunkStore.FindChunkByCid(Hash); + IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); AttachmentsSize += Payload.GetSize(); } CSVWriter << ", " << gsl::narrow<uint64_t>(AttachmentsSize); @@ -276,9 +274,6 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) for (const auto& NamespaceIt : ValueDetails.Namespaces) { const std::string& Namespace = NamespaceIt.first; - - CidStore& ChunkStore = m_RpcHandler.GetCidStore(Namespace); - Cbo.BeginObject(); { Cbo.AddString("name", Namespace); @@ -321,7 +316,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) { Cbo.BeginObject(); Cbo.AddHash("cid", Hash); - IoBuffer Payload = ChunkStore.FindChunkByCid(Hash); + IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); Cbo.AddInteger("size", gsl::narrow<uint64_t>(Payload.GetSize())); Cbo.EndObject(); } @@ -335,7 +330,7 @@ HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) size_t AttachmentsSize = 0; for (const IoHash& Hash : ValueIt.second.Attachments) { - IoBuffer Payload = ChunkStore.FindChunkByCid(Hash); + IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); AttachmentsSize += Payload.GetSize(); } Cbo.AddInteger("attachmentssize", gsl::narrow<uint64_t>(AttachmentsSize)); @@ -610,8 +605,6 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque ResponseWriter.AddInteger("EntryCount", Info->DiskLayerInfo.EntryCount); - CidStore& ChunkStore = m_RpcHandler.GetCidStore(NamespaceName); - if (auto Buckets = HttpServerRequest::Decode(Request.GetQueryParams().GetValue("bucketsizes")); !Buckets.empty()) { ResponseWriter.BeginObject("BucketSizes"); @@ -670,7 +663,7 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque uint64_t AttachmentsSize = 0; - ChunkStore.IterateChunks( + m_CidStore.IterateChunks( AllAttachments, [&](size_t Index, const IoBuffer& Payload) { ZEN_UNUSED(Index); @@ -738,8 +731,6 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, ResponseWriter.AddInteger("DiskEntryCount", Info->DiskLayerInfo.EntryCount); - CidStore& ChunkStore = m_RpcHandler.GetCidStore(NamespaceName); - if (auto GetBucketSize = Request.GetQueryParams().GetValue("bucketsize"); GetBucketSize == "true") { CacheContentStats ContentStats; @@ -766,7 +757,7 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background); - ChunkStore.IterateChunks( + m_CidStore.IterateChunks( ContentStats.Attachments, [&](size_t Index, const IoBuffer& Payload) { ZEN_UNUSED(Index); @@ -841,8 +832,6 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()}; Stopwatch Timer; - CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace); - if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal) && m_CacheStore.Get(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue)) { @@ -857,17 +846,17 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con uint32_t MissingCount = 0; CbObjectView CacheRecord(ClientResultValue.Value.Data()); - CacheRecord.IterateAttachments([this, &ChunkStore, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) { + CacheRecord.IterateAttachments([this, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) { if (SkipData) { - if (!ChunkStore.ContainsChunk(AttachmentHash.AsHash())) + if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash())) { MissingCount++; } } else { - if (IoBuffer Chunk = ChunkStore.FindChunkByCid(AttachmentHash.AsHash())) + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) { CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)); if (Compressed) @@ -967,8 +956,6 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con { Success = true; - CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace); - ClientResultValue.Value = UpstreamResult.Value; ClientResultValue.Value.SetContentType(AcceptType); @@ -1019,7 +1006,6 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con CacheRecord.IterateAttachments([this, &Package, &Ref, - &ChunkStore, &WriteAttachmentBuffers, &WriteRawHashes, &ReferencedAttachments, @@ -1054,12 +1040,12 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con { if (SkipData) { - if (ChunkStore.ContainsChunk(Hash)) + if (m_CidStore.ContainsChunk(Hash)) { Count.Valid++; } } - else if (IoBuffer Chunk = ChunkStore.FindChunkByCid(Hash)) + else if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Hash)) { CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)); if (Compressed) @@ -1101,7 +1087,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con if (!WriteAttachmentBuffers.empty()) { std::vector<CidStore::InsertResult> InsertResults = - ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); for (const CidStore::InsertResult& Result : InsertResults) { if (Result.New) @@ -1268,10 +1254,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con if (HasUpstream && EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote)) { - CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace); - m_UpstreamCache.EnqueueUpstream( - {.Type = ContentType, .Namespace = Ref.Namespace, .Key = {Ref.BucketSegment, Ref.HashKey}}, - [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); }); + m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Namespace = Ref.Namespace, .Key = {Ref.BucketSegment, Ref.HashKey}}); } ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' in {}", @@ -1305,18 +1288,15 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con std::vector<IoHash> ReferencedAttachments; int32_t TotalCount = 0; - CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace); - - CacheRecord.IterateAttachments( - [this, &ChunkStore, &TotalCount, &ValidAttachments, &ReferencedAttachments](CbFieldView AttachmentHash) { - const IoHash Hash = AttachmentHash.AsHash(); - ReferencedAttachments.push_back(Hash); - if (ChunkStore.ContainsChunk(Hash)) - { - ValidAttachments.emplace_back(Hash); - } - TotalCount++; - }); + CacheRecord.IterateAttachments([this, &TotalCount, &ValidAttachments, &ReferencedAttachments](CbFieldView AttachmentHash) { + const IoHash Hash = AttachmentHash.AsHash(); + ReferencedAttachments.push_back(Hash); + if (m_CidStore.ContainsChunk(Hash)) + { + ValidAttachments.emplace_back(Hash); + } + TotalCount++; + }); const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); @@ -1350,12 +1330,10 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con CachePolicy Policy = PolicyFromUrl; if (HasUpstream && EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) { - m_UpstreamCache.EnqueueUpstream( - {.Type = ZenContentType::kCbObject, - .Namespace = Ref.Namespace, - .Key = {Ref.BucketSegment, Ref.HashKey}, - .ValueContentIds = std::move(ValidAttachments)}, - [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); }); + m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbObject, + .Namespace = Ref.Namespace, + .Key = {Ref.BucketSegment, Ref.HashKey}, + .ValueContentIds = std::move(ValidAttachments)}); } Request.WriteResponse(HttpResponseCode::Created); @@ -1388,46 +1366,38 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con WriteAttachmentBuffers.reserve(NumAttachments); WriteRawHashes.reserve(NumAttachments); - CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace); - - CacheRecord.IterateAttachments([this, - &Ref, - &Package, - &ChunkStore, - &WriteAttachmentBuffers, - &WriteRawHashes, - &ValidAttachments, - &ReferencedAttachments, - &Count](CbFieldView HashView) { - const IoHash Hash = HashView.AsHash(); - ReferencedAttachments.push_back(Hash); - if (const CbAttachment* Attachment = Package.FindAttachment(Hash)) - { - if (Attachment->IsCompressedBinary()) + CacheRecord.IterateAttachments( + [this, &Ref, &Package, &WriteAttachmentBuffers, &WriteRawHashes, &ValidAttachments, &ReferencedAttachments, &Count]( + CbFieldView HashView) { + const IoHash Hash = HashView.AsHash(); + ReferencedAttachments.push_back(Hash); + if (const CbAttachment* Attachment = Package.FindAttachment(Hash)) { - WriteAttachmentBuffers.emplace_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); - WriteRawHashes.push_back(Hash); - ValidAttachments.emplace_back(Hash); - Count.Valid++; + if (Attachment->IsCompressedBinary()) + { + WriteAttachmentBuffers.emplace_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); + WriteRawHashes.push_back(Hash); + ValidAttachments.emplace_back(Hash); + Count.Valid++; + } + else + { + ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(HttpContentType::kCbPackage), + Hash); + Count.Invalid++; + } } - else + else if (m_CidStore.ContainsChunk(Hash)) { - ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed", - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - ToString(HttpContentType::kCbPackage), - Hash); - Count.Invalid++; + ValidAttachments.emplace_back(Hash); + Count.Valid++; } - } - else if (ChunkStore.ContainsChunk(Hash)) - { - ValidAttachments.emplace_back(Hash); - Count.Valid++; - } - Count.Total++; - }); + Count.Total++; + }); if (Count.Invalid > 0) { @@ -1451,7 +1421,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con if (!WriteAttachmentBuffers.empty()) { - std::vector<CidStore::InsertResult> InsertResults = ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); for (const CidStore::InsertResult& InsertResult : InsertResults) { if (InsertResult.New) @@ -1478,12 +1448,10 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con if (HasUpstream && EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) { - m_UpstreamCache.EnqueueUpstream( - {.Type = ZenContentType::kCbPackage, - .Namespace = Ref.Namespace, - .Key = {Ref.BucketSegment, Ref.HashKey}, - .ValueContentIds = std::move(ValidAttachments)}, - [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); }); + m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage, + .Namespace = Ref.Namespace, + .Key = {Ref.BucketSegment, Ref.HashKey}, + .ValueContentIds = std::move(ValidAttachments)}); } Request.WriteResponse(HttpResponseCode::Created); @@ -1517,9 +1485,7 @@ HttpStructuredCacheService::HandleGetCacheChunk(HttpServerRequest& Request, cons { Stopwatch Timer; - CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace); - - IoBuffer Value = ChunkStore.FindChunkByCid(Ref.ValueContentId); + IoBuffer Value = m_CidStore.FindChunkByCid(Ref.ValueContentId); const UpstreamEndpointInfo* Source = nullptr; CachePolicy Policy = PolicyFromUrl; @@ -1541,7 +1507,7 @@ HttpStructuredCacheService::HandleGetCacheChunk(HttpServerRequest& Request, cons { if (AreDiskWritesAllowed()) { - ChunkStore.AddChunk(UpstreamResult.Value, RawHash); + m_CidStore.AddChunk(UpstreamResult.Value, RawHash); } Source = UpstreamResult.Source; } @@ -1635,9 +1601,7 @@ HttpStructuredCacheService::HandlePutCacheChunk(HttpServerRequest& Request, cons "ValueContentId does not match attachment hash"sv); } - CidStore& ChunkStore = m_RpcHandler.GetCidStore(Ref.Namespace); - - CidStore::InsertResult Result = ChunkStore.AddChunk(Body, RawHash); + CidStore::InsertResult Result = m_CidStore.AddChunk(Body, RawHash); ZEN_DEBUG("PUTCACHECHUNK - '{}/{}/{}/{}' {} '{}' ({}) in {}", Ref.Namespace, @@ -1852,51 +1816,16 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) EmitSnapshot("requests", m_HttpRequests, Cbo); - const uint64_t HitCount = m_CacheStats.HitCount; - const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount; - const uint64_t MissCount = m_CacheStats.MissCount; - const uint64_t WriteCount = m_CacheStats.WriteCount; - const uint64_t BadRequestCount = m_CacheStats.BadRequestCount; - - uint64_t TotalChunkHitCount = 0; - uint64_t TotalChunkMissCount = 0; - uint64_t TotalChunkWriteCount = 0; - CidStoreSize TotalCidSize; - - tsl::robin_map<CidStore*, std::string> UniqueStores; - { - std::vector<std::string> NamespaceNames = m_CacheStore.GetNamespaces(); - - for (const std::string& NamespaceName : NamespaceNames) - { - CidStore* Store = &m_RpcHandler.GetCidStore(NamespaceName); - if (auto It = UniqueStores.find(Store); It == UniqueStores.end()) - { - UniqueStores.insert_or_assign(Store, NamespaceName); - } - else - { - UniqueStores.insert_or_assign(Store, std::string{}); - } - } - - for (auto It : UniqueStores) - { - CidStore* ChunkStore = It.first; - - CidStoreStats StoreStats = ChunkStore->Stats(); - CidStoreSize StoreSize = ChunkStore->TotalSize(); - - TotalChunkHitCount += StoreStats.HitCount; - TotalChunkMissCount += StoreStats.MissCount; - TotalChunkWriteCount += StoreStats.WriteCount; - - TotalCidSize.TinySize += StoreSize.TinySize; - TotalCidSize.SmallSize += StoreSize.SmallSize; - TotalCidSize.LargeSize += StoreSize.LargeSize; - TotalCidSize.TotalSize += StoreSize.TotalSize; - } - } + const uint64_t HitCount = m_CacheStats.HitCount; + const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount; + const uint64_t MissCount = m_CacheStats.MissCount; + const uint64_t WriteCount = m_CacheStats.WriteCount; + const uint64_t BadRequestCount = m_CacheStats.BadRequestCount; + struct CidStoreStats StoreStats = m_CidStore.Stats(); + const uint64_t ChunkHitCount = StoreStats.HitCount; + const uint64_t ChunkMissCount = StoreStats.MissCount; + const uint64_t ChunkWriteCount = StoreStats.WriteCount; + const uint64_t TotalCount = HitCount + MissCount; const uint64_t RpcRequests = m_CacheStats.RpcRequests; const uint64_t RpcRecordRequests = m_CacheStats.RpcRecordRequests; @@ -1906,11 +1835,17 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) const uint64_t RpcChunkRequests = m_CacheStats.RpcChunkRequests; const uint64_t RpcChunkBatchRequests = m_CacheStats.RpcChunkBatchRequests; + const CidStoreSize CidSize = m_CidStore.TotalSize(); const CacheStoreSize CacheSize = m_CacheStore.TotalSize(); bool ShowCidStoreStats = Request.GetQueryParams().GetValue("cidstorestats") == "true"; bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true"; + CidStoreStats CidStoreStats = {}; + if (ShowCidStoreStats) + { + CidStoreStats = m_CidStore.Stats(); + } ZenCacheStore::CacheStoreStats CacheStoreStats = {}; if (ShowCacheStoreStats) { @@ -1945,7 +1880,6 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) Cbo.EndObject(); Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount; - const uint64_t TotalCount = HitCount + MissCount; Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0); if (m_UpstreamCache.IsActive()) @@ -1956,9 +1890,7 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); } - Cbo << "cidhits" << TotalChunkHitCount << "cidmisses" << TotalChunkMissCount << "cidwrites" << TotalChunkWriteCount; - const uint64_t TotalChunkCount = TotalChunkHitCount + TotalChunkMissCount; - Cbo << "cidhit_ratio" << (TotalChunkHitCount ? (double(TotalChunkCount) / double(TotalChunkHitCount)) : 0.0); + Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount; if (ShowCacheStoreStats) { @@ -2067,58 +1999,20 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) { Cbo.BeginObject("size"); { - Cbo << "tiny" << TotalCidSize.TinySize; - Cbo << "small" << TotalCidSize.SmallSize; - Cbo << "large" << TotalCidSize.LargeSize; - Cbo << "total" << TotalCidSize.TotalSize; + Cbo << "tiny" << CidSize.TinySize; + Cbo << "small" << CidSize.SmallSize; + Cbo << "large" << CidSize.LargeSize; + Cbo << "total" << CidSize.TotalSize; } Cbo.EndObject(); if (ShowCidStoreStats) { Cbo.BeginObject("store"); - - auto OutputStats = [&](CidStore& ChunkStore) { - CidStoreStats StoreStats = ChunkStore.Stats(); - Cbo << "hits" << StoreStats.HitCount << "misses" << StoreStats.MissCount << "writes" << StoreStats.WriteCount; - const uint64_t Count = StoreStats.HitCount + StoreStats.MissCount; - Cbo << "hit_ratio" << (Count ? (double(StoreStats.HitCount) / double(Count)) : 0.0); - EmitSnapshot("read", StoreStats.FindChunkOps, Cbo); - EmitSnapshot("write", StoreStats.AddChunkOps, Cbo); - }; - - if (UniqueStores.size() > 1) - { - Cbo.BeginArray("namespaces"); - for (auto It : UniqueStores) - { - CidStore* ChunkStore = It.first; - const std::string& Namespace = It.second; - CidStoreSize ChunkStoreSize = ChunkStore->TotalSize(); - Cbo.BeginObject(); - { - Cbo << "namespace" << Namespace; - Cbo.BeginObject("stats"); - OutputStats(*ChunkStore); - Cbo.EndObject(); - - Cbo.BeginObject("size"); - { - Cbo << "tiny" << ChunkStoreSize.TinySize; - Cbo << "small" << ChunkStoreSize.SmallSize; - Cbo << "large" << ChunkStoreSize.LargeSize; - Cbo << "total" << ChunkStoreSize.TotalSize; - } - Cbo.EndObject(); - } - Cbo.EndObject(); - } - Cbo.EndArray(); // namespaces - } - else if (UniqueStores.size() != 0) - { - OutputStats(*UniqueStores.begin()->first); - } + Cbo << "hits" << CidStoreStats.HitCount << "misses" << CidStoreStats.MissCount << "writes" << CidStoreStats.WriteCount; + EmitSnapshot("read", CidStoreStats.FindChunkOps, Cbo); + EmitSnapshot("write", CidStoreStats.AddChunkOps, Cbo); + // EmitSnapshot("exists", CidStoreStats.ContainChunkOps, Cbo); Cbo.EndObject(); } } diff --git a/src/zenserver/cache/httpstructuredcache.h b/src/zenserver/cache/httpstructuredcache.h index d46ca145d..cb822f117 100644 --- a/src/zenserver/cache/httpstructuredcache.h +++ b/src/zenserver/cache/httpstructuredcache.h @@ -70,10 +70,8 @@ namespace cache { class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider, public IHttpStatusProvider { public: - typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc; - HttpStructuredCacheService(ZenCacheStore& InCacheStore, - GetCidStoreFunc&& GetCidStore, + CidStore& InCidStore, HttpStatsService& StatsService, HttpStatusService& StatusService, UpstreamCache& UpstreamCache, @@ -117,6 +115,7 @@ private: ZenCacheStore& m_CacheStore; HttpStatsService& m_StatsService; HttpStatusService& m_StatusService; + CidStore& m_CidStore; UpstreamCache& m_UpstreamCache; metrics::OperationTiming m_HttpRequests; metrics::OperationTiming m_UpstreamGetRequestTiming; diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 4049ae815..fb2d9b7f4 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -482,7 +482,8 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("server.pluginsconfigfile"sv, ServerOptions.PluginsConfigFile, "plugins-config"sv); LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv); LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv); - LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "quiet"sv); + LuaOptions.AddOption("server.quiet"sv, ServerOptions.QuietConsole, "quiet"sv); + LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "noconsole"sv); ////// objectstore LuaOptions.AddOption("server.objectstore.enabled"sv, ServerOptions.ObjectStoreEnabled, "objectstore-enabled"sv); @@ -812,16 +813,17 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) // clang-format off options.add_options("logging") - ("abslog", "Path to log file", cxxopts::value<std::string>(AbsLogFile)) - ("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId)) - ("quiet", "Disable console logging", cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false")) - ("log-trace", "Change selected loggers to level TRACE", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace])) - ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug])) - ("log-info", "Change selected loggers to level INFO", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info])) - ("log-warn", "Change selected loggers to level WARN", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn])) - ("log-error", "Change selected loggers to level ERROR", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err])) - ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical])) - ("log-off", "Change selected loggers to level OFF", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off])) + ("abslog", "Path to log file", cxxopts::value<std::string>(AbsLogFile)) + ("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId)) + ("quiet", "Configure console logger output to level WARN", cxxopts::value<bool>(ServerOptions.QuietConsole)->default_value("false")) + ("noconsole", "Disable console logging", cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false")) + ("log-trace", "Change selected loggers to level TRACE", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace])) + ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug])) + ("log-info", "Change selected loggers to level INFO", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info])) + ("log-warn", "Change selected loggers to level WARN", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn])) + ("log-error", "Change selected loggers to level ERROR", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err])) + ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical])) + ("log-off", "Change selected loggers to level OFF", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off])) ; // clang-format on @@ -1356,6 +1358,29 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) ZEN_TRACE_CPU("ConfigParse"); + if (ServerOptions.QuietConsole) + { + bool HasExplicitConsoleLevel = false; + for (int i = 0; i < logging::level::LogLevelCount; ++i) + { + if (ServerOptions.Loggers[i].find("console") != std::string::npos) + { + HasExplicitConsoleLevel = true; + break; + } + } + + if (!HasExplicitConsoleLevel) + { + std::string& WarnLoggers = ServerOptions.Loggers[logging::level::Warn]; + if (!WarnLoggers.empty()) + { + WarnLoggers += ","; + } + WarnLoggers += "console"; + } + } + for (int i = 0; i < logging::level::LogLevelCount; ++i) { logging::ConfigureLogLevels(logging::level::LogLevel(i), ServerOptions.Loggers[i]); @@ -1413,7 +1438,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) } catch (const zen::OptionParseException& e) { - ZEN_CONSOLE_ERROR("Error parsing zenserver arguments: {}\n\n{}", e.what(), options.help()); + ZEN_CONSOLE_ERROR("Invalid zenserver arguments: {}\n\n{}", e.what(), options.help()); throw; } diff --git a/src/zenserver/config.h b/src/zenserver/config.h index 9e8787957..8380e72e7 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -206,6 +206,7 @@ struct ZenServerOptions bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux) bool ObjectStoreEnabled = false; bool NoConsoleOutput = false; // Control default use of stdout for diagnostics + bool QuietConsole = false; // Configure console logger output to level WARN std::string Loggers[zen::logging::level::LogLevelCount]; std::string ScrubOptions; #if ZEN_WITH_TRACE diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp index f3d3377b0..34d9b05b7 100644 --- a/src/zenserver/diag/logging.cpp +++ b/src/zenserver/diag/logging.cpp @@ -27,6 +27,7 @@ InitializeServerLogging(const ZenServerOptions& InOptions) .IsVerbose = false, .IsTest = InOptions.IsTest, .NoConsoleOutput = InOptions.NoConsoleOutput, + .QuietConsole = InOptions.QuietConsole, .AbsLogFile = InOptions.AbsLogFile, .LogId = InOptions.LogId}; diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index d8922f885..d512d4894 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -139,21 +139,23 @@ ZenEntryPoint::Run() { if (Ec) { - ZEN_WARN( - "Sponsor owner pid {} can not be checked for running state, reason: '{}'. Will not add sponsor to process " - "listening to port {} (pid: {})", - m_ServerOptions.OwnerPid, - Ec.message(), - m_ServerOptions.BasePort, - Entry->Pid.load()); + ZEN_WARN(ZEN_APP_NAME + " exiting, sponsor owner pid {} can not be checked for running state, reason: '{}'. Will not add sponsor " + "to process " + "listening to port {} (pid: {})", + m_ServerOptions.OwnerPid, + Ec.message(), + m_ServerOptions.BasePort, + Entry->Pid.load()); } else { - ZEN_WARN( - "Sponsor owner pid {} is no longer running, will not add sponsor to process listening to port {} (pid: {})", - m_ServerOptions.OwnerPid, - m_ServerOptions.BasePort, - Entry->Pid.load()); + ZEN_WARN(ZEN_APP_NAME + " exiting, sponsor owner pid {} is no longer running, will not add sponsor to process listening to port " + "{} (pid: {})", + m_ServerOptions.OwnerPid, + m_ServerOptions.BasePort, + Entry->Pid.load()); } std::exit(1); } @@ -175,7 +177,7 @@ ZenEntryPoint::Run() } else { - ZEN_WARN("Failed to add sponsor owner pid {} to process listening to port {} (pid: {})", + ZEN_WARN(ZEN_APP_NAME " exiting, failed to add sponsor owner pid {} to process listening to port {} (pid: {})", m_ServerOptions.OwnerPid, m_ServerOptions.BasePort, Entry->Pid.load()); @@ -184,7 +186,7 @@ ZenEntryPoint::Run() } else { - ZEN_WARN("Exiting since there is already a process listening to port {} (pid: {})", + ZEN_WARN(ZEN_APP_NAME " exiting, there is already a process listening to port {} (pid: {})", m_ServerOptions.BasePort, Entry->Pid.load()); std::exit(1); @@ -208,14 +210,19 @@ ZenEntryPoint::Run() if (Ec) { - ZEN_WARN("Unable to grab lock at '{}' (reason: '{}'), retrying", LockFilePath, Ec.message()); - Sleep(500); + ZEN_INFO(ZEN_APP_NAME " unable to grab lock at '{}' (reason: '{}'), retrying", LockFilePath, Ec.message()); + Sleep(100); m_LockFile.Create(LockFilePath, MakeLockData(false), Ec); if (Ec) { - ZEN_WARN("ERROR: Unable to grab lock at '{}' (reason: '{}')", LockFilePath, Ec.message()); - std::exit(99); + ZEN_INFO(ZEN_APP_NAME " unable to grab lock at '{}' (reason: '{}'), retrying", LockFilePath, Ec.message()); + Sleep(500); + if (Ec) + { + ZEN_WARN(ZEN_APP_NAME " exiting, unable to grab lock at '{}' (reason: '{}')", LockFilePath, Ec.message()); + std::exit(99); + } } } @@ -316,12 +323,14 @@ ZenEntryPoint::Run() } catch (const AssertException& AssertEx) { - ZEN_CRITICAL("Caught assert exception in main for process {}: {}", zen::GetCurrentProcessId(), AssertEx.FullDescription()); + ZEN_CRITICAL(ZEN_APP_NAME " caught assert exception in main for process {}: {}", + zen::GetCurrentProcessId(), + AssertEx.FullDescription()); RequestApplicationExit(1); } catch (const std::system_error& e) { - ZEN_CRITICAL("Caught system error exception in main for process {}: {} ({})", + ZEN_CRITICAL(ZEN_APP_NAME " caught system error exception in main for process {}: {} ({})", zen::GetCurrentProcessId(), e.what(), e.code().value()); @@ -329,7 +338,7 @@ ZenEntryPoint::Run() } catch (const std::exception& e) { - ZEN_CRITICAL("Caught exception in main for process {}: {}", zen::GetCurrentProcessId(), e.what()); + ZEN_CRITICAL(ZEN_APP_NAME " caught exception in main for process {}: {}", zen::GetCurrentProcessId(), e.what()); RequestApplicationExit(1); } @@ -450,7 +459,7 @@ main(int argc, char* argv[]) { if (IsDir(ServerOptions.DataDir)) { - ZEN_CONSOLE_INFO("deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason); + ZEN_CONSOLE_INFO("Deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason); DeleteDirectories(ServerOptions.DataDir); } } @@ -463,7 +472,7 @@ main(int argc, char* argv[]) if (!ServerOptions.BaseSnapshotDir.empty()) { - ZEN_CONSOLE_INFO("copying snapshot from '{}' into '{}", ServerOptions.BaseSnapshotDir, ServerOptions.DataDir); + ZEN_CONSOLE_INFO("Copying snapshot from '{}' into '{}", ServerOptions.BaseSnapshotDir, ServerOptions.DataDir); CopyTree(ServerOptions.BaseSnapshotDir, ServerOptions.DataDir, {.EnableClone = true}); } @@ -499,17 +508,17 @@ main(int argc, char* argv[]) catch (const OptionParseException& ParseEx) { // The parsing error already outputs all the details so no need to output the command line here - fprintf(stderr, "ERROR: %s\n", ParseEx.what()); + fprintf(stderr, ZEN_APP_NAME " ERROR: %s\n", ParseEx.what()); return 1; } catch (const AssertException& AssertEx) { - fprintf(stderr, "ERROR: Caught assert exception in main: '%s'", AssertEx.FullDescription().c_str()); + fprintf(stderr, ZEN_APP_NAME " ERROR: Caught assert exception in main: '%s'", AssertEx.FullDescription().c_str()); return 1; } catch (const std::exception& Ex) { - fprintf(stderr, "ERROR: Caught exception in main: '%s'", Ex.what()); + fprintf(stderr, ZEN_APP_NAME " ERROR: Caught exception in main: '%s'", Ex.what()); return 1; } diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 9600133f3..317a419eb 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -235,11 +235,13 @@ namespace { ////////////////////////////////////////////////////////////////////////// -HttpProjectService::HttpProjectService(ProjectStore* Projects, +HttpProjectService::HttpProjectService(CidStore& Store, + ProjectStore* Projects, HttpStatusService& StatusService, HttpStatsService& StatsService, AuthMgr& AuthMgr) : m_Log(logging::Get("project")) +, m_CidStore(Store) , m_ProjectStore(Projects) , m_StatusService(StatusService) , m_StatsService(StatsService) @@ -405,45 +407,8 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) { ZEN_TRACE_CPU("ProjectService::Stats"); - bool ShowCidStoreStats = HttpReq.GetQueryParams().GetValue("cidstorestats") == "true"; - - const GcStorageSize StoreSize = m_ProjectStore->StorageSize(); - uint64_t TotalChunkHitCount = 0; - uint64_t TotalChunkMissCount = 0; - uint64_t TotalChunkWriteCount = 0; - CidStoreSize TotalCidSize; - - tsl::robin_map<CidStore*, std::string> UniqueStores; - { - m_ProjectStore->IterateProjects([&UniqueStores](ProjectStore::Project& Project) { - CidStore* Store = &Project.GetCidStore(); - if (auto It = UniqueStores.find(Store); It == UniqueStores.end()) - { - UniqueStores.insert_or_assign(Store, Project.Identifier); - } - else - { - UniqueStores.insert_or_assign(Store, std::string{}); - } - }); - - for (auto It : UniqueStores) - { - CidStore* ChunkStore = It.first; - - CidStoreStats ChunkStoreStats = ChunkStore->Stats(); - CidStoreSize ChunkStoreSize = ChunkStore->TotalSize(); - - TotalChunkHitCount += ChunkStoreStats.HitCount; - TotalChunkMissCount += ChunkStoreStats.MissCount; - TotalChunkWriteCount += ChunkStoreStats.WriteCount; - - TotalCidSize.TinySize += ChunkStoreSize.TinySize; - TotalCidSize.SmallSize += ChunkStoreSize.SmallSize; - TotalCidSize.LargeSize += ChunkStoreSize.LargeSize; - TotalCidSize.TotalSize += ChunkStoreSize.TotalSize; - } - } + const GcStorageSize StoreSize = m_ProjectStore->StorageSize(); + const CidStoreSize CidSize = m_CidStore.TotalSize(); CbObjectWriter Cbo; @@ -495,66 +460,12 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) { Cbo.BeginObject("size"); { - Cbo << "tiny" << TotalCidSize.TinySize; - Cbo << "small" << TotalCidSize.SmallSize; - Cbo << "large" << TotalCidSize.LargeSize; - Cbo << "total" << TotalCidSize.TotalSize; + Cbo << "tiny" << CidSize.TinySize; + Cbo << "small" << CidSize.SmallSize; + Cbo << "large" << CidSize.LargeSize; + Cbo << "total" << CidSize.TotalSize; } Cbo.EndObject(); - - if (ShowCidStoreStats) - { - Cbo << "cidhits" << TotalChunkHitCount << "cidmisses" << TotalChunkMissCount << "cidwrites" << TotalChunkWriteCount; - const uint64_t TotalChunkCount = TotalChunkHitCount + TotalChunkMissCount; - Cbo << "cidhit_ratio" << (TotalChunkHitCount ? (double(TotalChunkCount) / double(TotalChunkHitCount)) : 0.0); - - Cbo.BeginObject("store"); - - auto OutputStats = [&](CidStore& ChunkStore) { - CidStoreStats StoreStats = ChunkStore.Stats(); - Cbo << "hits" << StoreStats.HitCount << "misses" << StoreStats.MissCount << "writes" << StoreStats.WriteCount; - const uint64_t Count = StoreStats.HitCount + StoreStats.MissCount; - Cbo << "hit_ratio" << (Count ? (double(StoreStats.HitCount) / double(Count)) : 0.0); - EmitSnapshot("read", StoreStats.FindChunkOps, Cbo); - EmitSnapshot("write", StoreStats.AddChunkOps, Cbo); - }; - - if (UniqueStores.size() > 1) - { - Cbo.BeginArray("projects"); - for (auto It : UniqueStores) - { - CidStore* ChunkStore = It.first; - const std::string& ProjectId = It.second; - CidStoreSize ChunkStoreSize = ChunkStore->TotalSize(); - - Cbo.BeginObject(); - { - Cbo << "project" << ProjectId; - Cbo.BeginObject("stats"); - OutputStats(*ChunkStore); - Cbo.EndObject(); - - Cbo.BeginObject("size"); - { - Cbo << "tiny" << ChunkStoreSize.TinySize; - Cbo << "small" << ChunkStoreSize.SmallSize; - Cbo << "large" << ChunkStoreSize.LargeSize; - Cbo << "total" << ChunkStoreSize.TotalSize; - } - Cbo.EndObject(); - } - Cbo.EndObject(); - } - Cbo.EndArray(); // projects - } - else if (UniqueStores.size() != 0) - { - CidStore& ChunkStore = *UniqueStores.begin()->first; - OutputStats(ChunkStore); - } - Cbo.EndObject(); - } } Cbo.EndObject(); @@ -1214,8 +1125,6 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) } Project->TouchOplog(OplogId); - CidStore& ChunkStore = Project->GetCidStore(); - ProjectStore::Oplog& Oplog = *FoundLog; IoBuffer Payload = HttpReq.ReadPayload(); @@ -1228,7 +1137,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) std::vector<IoHash> MissingChunks; CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer { - if (ChunkStore.ContainsChunk(Hash)) + if (m_CidStore.ContainsChunk(Hash)) { // Return null attachment as we already have it, no point in reading it and storing it again return {}; @@ -1484,8 +1393,6 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) } Project->TouchOplog(OplogId); - CidStore& ChunkStore = Project->GetCidStore(); - ProjectStore::Oplog& Oplog = *FoundLog; if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString)) @@ -1500,7 +1407,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) Op.IterateAttachments([&](CbFieldView FieldView) { const IoHash AttachmentHash = FieldView.AsAttachment(); - IoBuffer Payload = ChunkStore.FindChunkByCid(AttachmentHash); + IoBuffer Payload = m_CidStore.FindChunkByCid(AttachmentHash); if (Payload) { switch (Payload.GetContentType()) @@ -2129,14 +2036,11 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req) CSVHeader(Details, AttachmentDetails, CSVWriter); m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) { - CidStore& ChunkStore = Project.GetCidStore(); - Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) { - Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, - const Oid& Key, - CbObjectView Op) { - CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); - }); + Oplog.IterateOplogWithKey( + [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) { + CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + }); }); }); @@ -2150,9 +2054,8 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req) m_ProjectStore->DiscoverProjects(); m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) { - CidStore& ChunkStore = Project.GetCidStore(); - std::vector<std::string> OpLogs = Project.ScanForOplogs(); - CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); + std::vector<std::string> OpLogs = Project.ScanForOplogs(); + CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); }); } Cbo.EndArray(); @@ -2181,8 +2084,7 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req) { return HttpReq.WriteResponse(HttpResponseCode::NotFound); } - ProjectStore::Project& Project = *FoundProject.Get(); - CidStore& ChunkStore = Project.GetCidStore(); + ProjectStore::Project& Project = *FoundProject.Get(); if (CSV) { @@ -2190,11 +2092,10 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req) CSVHeader(Details, AttachmentDetails, CSVWriter); FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) { - Oplog.IterateOplogWithKey([this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, - const Oid& Key, - CbObjectView Op) { - CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); - }); + Oplog.IterateOplogWithKey( + [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) { + CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + }); }); HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); } @@ -2204,7 +2105,7 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req) std::vector<std::string> OpLogs = FoundProject->ScanForOplogs(); Cbo.BeginArray("projects"); { - CbWriteProject(ChunkStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); + CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); } Cbo.EndArray(); HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); @@ -2240,17 +2141,16 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req) return HttpReq.WriteResponse(HttpResponseCode::NotFound); } - ProjectStore::Project& Project = *FoundProject.Get(); - CidStore& ChunkStore = Project.GetCidStore(); - ProjectStore::Oplog& Oplog = *FoundLog; + ProjectStore::Project& Project = *FoundProject.Get(); + ProjectStore::Oplog& Oplog = *FoundLog; if (CSV) { ExtendableStringBuilder<4096> CSVWriter; CSVHeader(Details, AttachmentDetails, CSVWriter); Oplog.IterateOplogWithKey( - [this, &Project, &Oplog, &ChunkStore, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) { - CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](uint32_t LSN, const Oid& Key, CbObjectView Op) { + CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); }); HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); } @@ -2259,7 +2159,7 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req) CbObjectWriter Cbo; Cbo.BeginArray("oplogs"); { - CbWriteOplog(ChunkStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo); + CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo); } Cbo.EndArray(); HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); @@ -2304,10 +2204,9 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req) fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, OpId)); } - const Oid ObjId = Oid::FromHexString(OpId); - ProjectStore::Project& Project = *FoundProject.Get(); - CidStore& ChunkStore = Project.GetCidStore(); - ProjectStore::Oplog& Oplog = *FoundLog; + const Oid ObjId = Oid::FromHexString(OpId); + ProjectStore::Project& Project = *FoundProject.Get(); + ProjectStore::Oplog& Oplog = *FoundLog; std::optional<CbObject> Op = Oplog.GetOpByKey(ObjId); if (!Op.has_value()) @@ -2325,7 +2224,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req) ExtendableStringBuilder<4096> CSVWriter; CSVHeader(Details, AttachmentDetails, CSVWriter); - CSVWriteOp(ChunkStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter); + CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN.value(), ObjId, Op.value(), CSVWriter); HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); } else @@ -2333,7 +2232,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req) CbObjectWriter Cbo; Cbo.BeginArray("ops"); { - CbWriteOp(ChunkStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo); + CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN.value(), ObjId, Op.value(), Cbo); } Cbo.EndArray(); HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h index 5782188e6..295defa5c 100644 --- a/src/zenserver/projectstore/httpprojectstore.h +++ b/src/zenserver/projectstore/httpprojectstore.h @@ -35,7 +35,11 @@ class ProjectStore; class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: - HttpProjectService(ProjectStore* InProjectStore, HttpStatusService& StatusService, HttpStatsService& StatsService, AuthMgr& AuthMgr); + HttpProjectService(CidStore& Store, + ProjectStore* InProjectStore, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr); ~HttpProjectService(); virtual const char* BaseUri() const override; @@ -88,6 +92,7 @@ private: inline LoggerRef Log() { return m_Log; } LoggerRef m_Log; + CidStore& m_CidStore; HttpRequestRouter m_Router; Ref<ProjectStore> m_ProjectStore; HttpStatusService& m_StatusService; diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 7463e9caa..a5ab24cfb 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -3832,7 +3832,7 @@ ProjectStore::Project::LastOplogAccessTime(std::string_view Oplog) const ////////////////////////////////////////////////////////////////////////// -ProjectStore::ProjectStore(GetCidStoreFunc&& GetCidStore, +ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue, @@ -3840,7 +3840,7 @@ ProjectStore::ProjectStore(GetCidStoreFunc&& GetCidStore, const Configuration& Config) : m_Log(logging::Get("project")) , m_Gc(Gc) -, m_GetCidStore(std::move(GetCidStore)) +, m_CidStore(Store) , m_JobQueue(JobQueue) , m_OpenProcessCache(InOpenProcessCache) , m_ProjectBasePath(BasePath) @@ -4025,8 +4025,6 @@ ProjectStore::OpenProject(std::string_view ProjectId) } } - CidStore& ChunkStore = m_GetCidStore(ProjectId); - RwLock::ExclusiveLockScope _(m_ProjectsLock); if (auto ProjIt = m_Projects.find(std::string{ProjectId}); ProjIt != m_Projects.end()) { @@ -4043,7 +4041,7 @@ ProjectStore::OpenProject(std::string_view ProjectId) Ref<Project>& Prj = m_Projects - .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, ChunkStore, BasePath))) + .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath))) .first->second; Prj->Identifier = ProjectId; Prj->Read(); @@ -4070,14 +4068,12 @@ ProjectStore::NewProject(const std::filesystem::path& BasePath, ZEN_MEMSCOPE(GetProjectstoreTag()); ZEN_TRACE_CPU("Store::NewProject"); - CidStore& ChunkStore = m_GetCidStore(ProjectId); - RwLock::ExclusiveLockScope _(m_ProjectsLock); ZEN_INFO("project '{}': creating project at '{}'", ProjectId, BasePath); Ref<Project>& Prj = - m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, ChunkStore, BasePath))) + m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath))) .first->second; Prj->Identifier = ProjectId; Prj->RootDir = RootDir; @@ -4806,7 +4802,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId, } const IoHash Hash = IoHash::FromHexString(Cid); - OutChunk = Project->GetCidStore().FindChunkByCid(Hash); + OutChunk = m_CidStore.FindChunkByCid(Hash); if (!OutChunk) { @@ -4869,7 +4865,7 @@ ProjectStore::PutChunk(const std::string_view ProjectId, } FoundLog->CaptureAddedAttachments(std::vector<IoHash>{Hash}); - CidStore::InsertResult Result = Project->GetCidStore().AddChunk(Chunk, Hash); + CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk, Hash); return {Result.New ? HttpResponseCode::Created : HttpResponseCode::OK, {}}; } @@ -4898,19 +4894,18 @@ ProjectStore::GetChunks(const std::string_view ProjectId, } Project->TouchOplog(OplogId); - CidStore& ChunkStore = Project->GetCidStore(); - if (RequestObject["chunks"sv].IsArray()) { // Legacy full chunks only by rawhash - CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView(); + CbArrayView ChunksArray = RequestObject["chunks"sv].AsArrayView(); + CbObjectWriter ResponseWriter; ResponseWriter.BeginArray("chunks"sv); for (CbFieldView FieldView : ChunksArray) { IoHash RawHash = FieldView.AsHash(); - IoBuffer ChunkBuffer = ChunkStore.FindChunkByCid(RawHash); + IoBuffer ChunkBuffer = m_CidStore.FindChunkByCid(RawHash); if (ChunkBuffer) { CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(ChunkBuffer)); @@ -5062,7 +5057,7 @@ ProjectStore::GetChunks(const std::string_view ProjectId, if (ChunkRequest.Input.Id.index() == 0) { const IoHash& ChunkHash = std::get<IoHash>(ChunkRequest.Input.Id); - IoBuffer Payload = ChunkStore.FindChunkByCid(ChunkHash); + IoBuffer Payload = m_CidStore.FindChunkByCid(ChunkHash); if (Payload) { ChunkRequest.Output.Exists = true; @@ -5249,7 +5244,7 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie return {HttpResponseCode::BadRequest, "Invalid payload format"}; } - CidStore& ChunkStore = Project->GetCidStore(); + CidStore& ChunkStore = m_CidStore; RwLock AttachmentsLock; tsl::robin_set<IoHash, IoHash::Hasher> Attachments; @@ -5355,7 +5350,7 @@ ProjectStore::ReadOplog(const std::string_view ProjectId, } } - CidStore& ChunkStore = Project->GetCidStore(); + CidStore& ChunkStore = m_CidStore; RemoteProjectStore::LoadContainerResult ContainerResult = BuildContainer( ChunkStore, @@ -5467,8 +5462,6 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, } Project->TouchOplog(OplogId); - CidStore& ChunkStore = Project->GetCidStore(); - if (Method == "import"sv) { if (!AreDiskWritesAllowed()) @@ -5550,7 +5543,7 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, } Oplog->CaptureAddedAttachments(WriteRawHashes); - ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly); + m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly); } HttpReq.WriteResponse(HttpResponseCode::OK); return true; @@ -5723,14 +5716,14 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, ResponseObj.EndArray(); } - // Ops that have moved chunks to a compressed buffer for storage in ChunkStore have been rewritten with references to the new - // chunk(s). Make sure we add the chunks to ChunkStore, and do it after we update the oplog so GC doesn't think we have + // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the new + // chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have // unreferenced chunks. for (auto It : AddedChunks) { const IoHash& RawHash = It.first; AddedChunk& Chunk = It.second; - CidStore::InsertResult Result = ChunkStore.AddChunk(Chunk.Buffer, RawHash); + CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk.Buffer, RawHash); if (Result.New) { InlinedBytes += Chunk.RawSize; @@ -5793,7 +5786,7 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op NiceBytes(MaxBlockSize), NiceBytes(MaxChunkEmbedSize))); - RemoteProjectStore::Result Result = SaveOplog(Project->GetCidStore(), + RemoteProjectStore::Result Result = SaveOplog(m_CidStore, *ActualRemoteStore, *Project.Get(), *OplogPtr, @@ -5838,12 +5831,11 @@ ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog, } std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store); RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); - CidStore& ChunkStore = Project.GetCidStore(); JobId JobId = m_JobQueue.QueueJob( fmt::format("Import oplog '{}/{}'", Project.Identifier, Oplog.OplogId()), [this, - ChunkStore = &ChunkStore, + ChunkStore = &m_CidStore, ActualRemoteStore = std::move(RemoteStore), OplogPtr = &Oplog, Force, @@ -5855,7 +5847,7 @@ ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog, ActualRemoteStore->GetInfo().Description)); RemoteProjectStore::Result Result = - LoadOplog(*ChunkStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, CleanOplog, &Context); + LoadOplog(m_CidStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, CleanOplog, &Context); auto Response = ConvertResult(Result); ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second); if (!IsHttpSuccessCode(Response.first)) @@ -6894,11 +6886,6 @@ namespace testutils { return BuildChunksRequest<IoHash>(SkipData, "RawHash", Chunks, Ranges, ModTags); } - ProjectStore::GetCidStoreFunc SingleChunkStore(CidStore& ChunkStore) - { - return [ChunkStore = &ChunkStore](std::string_view) -> CidStore& { return *ChunkStore; }; - } - } // namespace testutils TEST_CASE("project.opkeys") @@ -6961,18 +6948,13 @@ TEST_CASE("project.store.create") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore ChunkStore(Gc); + CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - ChunkStore.Initialize(CidConfig); + CidStore.Initialize(CidConfig); std::string_view ProjectName("proj1"sv); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), - BasePath, - Gc, - *JobQueue, - ProcessCache, - ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; @@ -6997,17 +6979,12 @@ TEST_CASE("project.store.lifetimes") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore ChunkStore(Gc); + CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - ChunkStore.Initialize(CidConfig); + CidStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), - BasePath, - Gc, - *JobQueue, - ProcessCache, - ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; @@ -7065,17 +7042,12 @@ TEST_CASE_TEMPLATE("project.store.export", auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore ChunkStore(Gc); + CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - ChunkStore.Initialize(CidConfig); + CidStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), - BasePath, - Gc, - *JobQueue, - ProcessCache, - ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; @@ -7109,7 +7081,7 @@ TEST_CASE_TEMPLATE("project.store.export", std::shared_ptr<RemoteProjectStore> RemoteStore = CreateFileRemoteStore(Options); RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); - RemoteProjectStore::Result ExportResult = SaveOplog(ChunkStore, + RemoteProjectStore::Result ExportResult = SaveOplog(CidStore, *RemoteStore, *Project.Get(), *Oplog, @@ -7126,7 +7098,7 @@ TEST_CASE_TEMPLATE("project.store.export", ProjectStore::Oplog* OplogImport = Project->NewOplog("oplog2", {}); CHECK(OplogImport != nullptr); - RemoteProjectStore::Result ImportResult = LoadOplog(ChunkStore, + RemoteProjectStore::Result ImportResult = LoadOplog(CidStore, *RemoteStore, *OplogImport, /*Force*/ false, @@ -7135,7 +7107,7 @@ TEST_CASE_TEMPLATE("project.store.export", nullptr); CHECK(ImportResult.ErrorCode == 0); - RemoteProjectStore::Result ImportForceResult = LoadOplog(ChunkStore, + RemoteProjectStore::Result ImportForceResult = LoadOplog(CidStore, *RemoteStore, *OplogImport, /*Force*/ true, @@ -7144,7 +7116,7 @@ TEST_CASE_TEMPLATE("project.store.export", nullptr); CHECK(ImportForceResult.ErrorCode == 0); - RemoteProjectStore::Result ImportCleanResult = LoadOplog(ChunkStore, + RemoteProjectStore::Result ImportCleanResult = LoadOplog(CidStore, *RemoteStore, *OplogImport, /*Force*/ false, @@ -7153,7 +7125,7 @@ TEST_CASE_TEMPLATE("project.store.export", nullptr); CHECK(ImportCleanResult.ErrorCode == 0); - RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(ChunkStore, + RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(CidStore, *RemoteStore, *OplogImport, /*Force*/ true, @@ -7173,17 +7145,12 @@ TEST_CASE("project.store.gc") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore ChunkStore(Gc); + CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - ChunkStore.Initialize(CidConfig); + CidStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), - BasePath, - Gc, - *JobQueue, - ProcessCache, - ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; @@ -7379,17 +7346,12 @@ TEST_CASE("project.store.gc.prep") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore ChunkStore(Gc); + CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - ChunkStore.Initialize(CidConfig); + CidStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), - BasePath, - Gc, - *JobQueue, - ProcessCache, - ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; @@ -7432,7 +7394,7 @@ TEST_CASE("project.store.gc.prep") // Equivalent of a `prep` existance check call for (auto Attachment : OpAttachments) { - CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7446,7 +7408,7 @@ TEST_CASE("project.store.gc.prep") // If a gc comes in between our prep and op write the chunks will be removed for (auto Attachment : OpAttachments) { - CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7466,7 +7428,7 @@ TEST_CASE("project.store.gc.prep") for (auto Attachment : OpAttachments) { - CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7480,7 +7442,7 @@ TEST_CASE("project.store.gc.prep") // Attachments should now be retained for (auto Attachment : OpAttachments) { - CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7494,7 +7456,7 @@ TEST_CASE("project.store.gc.prep") // Attachments should now be retained across multiple GCs if retain time is still valud for (auto Attachment : OpAttachments) { - CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7506,7 +7468,7 @@ TEST_CASE("project.store.gc.prep") } for (auto Attachment : OpAttachments) { - CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv); @@ -7523,7 +7485,7 @@ TEST_CASE("project.store.gc.prep") for (auto Attachment : OpAttachments) { - CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); } { @@ -7556,7 +7518,7 @@ TEST_CASE("project.store.gc.prep") } for (auto Attachment : OpAttachments) { - CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); } Sleep(200); @@ -7571,7 +7533,7 @@ TEST_CASE("project.store.gc.prep") } for (auto Attachment : OpAttachments) { - CHECK(ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); } // This pass the retention time has expired and the last GC pass cleared the entries @@ -7585,7 +7547,7 @@ TEST_CASE("project.store.gc.prep") for (auto Attachment : OpAttachments) { - CHECK(!ChunkStore.ContainsChunk(Attachment.second.DecodeRawHash())); + CHECK(!CidStore.ContainsChunk(Attachment.second.DecodeRawHash())); } } @@ -7599,17 +7561,12 @@ TEST_CASE("project.store.rpc.getchunks") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore ChunkStore(Gc); + CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - ChunkStore.Initialize(CidConfig); + CidStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; - ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), - BasePath, - Gc, - *JobQueue, - ProcessCache, - ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"sv; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv; @@ -8526,17 +8483,12 @@ TEST_CASE("project.store.partial.read") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore ChunkStore(Gc); + CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - ChunkStore.Initialize(CidConfig); + CidStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; - ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), - BasePath, - Gc, - *JobQueue, - ProcessCache, - ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"sv; std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv; @@ -8709,17 +8661,12 @@ TEST_CASE("project.store.iterateoplog") auto JobQueue = MakeJobQueue(1, ""sv); OpenProcessCache ProcessCache; GcManager Gc; - CidStore ChunkStore(Gc); + CidStore CidStore(Gc); CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - ChunkStore.Initialize(CidConfig); + CidStore.Initialize(CidConfig); std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; - ProjectStore ProjectStore(testutils::SingleChunkStore(ChunkStore), - BasePath, - Gc, - *JobQueue, - ProcessCache, - ProjectStore::Configuration{}); + ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProcessCache, ProjectStore::Configuration{}); std::filesystem::path RootDir = TempDir.Path() / "root"sv; std::filesystem::path EngineRootDir = TempDir.Path() / "enginesv"; diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index eb27665f9..2595d7198 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -68,9 +68,7 @@ public: { }; - typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc; - - ProjectStore(GetCidStoreFunc&& GetCidStore, + ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue, @@ -328,8 +326,6 @@ public: Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath); virtual ~Project(); - CidStore& GetCidStore() { return m_CidStore; }; - void Read(); void Write(); [[nodiscard]] static bool Exists(const std::filesystem::path& BasePath); @@ -503,7 +499,7 @@ public: private: LoggerRef m_Log; GcManager& m_Gc; - GetCidStoreFunc m_GetCidStore; + CidStore& m_CidStore; JobQueue& m_JobQueue; OpenProcessCache& m_OpenProcessCache; std::filesystem::path m_ProjectBasePath; diff --git a/src/zenserver/upstream/upstreamcache.cpp b/src/zenserver/upstream/upstreamcache.cpp index a1c460bc0..744b861dd 100644 --- a/src/zenserver/upstream/upstreamcache.cpp +++ b/src/zenserver/upstream/upstreamcache.cpp @@ -1475,17 +1475,12 @@ namespace detail { class UpstreamCacheImpl final : public UpstreamCache { - struct EnqueuedRequest - { - UpstreamCacheRecord Record; - std::function<IoBuffer(const IoHash& ChunkHash)> GetValueFunc; - }; - public: - UpstreamCacheImpl(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore) + UpstreamCacheImpl(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore) : m_Log(logging::Get("upstream")) , m_Options(Options) , m_CacheStore(CacheStore) + , m_CidStore(CidStore) { } @@ -1841,17 +1836,17 @@ public: } } - virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord, std::function<IoBuffer(const IoHash&)>&& GetValueFunc) override + virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord) override { if (m_RunState.IsRunning && m_Options.WriteUpstream && m_Endpoints.size() > 0) { if (!m_UpstreamThreads.empty()) { - m_UpstreamQueue.Enqueue(EnqueuedRequest{.Record = std::move(CacheRecord), .GetValueFunc = GetValueFunc}); + m_UpstreamQueue.Enqueue(std::move(CacheRecord)); } else { - ProcessCacheRecord(std::move(CacheRecord), std::move(GetValueFunc)); + ProcessCacheRecord(std::move(CacheRecord)); } } } @@ -1905,7 +1900,7 @@ public: } private: - void ProcessCacheRecord(const UpstreamCacheRecord& CacheRecord, std::function<IoBuffer(const IoHash& ChunkHash)>&& GetValueFunc) + void ProcessCacheRecord(UpstreamCacheRecord CacheRecord) { ZEN_TRACE_CPU("Upstream::ProcessCacheRecord"); @@ -1923,7 +1918,7 @@ private: for (const IoHash& ValueContentId : CacheRecord.ValueContentIds) { - if (IoBuffer Payload = GetValueFunc(ValueContentId)) + if (IoBuffer Payload = m_CidStore.FindChunkByCid(ValueContentId)) { Payloads.push_back(Payload); } @@ -1975,19 +1970,19 @@ private: for (;;) { - EnqueuedRequest Request; - if (m_UpstreamQueue.WaitAndDequeue(Request)) + UpstreamCacheRecord CacheRecord; + if (m_UpstreamQueue.WaitAndDequeue(CacheRecord)) { try { - ProcessCacheRecord(Request.Record, std::move(Request.GetValueFunc)); + ProcessCacheRecord(std::move(CacheRecord)); } catch (const std::exception& Err) { ZEN_ERROR("upload cache record '{}/{}/{}' FAILED, reason '{}'", - Request.Record.Namespace, - Request.Record.Key.Bucket, - Request.Record.Key.Hash, + CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, Err.what()); } } @@ -2081,7 +2076,7 @@ private: LoggerRef Log() { return m_Log; } - using UpstreamQueue = BlockingQueue<EnqueuedRequest>; + using UpstreamQueue = BlockingQueue<UpstreamCacheRecord>; struct RunState { @@ -2107,6 +2102,7 @@ private: LoggerRef m_Log; UpstreamCacheOptions m_Options; ZenCacheStore& m_CacheStore; + CidStore& m_CidStore; UpstreamQueue m_UpstreamQueue; std::shared_mutex m_EndpointsMutex; std::vector<std::unique_ptr<UpstreamEndpoint>> m_Endpoints; @@ -2130,9 +2126,9 @@ UpstreamEndpoint::CreateJupiterEndpoint(const JupiterClientOptions& Options, con } std::unique_ptr<UpstreamCache> -CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore) +CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore) { - return std::make_unique<UpstreamCacheImpl>(Options, CacheStore); + return std::make_unique<UpstreamCacheImpl>(Options, CacheStore, CidStore); } } // namespace zen diff --git a/src/zenserver/upstream/upstreamcache.h b/src/zenserver/upstream/upstreamcache.h index e4b9a73ad..26e5decac 100644 --- a/src/zenserver/upstream/upstreamcache.h +++ b/src/zenserver/upstream/upstreamcache.h @@ -24,6 +24,7 @@ class AuthMgr; class CbObjectView; class CbPackage; class CbObjectWriter; +class CidStore; class ZenCacheStore; struct JupiterClientOptions; class JupiterAccessTokenProvider; @@ -161,6 +162,6 @@ struct UpstreamCacheOptions bool WriteUpstream = true; }; -std::unique_ptr<UpstreamCache> CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore); +std::unique_ptr<UpstreamCache> CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore); } // namespace zen diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 71b52817c..44c25368c 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -187,7 +187,14 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_Http = CreateHttpServer(ServerOptions.HttpServerConfig); int EffectiveBasePort = m_Http->Initialize(ServerOptions.BasePort, ServerOptions.DataDir); - ZEN_ASSERT(EffectiveBasePort > 0); + if (EffectiveBasePort == 0) + { + ZEN_WARN("Failed to initialize http service '{}' using base port {} and data dir {}", + ServerOptions.HttpServerConfig.ServerClass, + ServerOptions.BasePort, + ServerOptions.DataDir); + return -1; + } // Setup authentication manager { @@ -198,7 +205,10 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen { EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; - ZEN_WARN("using default encryption key"); + if (ServerOptions.IsDedicated) + { + ZEN_WARN("Using default encryption key for authentication state"); + } } std::string EncryptionIV = ServerOptions.EncryptionIV; @@ -207,7 +217,10 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen { EncryptionIV = "0123456789abcdef"; - ZEN_WARN("using default encryption initialization vector"); + if (ServerOptions.IsDedicated) + { + ZEN_WARN("Using default encryption initialization vector for authentication state"); + } } m_AuthMgr = AuthMgr::Create({.RootDirectory = m_DataRoot / "auth", @@ -242,18 +255,18 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen CidStoreConfiguration Config; Config.RootDirectory = m_DataRoot / "cas"; - m_CidStores.insert_or_assign({}, std::make_unique<CidStore>(m_GcManager)); - m_CidStores.at({})->Initialize(Config); + m_CidStore = std::make_unique<CidStore>(m_GcManager); + m_CidStore->Initialize(Config); ZEN_INFO("instantiating project service"); - m_ProjectStore = new ProjectStore([this](std::string_view) -> CidStore& { return *m_CidStores.at({}).get(); }, + m_ProjectStore = new ProjectStore(*m_CidStore, m_DataRoot / "projects", m_GcManager, *m_JobQueue, *m_OpenProcessCache, ProjectStore::Configuration{}); - m_HttpProjectService.reset(new HttpProjectService(m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr)); + m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr}); if (ServerOptions.WorksSpacesConfig.Enabled) { @@ -600,7 +613,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) UpstreamOptions.ThreadCount = static_cast<uint32_t>(UpstreamConfig.UpstreamThreadCount); } - m_UpstreamCache = CreateUpstreamCache(UpstreamOptions, *m_CacheStore); + m_UpstreamCache = CreateUpstreamCache(UpstreamOptions, *m_CacheStore, *m_CidStore); m_UpstreamService = std::make_unique<HttpUpstreamService>(*m_UpstreamCache, *m_AuthMgr); m_UpstreamCache->Initialize(); @@ -664,23 +677,19 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) } } - m_StructuredCacheService = std::make_unique<HttpStructuredCacheService>( - *m_CacheStore, - [this](std::string_view) -> CidStore& { return *m_CidStores.at({}).get(); }, - m_StatsService, - m_StatusService, - *m_UpstreamCache, - m_GcManager.GetDiskWriteBlocker(), - *m_OpenProcessCache); + m_StructuredCacheService = std::make_unique<HttpStructuredCacheService>(*m_CacheStore, + *m_CidStore, + m_StatsService, + m_StatusService, + *m_UpstreamCache, + m_GcManager.GetDiskWriteBlocker(), + *m_OpenProcessCache); m_Http->RegisterService(*m_StructuredCacheService); m_Http->RegisterService(*m_UpstreamService); m_StatsReporter.AddProvider(m_CacheStore.Get()); - for (const auto& It : m_CidStores) - { - m_StatsReporter.AddProvider(It.second.get()); - } + m_StatsReporter.AddProvider(m_CidStore.get()); m_StatsReporter.AddProvider(m_BuildCidStore.get()); } @@ -865,7 +874,7 @@ ZenServer::Cleanup() m_Workspaces.reset(); m_HttpProjectService.reset(); m_ProjectStore = {}; - m_CidStores.clear(); + m_CidStore.reset(); m_AuthService.reset(); m_AuthMgr.reset(); m_Http = {}; @@ -1053,10 +1062,8 @@ ZenServer::Flush() { ZEN_TRACE_CPU("ZenServer::Flush"); - for (auto& It : m_CidStores) - { - It.second->Flush(); - } + if (m_CidStore) + m_CidStore->Flush(); if (m_StructuredCacheService) m_StructuredCacheService->Flush(); diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h index 0a17446ae..bcb02d336 100644 --- a/src/zenserver/zenserver.h +++ b/src/zenserver/zenserver.h @@ -115,20 +115,20 @@ private: inline void SetNewState(ServerState NewState) { m_CurrentState = NewState; } static std::string_view ToString(ServerState Value); - StatsReporter m_StatsReporter; - Ref<HttpServer> m_Http; - std::unique_ptr<AuthMgr> m_AuthMgr; - std::unique_ptr<HttpAuthService> m_AuthService; - HttpStatusService m_StatusService; - HttpStatsService m_StatsService; - GcManager m_GcManager; - GcScheduler m_GcScheduler{m_GcManager}; - tsl::robin_map<std::string, std::unique_ptr<CidStore>> m_CidStores; - Ref<ZenCacheStore> m_CacheStore; - std::unique_ptr<OpenProcessCache> m_OpenProcessCache; - HttpTestService m_TestService; - std::unique_ptr<CidStore> m_BuildCidStore; - std::unique_ptr<BuildStore> m_BuildStore; + StatsReporter m_StatsReporter; + Ref<HttpServer> m_Http; + std::unique_ptr<AuthMgr> m_AuthMgr; + std::unique_ptr<HttpAuthService> m_AuthService; + HttpStatusService m_StatusService; + HttpStatsService m_StatsService; + GcManager m_GcManager; + GcScheduler m_GcScheduler{m_GcManager}; + std::unique_ptr<CidStore> m_CidStore; + Ref<ZenCacheStore> m_CacheStore; + std::unique_ptr<OpenProcessCache> m_OpenProcessCache; + HttpTestService m_TestService; + std::unique_ptr<CidStore> m_BuildCidStore; + std::unique_ptr<BuildStore> m_BuildStore; #if ZEN_WITH_TESTS HttpTestingService m_TestingService; diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index ff21d1ede..5d9a68919 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -153,13 +153,13 @@ CacheRpcHandler::CacheRpcHandler(LoggerRef InLog, CacheStats& InCacheStats, UpstreamCacheClient& InUpstreamCache, ZenCacheStore& InCacheStore, - GetCidStoreFunc&& InGetCidStore, + CidStore& InCidStore, const DiskWriteBlocker* InDiskWriteBlocker) : m_Log(InLog) , m_CacheStats(InCacheStats) , m_UpstreamCache(InUpstreamCache) , m_CacheStore(InCacheStore) -, m_GetCidStore(std::move(InGetCidStore)) +, m_CidStore(InCidStore) , m_DiskWriteBlocker(InDiskWriteBlocker) { } @@ -174,12 +174,6 @@ CacheRpcHandler::AreDiskWritesAllowed() const return (m_DiskWriteBlocker == nullptr || m_DiskWriteBlocker->AreDiskWritesAllowed()); } -CidStore& -CacheRpcHandler::GetCidStore(std::string_view Namespace) -{ - return m_GetCidStore(Namespace); -} - CacheRpcHandler::RpcResponseCode CacheRpcHandler::HandleRpcRequest(const CacheRequestContext& Context, std::string_view UriNamespace, @@ -387,12 +381,9 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag Stopwatch Timer; - CidStore& ChunkStore = m_GetCidStore(Request.Namespace); - Request.RecordObject.IterateAttachments([this, &Request, Package, - &ChunkStore, &WriteAttachmentBuffers, &WriteRawHashes, &ValidAttachments, @@ -421,7 +412,7 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag Count.Invalid++; } } - else if (ChunkStore.ContainsChunk(ValueHash)) + else if (m_CidStore.ContainsChunk(ValueHash)) { ValidAttachments.emplace_back(ValueHash); Count.Valid++; @@ -457,7 +448,7 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag if (!WriteAttachmentBuffers.empty()) { - std::vector<CidStore::InsertResult> InsertResults = ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); for (size_t Index = 0; Index < InsertResults.size(); Index++) { if (InsertResults[Index].New) @@ -484,12 +475,10 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag if (HasUpstream && EnumHasAllFlags(Request.Policy.GetRecordPolicy(), CachePolicy::StoreRemote) && !IsPartialRecord) { - m_UpstreamCache.EnqueueUpstream( - {.Type = ZenContentType::kCbPackage, - .Namespace = Request.Namespace, - .Key = Request.Key, - .ValueContentIds = std::move(ValidAttachments)}, - [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); }); + m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage, + .Namespace = Request.Namespace, + .Key = Request.Key, + .ValueContentIds = std::move(ValidAttachments)}); } return PutStatus::Success; } @@ -532,8 +521,6 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb return CbPackage{}; } - CidStore& ChunkStore = m_GetCidStore(Namespace.value()); - const bool HasUpstream = m_UpstreamCache.IsActive(); eastl::fixed_vector<RecordRequestData, 16> Requests; @@ -633,7 +620,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb } else if (EnumHasAllFlags(ValuePolicy, CachePolicy::SkipData)) { - if (ChunkStore.ContainsChunk(Value.ContentId)) + if (m_CidStore.ContainsChunk(Value.ContentId)) { Value.Exists = true; } @@ -654,7 +641,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb } else { - if (IoBuffer Chunk = ChunkStore.FindChunkByCid(Value.ContentId)) + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Value.ContentId)) { if (Chunk.GetSize() > 0) { @@ -678,7 +665,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb } if (!RequestValueIndexes.empty()) { - ChunkStore.IterateChunks( + m_CidStore.IterateChunks( CidHashes, [this, &Request, &RequestValueIndexes](size_t Index, const IoBuffer& Payload) -> bool { try @@ -771,7 +758,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb } } - const auto OnCacheRecordGetComplete = [this, Namespace, &ChunkStore, &ParseValues, Context](CacheRecordGetCompleteParams&& Params) { + const auto OnCacheRecordGetComplete = [this, Namespace, &ParseValues, Context](CacheRecordGetCompleteParams&& Params) { if (!Params.Record) { return; @@ -840,7 +827,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb Value.Exists = true; if (StoreLocal) { - ChunkStore.AddChunk(Compressed.GetCompressed().Flatten().AsIoBuffer(), Attachment->GetHash()); + m_CidStore.AddChunk(Compressed.GetCompressed().Flatten().AsIoBuffer(), Attachment->GetHash()); } if (!EnumHasAllFlags(ValuePolicy, CachePolicy::SkipData)) { @@ -957,8 +944,6 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con const bool HasUpstream = m_UpstreamCache.IsActive(); CbArrayView RequestsArray = Params["Requests"sv].AsArrayView(); - CidStore& ChunkStore = m_GetCidStore(Namespace.value()); - std::vector<ZenCacheStore::PutResult> BatchResults; eastl::fixed_vector<size_t, 32> BatchResultIndexes; eastl::fixed_vector<ZenCacheStore::PutResult, 32> Results; @@ -1109,8 +1094,7 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con if ((Results[Index].Status == zen::PutStatus::Success) && UpstreamCacheKeys[Index] != CacheKey::Empty) { m_UpstreamCache.EnqueueUpstream( - {.Type = ZenContentType::kCompressedBinary, .Namespace = *Namespace, .Key = UpstreamCacheKeys[Index]}, - [ChunkStore = &ChunkStore](const IoHash& ValueHash) { return ChunkStore->FindChunkByCid(ValueHash); }); + {.Type = ZenContentType::kCompressedBinary, .Namespace = *Namespace, .Key = UpstreamCacheKeys[Index]}); } } { @@ -1562,8 +1546,6 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, using namespace cache::detail; const bool HasUpstream = m_UpstreamCache.IsActive(); - CidStore& ChunkStore = m_GetCidStore(Namespace); - // TODO: BatchGet records? std::vector<CacheKeyRequest*> UpstreamRecordRequests; for (size_t RecordIndex = 0; RecordIndex < Records.size(); ++RecordIndex) @@ -1702,12 +1684,12 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, { if (EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData) && Request->RawSizeKnown) { - if (ChunkStore.ContainsChunk(Request->Key->ChunkId)) + if (m_CidStore.ContainsChunk(Request->Key->ChunkId)) { Request->Exists = true; } } - else if (IoBuffer Payload = ChunkStore.FindChunkByCid(Request->Key->ChunkId)) + else if (IoBuffer Payload = m_CidStore.FindChunkByCid(Request->Key->ChunkId)) { if (!EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData)) { @@ -1840,8 +1822,6 @@ CacheRpcHandler::GetUpstreamCacheChunks(const CacheRequestContext& Context, return; } - CidStore& ChunkStore = m_GetCidStore(Namespace); - CacheChunkRequest& Key = Params.Request; size_t RequestIndex = std::distance(RequestKeys.data(), &Key); ChunkRequest& Request = Requests[RequestIndex]; @@ -1861,7 +1841,7 @@ CacheRpcHandler::GetUpstreamCacheChunks(const CacheRequestContext& Context, bool Overwrite = !EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::QueryLocal); if (Request.IsRecordRequest) { - ChunkStore.AddChunk(Params.Value, Params.RawHash); + m_CidStore.AddChunk(Params.Value, Params.RawHash); } else { diff --git a/src/zenstore/include/zenstore/cache/cacherpc.h b/src/zenstore/include/zenstore/cache/cacherpc.h index 80340d72c..104746aba 100644 --- a/src/zenstore/include/zenstore/cache/cacherpc.h +++ b/src/zenstore/include/zenstore/cache/cacherpc.h @@ -70,13 +70,11 @@ IsCompressedBinary(ZenContentType Type) struct CacheRpcHandler { - typedef std::function<CidStore&(std::string_view Context)> GetCidStoreFunc; - CacheRpcHandler(LoggerRef InLog, CacheStats& InCacheStats, UpstreamCacheClient& InUpstreamCache, ZenCacheStore& InCacheStore, - GetCidStoreFunc&& InGetCidStore, + CidStore& InCidStore, const DiskWriteBlocker* InDiskWriteBlocker); ~CacheRpcHandler(); @@ -96,8 +94,6 @@ struct CacheRpcHandler int& OutTargetProcessId, CbPackage& OutPackage); - CidStore& GetCidStore(std::string_view Namespace); - private: CbPackage HandleRpcPutCacheRecords(const CacheRequestContext& Context, const CbPackage& BatchRequest); CbPackage HandleRpcGetCacheRecords(const CacheRequestContext& Context, CbObjectView BatchRequest); @@ -146,7 +142,7 @@ private: CacheStats& m_CacheStats; UpstreamCacheClient& m_UpstreamCache; ZenCacheStore& m_CacheStore; - GetCidStoreFunc m_GetCidStore; + CidStore& m_CidStore; const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; bool AreDiskWritesAllowed() const; diff --git a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h index c3993c028..152031c3a 100644 --- a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h +++ b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h @@ -113,7 +113,7 @@ public: std::span<CacheChunkRequest*> CacheChunkRequests, OnCacheChunksGetComplete&& OnComplete) = 0; - virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord, std::function<IoBuffer(const IoHash&)>&& GetValueFunc) = 0; + virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord) = 0; }; } // namespace zen diff --git a/src/zenutil/include/zenutil/logging.h b/src/zenutil/include/zenutil/logging.h index d64eef207..85ddc86cd 100644 --- a/src/zenutil/include/zenutil/logging.h +++ b/src/zenutil/include/zenutil/logging.h @@ -34,6 +34,7 @@ struct LoggingOptions bool IsTest = false; bool AllowAsync = true; bool NoConsoleOutput = false; + bool QuietConsole = false; std::filesystem::path AbsLogFile; // Absolute path to main log file std::string LogId; }; diff --git a/src/zenutil/logging.cpp b/src/zenutil/logging.cpp index cb0fd6679..8ff58ee73 100644 --- a/src/zenutil/logging.cpp +++ b/src/zenutil/logging.cpp @@ -121,6 +121,10 @@ BeginInitializeLogging(const LoggingOptions& LogOptions) else { auto ConsoleSink = std::make_shared<spdlog::sinks::ansicolor_stdout_sink_mt>(); + if (LogOptions.QuietConsole) + { + ConsoleSink->set_level(spdlog::level::warn); + } Sinks.push_back(ConsoleSink); } |