From 6d07b0437ccb7800652708f76a7ee84e551f43cf Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Sun, 2 Mar 2025 00:15:35 -0700 Subject: Control overwrite enforcement with a config setting --- src/zenserver/zenserver.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index f84bc0b00..cc1be13e2 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -530,7 +530,8 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) Config.AllowAutomaticCreationOfNamespaces = true; Config.Logging = {.EnableWriteLog = ServerOptions.StructuredCacheConfig.WriteLogEnabled, .EnableAccessLog = ServerOptions.StructuredCacheConfig.AccessLogEnabled}; - Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold = ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold = ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold; + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LimitOverwrites = ServerOptions.StructuredCacheConfig.LimitOverwrites; Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes; Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds; Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds; -- cgit v1.2.3 From 72c4cc46e1bdc147e64b5efca59de7f1560d4788 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 6 Aug 2025 10:52:49 +0200 Subject: refactor blobstore (#458) - Improvement: Refactored build store cache to use existing CidStore implementation instead of implementation specific blob storage - **CAUTION** This will clear any existing cache when updating as the manifest version and storage strategy has changed - Bugfix: BuildStorage cache return "true" for metadata existance for all blobs that had payloads regardless of actual existance for metadata --- src/zenserver/zenserver.cpp | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 27ec4c690..54da51d77 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -265,10 +265,15 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen if (ServerOptions.BuildStoreConfig.Enabled) { + CidStoreConfiguration BuildCidConfig; + BuildCidConfig.RootDirectory = m_DataRoot / "builds_cas"; + m_BuildCidStore = std::make_unique(m_GcManager); + m_BuildCidStore->Initialize(BuildCidConfig); + BuildStoreConfig BuildsCfg; BuildsCfg.RootDirectory = m_DataRoot / "builds"; BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit; - m_BuildStore = std::make_unique(std::move(BuildsCfg), m_GcManager); + m_BuildStore = std::make_unique(std::move(BuildsCfg), m_GcManager, *m_BuildCidStore); } if (ServerOptions.StructuredCacheConfig.Enabled) @@ -664,6 +669,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) m_StatsReporter.AddProvider(m_CacheStore.Get()); m_StatsReporter.AddProvider(m_CidStore.get()); + m_StatsReporter.AddProvider(m_BuildCidStore.get()); } void @@ -836,6 +842,7 @@ ZenServer::Cleanup() m_BuildStoreService.reset(); m_BuildStore = {}; + m_BuildCidStore.reset(); m_StructuredCacheService.reset(); m_UpstreamService.reset(); @@ -1035,9 +1042,18 @@ ZenServer::ScrubStorage() WorkerThreadPool ThreadPool{1, "Scrub"}; ScrubContext Ctx{ThreadPool}; - m_CidStore->ScrubStorage(Ctx); - m_ProjectStore->ScrubStorage(Ctx); - m_StructuredCacheService->ScrubStorage(Ctx); + + if (m_CidStore) + m_CidStore->ScrubStorage(Ctx); + + if (m_ProjectStore) + m_ProjectStore->ScrubStorage(Ctx); + + if (m_StructuredCacheService) + m_StructuredCacheService->ScrubStorage(Ctx); + + if (m_BuildCidStore) + m_BuildCidStore->ScrubStorage(Ctx); const uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); @@ -1059,6 +1075,9 @@ ZenServer::Flush() if (m_ProjectStore) m_ProjectStore->Flush(); + + if (m_BuildCidStore) + m_BuildCidStore->Flush(); } void -- cgit v1.2.3 From 39e750f78b0944157f0179266b7593b2e492453f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 12 Aug 2025 17:36:41 +0200 Subject: add limitoverwrites option per bucket (#466) - Feature: Added global zenserver option `--cache-bucket-limit-overwrites` controlling Whether to require policy flag pattern before allowing overwrites or not. Default `false` = overwrites always allowed - Feature: Add per bucket cache configuration option `limitoverwrites` (Lua options file only) cache = { bucket = { -- This is the default for all namespaces limitoverwrites = true }, buckets = { -- Here you can add matching per bucket name (matches accross namespaces) iostorecompression = { limitoverwrites = false }, }, } --- src/zenserver/zenserver.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 6c9f2ed21..5307d58d9 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -560,7 +560,8 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) ZenCacheDiskLayer::BucketConfiguration BucketConfig = {.MaxBlockSize = ZenBucketConfig.MaxBlockSize, .PayloadAlignment = ZenBucketConfig.PayloadAlignment, .MemCacheSizeThreshold = ZenBucketConfig.MemCacheSizeThreshold, - .LargeObjectThreshold = ZenBucketConfig.LargeObjectThreshold}; + .LargeObjectThreshold = ZenBucketConfig.LargeObjectThreshold, + .LimitOverwrites = ZenBucketConfig.LimitOverwrites}; Config.NamespaceConfig.DiskLayerConfig.BucketConfigMap.insert_or_assign(BucketName, BucketConfig); } Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MaxBlockSize = ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, @@ -570,7 +571,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LargeObjectThreshold = ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, - Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LimitOverwrites = ServerOptions.StructuredCacheConfig.LimitOverwrites; + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LimitOverwrites = ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites; Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes; Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds; Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds; -- cgit v1.2.3 From 6bdaf6ad6e1308aae12845b20bf06a4406ba0c03 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 19 Aug 2025 14:03:02 +0200 Subject: zen print fixes/improvements (#469) - Improvement: `zen print` now allows output of compact binary content even if they are in non-optimal format (Unifom vs Non-Uniform arrays and objects) - Feature: `zen print` now has a `--show-type-info` option to add type information to output of compact binary content - Bugfix: Stats information for Build Store (Zen Store Cache) no longer throws exception and outputs invalid state information --- src/zenserver/zenserver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 5307d58d9..ba1413819 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -424,10 +424,10 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) if (CbValidateError ValidationResult = ValidateCompactBinary(Manifest, CbValidateMode::All); ValidationResult != CbValidateError::None) { - ZEN_WARN("Manifest validation failed: {}, state will be wiped", uint32_t(ValidationResult)); + ZEN_WARN("Manifest validation failed: {}, state will be wiped", zen::ToString(ValidationResult)); WipeState = true; - WipeReason = fmt::format("Validation of manifest at '{}' failed: {}", ManifestPath, uint32_t(ValidationResult)); + WipeReason = fmt::format("Validation of manifest at '{}' failed: {}", ManifestPath, zen::ToString(ValidationResult)); } else { -- cgit v1.2.3 From 4c05d1041461b630cd5770dae5e8d03147d5674b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 20 Aug 2025 12:33:03 +0200 Subject: per namespace/project cas prep refactor (#470) - Refactor so we can have more than one cas store for project store and cache. - Refactor `UpstreamCacheClient` so it is not tied to a specific CidStore - Refactor scrub to keep the GC interface ScrubStorage function separate from scrub accessor functions (renamed to Scrub). - Refactor storage size to keep GC interface StorageSize function separate from size accessor functions (renamed to TotalSize) - Refactor cache storage so `ZenCacheDiskLayer::CacheBucket` implements GcStorage interface rather than `ZenCacheNamespace` --- src/zenserver/zenserver.cpp | 88 ++++++++++++++++----------------------------- 1 file changed, 31 insertions(+), 57 deletions(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index ba1413819..5cab54acc 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -239,18 +239,18 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen CidStoreConfiguration Config; Config.RootDirectory = m_DataRoot / "cas"; - m_CidStore = std::make_unique(m_GcManager); - m_CidStore->Initialize(Config); + m_CidStores.insert_or_assign({}, std::make_unique(m_GcManager)); + m_CidStores.at({})->Initialize(Config); ZEN_INFO("instantiating project service"); - m_ProjectStore = new ProjectStore(*m_CidStore, + m_ProjectStore = new ProjectStore([this](std::string_view) -> CidStore& { return *m_CidStores.at({}).get(); }, m_DataRoot / "projects", m_GcManager, *m_JobQueue, *m_OpenProcessCache, ProjectStore::Configuration{}); - m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr}); + m_HttpProjectService.reset(new HttpProjectService(m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr)); if (ServerOptions.WorksSpacesConfig.Enabled) { @@ -362,17 +362,15 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_GcScheduler.Initialize(GcConfig); // Create and register admin interface last to make sure all is properly initialized - m_AdminService = - std::make_unique(m_GcScheduler, - *m_JobQueue, - m_CacheStore.Get(), - m_CidStore.get(), - m_ProjectStore, - m_BuildStore.get(), - HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, - .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", - .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}, - ServerOptions); + m_AdminService = std::make_unique( + m_GcScheduler, + *m_JobQueue, + m_CacheStore.Get(), + [this]() { Flush(); }, + HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, + .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", + .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}, + ServerOptions); m_Http->RegisterService(*m_AdminService); return EffectiveBasePort; @@ -595,7 +593,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) UpstreamOptions.ThreadCount = static_cast(UpstreamConfig.UpstreamThreadCount); } - m_UpstreamCache = CreateUpstreamCache(UpstreamOptions, *m_CacheStore, *m_CidStore); + m_UpstreamCache = CreateUpstreamCache(UpstreamOptions, *m_CacheStore); m_UpstreamService = std::make_unique(*m_UpstreamCache, *m_AuthMgr); m_UpstreamCache->Initialize(); @@ -659,19 +657,23 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) } } - m_StructuredCacheService = std::make_unique(*m_CacheStore, - *m_CidStore, - m_StatsService, - m_StatusService, - *m_UpstreamCache, - m_GcManager.GetDiskWriteBlocker(), - *m_OpenProcessCache); + m_StructuredCacheService = std::make_unique( + *m_CacheStore, + [this](std::string_view) -> CidStore& { return *m_CidStores.at({}).get(); }, + m_StatsService, + m_StatusService, + *m_UpstreamCache, + m_GcManager.GetDiskWriteBlocker(), + *m_OpenProcessCache); m_Http->RegisterService(*m_StructuredCacheService); m_Http->RegisterService(*m_UpstreamService); m_StatsReporter.AddProvider(m_CacheStore.Get()); - m_StatsReporter.AddProvider(m_CidStore.get()); + for (const auto& It : m_CidStores) + { + m_StatsReporter.AddProvider(It.second.get()); + } m_StatsReporter.AddProvider(m_BuildCidStore.get()); } @@ -857,7 +859,7 @@ ZenServer::Cleanup() m_Workspaces.reset(); m_HttpProjectService.reset(); m_ProjectStore = {}; - m_CidStore.reset(); + m_CidStores.clear(); m_AuthService.reset(); m_AuthMgr.reset(); m_Http = {}; @@ -1037,41 +1039,13 @@ ZenServer::CheckOwnerPid() } } -void -ZenServer::ScrubStorage() -{ - Stopwatch Timer; - ZEN_INFO("Storage validation STARTING"); - - WorkerThreadPool ThreadPool{1, "Scrub"}; - ScrubContext Ctx{ThreadPool}; - - if (m_CidStore) - m_CidStore->ScrubStorage(Ctx); - - if (m_ProjectStore) - m_ProjectStore->ScrubStorage(Ctx); - - if (m_StructuredCacheService) - m_StructuredCacheService->ScrubStorage(Ctx); - - if (m_BuildCidStore) - m_BuildCidStore->ScrubStorage(Ctx); - - const uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); - - ZEN_INFO("Storage validation DONE in {}, ({} in {} chunks - {})", - NiceTimeSpanMs(ElapsedTimeMs), - NiceBytes(Ctx.ScrubbedBytes()), - Ctx.ScrubbedChunks(), - NiceByteRate(Ctx.ScrubbedBytes(), ElapsedTimeMs)); -} - void ZenServer::Flush() { - if (m_CidStore) - m_CidStore->Flush(); + for (auto& It : m_CidStores) + { + It.second->Flush(); + } if (m_StructuredCacheService) m_StructuredCacheService->Flush(); -- cgit v1.2.3