From 28bc5ebf05984385cc0567c89b1d8e7a541ebef8 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 26 Mar 2025 17:06:23 +0100 Subject: zen build cache service (#318) - **EXPERIMENTAL** `zen builds` - Feature: `--zen-cache-host` option for `upload` and `download` operations to use a zenserver host `/builds` endpoint for storing build blob and blob metadata - Feature: New `/builds` endpoint for caching build blobs and blob metadata - `/builds/{namespace}/{bucket}/{buildid}/blobs/{hash}` `GET` and `PUT` method for storing and fetching blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata` `POST` method for storing metadata about blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata` `POST` method for fetching metadata about blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/exists` `POST` method for checking existance of blobs --- src/zenserver/zenserver.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index f84bc0b00..03e269d49 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -262,6 +263,13 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen *m_Workspaces)); } + if (ServerOptions.BuildStoreConfig.Enabled) + { + BuildStoreConfig ObjCfg; + ObjCfg.RootDirectory = m_DataRoot / "builds"; + m_BuildStore = std::make_unique(std::move(ObjCfg), m_GcManager); + } + if (ServerOptions.StructuredCacheConfig.Enabled) { InitializeStructuredCache(ServerOptions); @@ -310,6 +318,12 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_Http->RegisterService(*m_ObjStoreService); } + if (ServerOptions.BuildStoreConfig.Enabled) + { + m_BuildStoreService = std::make_unique(m_StatsService, *m_BuildStore); + m_Http->RegisterService(*m_BuildStoreService); + } + #if ZEN_WITH_VFS m_VfsService = std::make_unique(); m_VfsService->AddService(Ref(m_ProjectStore)); @@ -327,6 +341,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen .Interval = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds), .MaxCacheDuration = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds), .MaxProjectStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds), + .MaxBuildStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds), .CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects, .Enabled = ServerOptions.GcConfig.Enabled, .DiskReserveSize = ServerOptions.GcConfig.DiskReserveSize, @@ -347,6 +362,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_CacheStore.Get(), m_CidStore.get(), m_ProjectStore, + m_BuildStore.get(), HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}, @@ -801,6 +817,9 @@ ZenServer::Cleanup() m_ObjStoreService.reset(); m_FrontendService.reset(); + m_BuildStoreService.reset(); + m_BuildStore = {}; + m_StructuredCacheService.reset(); m_UpstreamService.reset(); m_UpstreamCache.reset(); -- cgit v1.2.3 From fd2efb5af872a357dbc0f729f4101a330dcb4fda Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 31 Mar 2025 10:24:39 +0200 Subject: long filename support (#330) - Bugfix: Long file paths now works correctly on Windows --- src/zenserver/zenserver.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 03e269d49..45c91d691 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -434,7 +434,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION) { std::filesystem::path ManifestSkipSchemaChangePath = m_DataRoot / "root_manifest.ignore_schema_mismatch"; - if (ManifestVersion != 0 && std::filesystem::is_regular_file(ManifestSkipSchemaChangePath)) + if (ManifestVersion != 0 && IsFile(ManifestSkipSchemaChangePath)) { ZEN_INFO( "Schema version {} found in '{}' does not match {}, ignoring mismatch due to existance of '{}' and updating " @@ -483,7 +483,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) { ZEN_INFO("Deleting '{}'", DirEntry.path()); - std::filesystem::remove_all(DirEntry.path(), Ec); + DeleteDirectories(DirEntry.path(), Ec); if (Ec) { @@ -914,7 +914,7 @@ ZenServer::CheckStateMarker() std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker"; try { - if (!std::filesystem::exists(StateMarkerPath)) + if (!IsFile(StateMarkerPath)) { ZEN_WARN("state marker at {} has been deleted, exiting", StateMarkerPath); RequestExit(1); -- cgit v1.2.3 From 571aad8c5d2ed9d975ba2fed2890116e3c8bd6ea Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 1 Apr 2025 18:27:05 +0200 Subject: builds url discovery (#334) - Feature: Added `--host` option to use Jupiters list of cloud host and zen servers to resolve best hosts - Feature: Use local zenserver as builds cache if it has the `builds` service enabled and `--cloud-discovery-host` is provided and no remote zenserver cache hosts can be found - Improvement: Added `--override-host` option as a replacement for `--url` (`--url` still works, but `--override-host` is preferred) --- src/zenserver/zenserver.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 45c91d691..c7cb2ba6e 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -251,13 +251,14 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen *m_JobQueue, *m_OpenProcessCache, ProjectStore::Configuration{}); - m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatsService, *m_AuthMgr}); + m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr}); if (ServerOptions.WorksSpacesConfig.Enabled) { m_Workspaces.reset(new Workspaces()); m_HttpWorkspacesService.reset( - new HttpWorkspacesService(m_StatsService, + new HttpWorkspacesService(m_StatusService, + m_StatsService, {.SystemRootDir = ServerOptions.SystemRootDir, .AllowConfigurationChanges = ServerOptions.WorksSpacesConfig.AllowConfigurationChanges}, *m_Workspaces)); @@ -295,7 +296,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_Http->RegisterService(*m_HttpWorkspacesService); } - m_FrontendService = std::make_unique(m_ContentRoot); + m_FrontendService = std::make_unique(m_ContentRoot, m_StatusService); if (m_FrontendService) { @@ -314,18 +315,18 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen ObjCfg.Buckets.push_back(std::move(NewBucket)); } - m_ObjStoreService = std::make_unique(std::move(ObjCfg)); + m_ObjStoreService = std::make_unique(m_StatusService, std::move(ObjCfg)); m_Http->RegisterService(*m_ObjStoreService); } if (ServerOptions.BuildStoreConfig.Enabled) { - m_BuildStoreService = std::make_unique(m_StatsService, *m_BuildStore); + m_BuildStoreService = std::make_unique(m_StatusService, m_StatsService, *m_BuildStore); m_Http->RegisterService(*m_BuildStoreService); } #if ZEN_WITH_VFS - m_VfsService = std::make_unique(); + m_VfsService = std::make_unique(m_StatusService); m_VfsService->AddService(Ref(m_ProjectStore)); m_VfsService->AddService(Ref(m_CacheStore)); m_Http->RegisterService(*m_VfsService); -- cgit v1.2.3 From 6d22f7e086646dcfb0d7f5e2c784acba53235c35 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 4 Apr 2025 13:07:42 +0200 Subject: blobstore size limit (#342) - Feature: zenserver option `--buildstore-disksizelimit` to set an soft upper limit for build storage data. Defaults to 1TB. --- src/zenserver/zenserver.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index c7cb2ba6e..3f2f01d5a 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -266,9 +266,10 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen if (ServerOptions.BuildStoreConfig.Enabled) { - BuildStoreConfig ObjCfg; - ObjCfg.RootDirectory = m_DataRoot / "builds"; - m_BuildStore = std::make_unique(std::move(ObjCfg), m_GcManager); + BuildStoreConfig BuildsCfg; + BuildsCfg.RootDirectory = m_DataRoot / "builds"; + BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit; + m_BuildStore = std::make_unique(std::move(BuildsCfg), m_GcManager); } if (ServerOptions.StructuredCacheConfig.Enabled) -- cgit v1.2.3 From 6d9ff7e404a22ed1cc7e529cfa77ef7d593d9547 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 6 May 2025 16:50:57 +0200 Subject: add sentry for zen command (#373) * refactor sentry integration and add to zen command line tool * move add_ldflags("-framework Security") --- src/zenserver/zenserver.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 3f2f01d5a..366944ffc 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -2,8 +2,6 @@ #include "zenserver.h" -#include "sentryintegration.h" - #include #include #include @@ -16,6 +14,7 @@ #include #include #include +#include #include #include #include -- cgit v1.2.3 From 00dc4e3d0976e0f0cbd8a09a7693bad31b8db511 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 12 May 2025 10:22:17 +0200 Subject: enable per bucket config (#388) Feature: Add per bucket cache configuration (Lua options file only) Improvement: --cache-memlayer-sizethreshold is now deprecated and has a new name: --cache-bucket-memlayer-sizethreshold to line up with per cache bucket configuration --- src/zenserver/zenserver.cpp | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 366944ffc..1ad94ed63 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -547,10 +547,26 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) Config.AllowAutomaticCreationOfNamespaces = true; Config.Logging = {.EnableWriteLog = ServerOptions.StructuredCacheConfig.WriteLogEnabled, .EnableAccessLog = ServerOptions.StructuredCacheConfig.AccessLogEnabled}; - Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold = ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold, - Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes; - Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds; - Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds; + for (const auto& It : ServerOptions.StructuredCacheConfig.PerBucketConfigs) + { + const std::string& BucketName = It.first; + const ZenStructuredCacheBucketConfig& ZenBucketConfig = It.second; + ZenCacheDiskLayer::BucketConfiguration BucketConfig = {.MaxBlockSize = ZenBucketConfig.MaxBlockSize, + .PayloadAlignment = ZenBucketConfig.PayloadAlignment, + .MemCacheSizeThreshold = ZenBucketConfig.MemCacheSizeThreshold, + .LargeObjectThreshold = ZenBucketConfig.LargeObjectThreshold}; + Config.NamespaceConfig.DiskLayerConfig.BucketConfigMap.insert_or_assign(BucketName, BucketConfig); + } + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MaxBlockSize = ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.PayloadAlignment = + ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold = + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LargeObjectThreshold = + ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, + Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes; + Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds; + Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds; if (ServerOptions.IsDedicated) { -- cgit v1.2.3 From cadaad632b05e231dadfc977819189d9c486c74b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 Jun 2025 13:06:40 +0200 Subject: add sentry configurations options for debug/environment add env-variable parsing for sentry option --- src/zenserver/zenserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 1ad94ed63..27ec4c690 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -137,7 +137,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen return -1; } - m_UseSentry = ServerOptions.NoSentry == false; + m_UseSentry = ServerOptions.SentryConfig.Disable == false; m_ServerEntry = ServerEntry; m_DebugOptionForcedCrash = ServerOptions.ShouldCrash; m_IsPowerCycle = ServerOptions.IsPowerCycle; -- cgit v1.2.3 From f696e52d150ae284e26de5bdd78d1b1edf914314 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 Jun 2025 13:08:59 +0200 Subject: revert 61b4a88f and cadaad63 --- src/zenserver/zenserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 27ec4c690..1ad94ed63 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -137,7 +137,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen return -1; } - m_UseSentry = ServerOptions.SentryConfig.Disable == false; + m_UseSentry = ServerOptions.NoSentry == false; m_ServerEntry = ServerEntry; m_DebugOptionForcedCrash = ServerOptions.ShouldCrash; m_IsPowerCycle = ServerOptions.IsPowerCycle; -- cgit v1.2.3 From fdad92dddba8047930c4e7496a7f412d760c312e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 Jun 2025 09:30:54 +0200 Subject: sentry config (#430) - Feature: Added `--sentry-environment` to `zen` and `zenserver` - Feature: Added `--sentry-debug` to `zen` and `zenserver` - Feature: Added environment variable parsing for the following options: - `UE_ZEN_SENTRY_ENABLED`: `--no-sentry` (inverted) - `UE_ZEN_SENTRY_DEBUG`: `--sentry-debug` - `UE_ZEN_SENTRY_ALLOWPERSONALINFO`: `--sentry-allow-personal-info` - `UE_ZEN_SENTRY_DSN`: `--sentry-dsn` - `UE_ZEN_SENTRY_ENVIRONMENT`: `--sentry-environment` --- src/zenserver/zenserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 1ad94ed63..27ec4c690 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -137,7 +137,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen return -1; } - m_UseSentry = ServerOptions.NoSentry == false; + m_UseSentry = ServerOptions.SentryConfig.Disable == false; m_ServerEntry = ServerEntry; m_DebugOptionForcedCrash = ServerOptions.ShouldCrash; m_IsPowerCycle = ServerOptions.IsPowerCycle; -- cgit v1.2.3