From ab68e79d87bd00682fd4a68d3c969767cce588bf Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 16 Apr 2026 20:22:46 +0200 Subject: add --buildstore-disksizelimit-percent to zenstorageserver (#966) - Improvement: Add option `--buildstore-disksizelimit-percent` - Max percentage of total drive capacity (of --data-dir drive) for build storage. When combined with `--buildstore-disksizelimit`, the lower value wins. --- CHANGELOG.md | 1 + src/zenserver/storage/storageconfig.cpp | 16 ++++++++++++++++ src/zenserver/storage/storageconfig.h | 5 +++-- src/zenserver/storage/zenstorageserver.cpp | 26 +++++++++++++++++++++++++- 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f0bca49c..0fd880bb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ - Improvement: Session service and endpoint moved from storage server to base server class; now available in hub, compute, and proxy modes - Improvement: HTTP client Content-Type in additional headers now overrides the payload content type - Improvement: Dashboard copy-to-clipboard buttons on hub module IDs and ports, compute worker/action IDs and queue tokens, orchestrator client IDs, cache namespace names and directories, project names and directories, and workspace IDs +- Improvement: `--buildstore-disksizelimit-percent` - Max percentage of total drive capacity (of --data-dir drive) for build storage. When combined with `--buildstore-disksizelimit`, the lower value wins. - Bugfix: OAuth client credentials token request now sends correct `application/x-www-form-urlencoded` content type - Bugfix: `builds download` with cache upload enabled no longer holds downloaded blobs in memory when boost-worker-memory is active; blobs are written to disk before upload - Bugfix: Removed obsolete `--cache-prime-only` flag from `builds download` diff --git a/src/zenserver/storage/storageconfig.cpp b/src/zenserver/storage/storageconfig.cpp index b615af280..bb4f053e4 100644 --- a/src/zenserver/storage/storageconfig.cpp +++ b/src/zenserver/storage/storageconfig.cpp @@ -57,6 +57,12 @@ ZenStorageServerConfigurator::ValidateOptions() ZEN_WARN("'--gc-v2=false' is deprecated, reverting to '--gc-v2=true'"); ServerOptions.GcConfig.UseGCV2 = true; } + if (ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent > 100) + { + throw OptionParseException(fmt::format("'--buildstore-disksizelimit-percent' ('{}') is invalid, must be between 1 and 100.", + ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent), + {}); + } } class ZenStructuredCacheBucketsConfigOption : public LuaConfig::OptionValue @@ -382,6 +388,9 @@ ZenStorageServerConfigurator::AddConfigOptions(LuaConfig::Options& LuaOptions) ////// buildsstore LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv); LuaOptions.AddOption("server.buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit"); + LuaOptions.AddOption("server.buildstore.disksizelimitpercent"sv, + ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent, + "buildstore-disksizelimit-percent"); ////// cache LuaOptions.AddOption("cache.enable"sv, ServerOptions.StructuredCacheConfig.Enabled); @@ -1035,6 +1044,13 @@ ZenStorageServerCmdLineOptions::AddBuildStoreOptions(cxxopts::Options& options, "Max number of bytes before build store entries get evicted. Default set to 1099511627776 (1TB week)", cxxopts::value(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit)->default_value("1099511627776"), ""); + options.add_option("buildstore", + "", + "buildstore-disksizelimit-percent", + "Max percentage (1-100) of total drive capacity (of --data-dir drive) before build store entries get evicted. " + "0 (default) disables this limit. When combined with --buildstore-disksizelimit, the lower value wins.", + cxxopts::value(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent)->default_value("0"), + ""); } void diff --git a/src/zenserver/storage/storageconfig.h b/src/zenserver/storage/storageconfig.h index 18af4f096..fec8fd70b 100644 --- a/src/zenserver/storage/storageconfig.h +++ b/src/zenserver/storage/storageconfig.h @@ -135,8 +135,9 @@ struct ZenProjectStoreConfig struct ZenBuildStoreConfig { - bool Enabled = false; - uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB + bool Enabled = false; + uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB + uint32_t MaxDiskSpaceLimitPercent = 0; }; struct ZenWorkspacesConfig diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp index 34f413639..e53678005 100644 --- a/src/zenserver/storage/zenstorageserver.cpp +++ b/src/zenserver/storage/zenstorageserver.cpp @@ -266,7 +266,31 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions BuildStoreConfig BuildsCfg; BuildsCfg.RootDirectory = m_DataRoot / "builds"; BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit; - m_BuildStore = std::make_unique(std::move(BuildsCfg), m_GcManager, *m_BuildCidStore); + + if (ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent > 0) + { + DiskSpace Space; + if (DiskSpaceInfo(m_DataRoot, Space) && Space.Total > 0) + { + uint64_t PercentLimit = Space.Total * ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent / 100; + BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit > 0 + ? std::min(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, PercentLimit) + : PercentLimit; + ZEN_INFO("buildstore disk limit: {}% of {} = {} (effective limit: {})", + ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent, + NiceBytes(Space.Total), + NiceBytes(PercentLimit), + NiceBytes(BuildsCfg.MaxDiskSpaceLimit)); + } + else + { + ZEN_WARN("buildstore-disksizelimit-percent: failed to query disk space for {}, using absolute limit {}", + m_DataRoot.string(), + NiceBytes(BuildsCfg.MaxDiskSpaceLimit)); + } + } + + m_BuildStore = std::make_unique(std::move(BuildsCfg), m_GcManager, *m_BuildCidStore); } if (ServerOptions.StructuredCacheConfig.Enabled) -- cgit v1.2.3