add `--boost-worker-memory` option to zen builds (#639)

1 files changed, 69 insertions, 46 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 213b1bffa..e67792672 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -263,10 +263,9 @@ namespace {
 	const bool SingleThreaded = false;
 	bool	   UseSparseFiles = false;
 
-	static bool				 IsVerbose		= false;
-	static bool				 IsQuiet		= false;
-	static ProgressBar::Mode ProgressMode	= ProgressBar::Mode::Pretty;
-	static bool				 AllowFileClone = true;
+	static bool				 IsVerbose	  = false;
+	static bool				 IsQuiet	  = false;
+	static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty;
 
 #define ZEN_CONSOLE_VERBOSE(fmtstr, ...)                                   \
 	if (IsVerbose)                                                         \
@@ -284,6 +283,11 @@ namespace {
 
 	);
 
+	static uint64_t GetMaxMemoryBufferSize(bool BoostWorkerMemory)
+	{
+		return BoostWorkerMemory ? DefaultSplitAndCompressChunkSize : 1024u * 1024u;
+	}
+
 	bool IncludePath(std::span<const std::string> IncludeWildcards,
 					 std::span<const std::string> ExcludeWildcards,
 					 const std::filesystem::path& Path)
@@ -1749,8 +1753,10 @@ namespace {
 		bool					 PrimeCacheOnly					= false;
 		bool					 EnableOtherDownloadsScavenging = true;
 		bool					 EnableTargetFolderScavenging	= true;
+		bool					 AllowFileClone					= true;
 		std::vector<std::string> IncludeWildcards;
 		std::vector<std::string> ExcludeWildcards;
+		uint64_t				 MaximumInMemoryPayloadSize = 512u * 1024u;
 	};
 
 	void DownloadFolder(ThreadWorkers&				 Workers,
@@ -1906,7 +1912,7 @@ namespace {
 				LooseChunkHashes,
 				BuildsOperationUpdateFolder::Options{.IsQuiet						 = IsQuiet,
 															 .IsVerbose						 = IsVerbose,
-															 .AllowFileClone				 = AllowFileClone,
+															 .AllowFileClone				 = Options.AllowFileClone,
 															 .UseSparseFiles				 = UseSparseFiles,
 															 .SystemRootDir					 = Options.SystemRootDir,
 															 .ZenFolderPath					 = Options.ZenFolderPath,
@@ -1919,7 +1925,8 @@ namespace {
 															 .EnableTargetFolderScavenging	 = Options.EnableTargetFolderScavenging,
 															 .ValidateCompletedSequences	 = Options.PostDownloadVerify,
 															 .ExcludeFolders				 = DefaultExcludeFolders,
-															 .ExcludeExtensions				 = DefaultExcludeExtensions});
+															 .ExcludeExtensions				 = DefaultExcludeExtensions,
+															 .MaximumInMemoryPayloadSize	 = Options.MaximumInMemoryPayloadSize});
 			{
 				ProgressBar::PushLogOperation(ProgressMode, "Download");
 				auto _ = MakeGuard([]() { ProgressBar::PopLogOperation(ProgressMode); });
@@ -2387,6 +2394,13 @@ BuildsCommand::BuildsCommand()
 					   "Increase the number of worker threads - may cause computer to be less responsive",
 					   cxxopts::value(m_BoostWorkerThreads),
 					   "<boostworkers>");
+		Ops.add_option("",
+					   "",
+					   "boost-worker-memory",
+					   "Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to "
+					   "be less responsive due to high memory usage",
+					   cxxopts::value(m_BoostWorkerMemory),
+					   "<boostworkermemory>");
 	};
 
 	auto AddZenFolderOptions = [this](cxxopts::Options& Ops) {
@@ -2964,14 +2978,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	};
 	ParseOutputOptions();
 
-	AllowFileClone = m_AllowFileClone;
-
 	std::unique_ptr<AuthMgr> Auth;
-	HttpClientSettings		 ClientSettings{.LogCategory = "httpbuildsclient",
-											.AssumeHttp2 = m_AssumeHttp2,
-											.AllowResume = true,
-											.RetryCount	 = 2,
-											.Verbose	 = m_VerboseHttp};
 
 	auto CreateBuildStorage = [&](BuildStorageBase::Statistics&	 StorageStats,
 								  BuildStorageCache::Statistics& StorageCacheStats,
@@ -2981,11 +2988,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 								  bool							 BoostCacheBackgroundWorkerPool) -> StorageInstance {
 		ParseStorageOptions(RequireNamespace, RequireBucket);
 
-		HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient",
-										  .AssumeHttp2 = m_AssumeHttp2,
-										  .AllowResume = true,
-										  .RetryCount  = 2,
-										  .Verbose	   = m_VerboseHttp};
+		HttpClientSettings ClientSettings{.LogCategory				   = "httpbuildsclient",
+										  .AssumeHttp2				   = m_AssumeHttp2,
+										  .AllowResume				   = true,
+										  .RetryCount				   = 2,
+										  .Verbose					   = m_VerboseHttp,
+										  .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(m_BoostWorkerMemory)};
 
 		std::unique_ptr<AuthMgr> Auth;
 
@@ -3037,15 +3045,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 				if (!ResolveRes.CacheUrl.empty())
 				{
-					Result.CacheHttp = std::make_unique<HttpClient>(ResolveRes.CacheUrl,
-																	HttpClientSettings{.LogCategory	   = "httpcacheclient",
-																					   .ConnectTimeout = std::chrono::milliseconds{3000},
-																					   .Timeout		   = std::chrono::milliseconds{30000},
-																					   .AssumeHttp2	   = ResolveRes.CacheAssumeHttp2,
-																					   .AllowResume	   = true,
-																					   .RetryCount	   = 0,
-																					   .Verbose		   = m_VerboseHttp},
-																	[]() { return AbortFlag.load(); });
+					Result.CacheHttp = std::make_unique<HttpClient>(
+						ResolveRes.CacheUrl,
+						HttpClientSettings{.LogCategory					= "httpcacheclient",
+										   .ConnectTimeout				= std::chrono::milliseconds{3000},
+										   .Timeout						= std::chrono::milliseconds{30000},
+										   .AssumeHttp2					= ResolveRes.CacheAssumeHttp2,
+										   .AllowResume					= true,
+										   .RetryCount					= 0,
+										   .Verbose						= m_VerboseHttp,
+										   .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(m_BoostWorkerMemory)},
+						[]() { return AbortFlag.load(); });
 					Result.BuildCacheStorage =
 						CreateZenBuildStorageCache(*Result.CacheHttp,
 												   StorageCacheStats,
@@ -3081,15 +3091,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 			if (!m_ZenCacheHost.empty())
 			{
-				Result.CacheHttp = std::make_unique<HttpClient>(m_ZenCacheHost,
-																HttpClientSettings{.LogCategory	   = "httpcacheclient",
-																				   .ConnectTimeout = std::chrono::milliseconds{3000},
-																				   .Timeout		   = std::chrono::milliseconds{30000},
-																				   .AssumeHttp2	   = m_AssumeHttp2,
-																				   .AllowResume	   = true,
-																				   .RetryCount	   = 0,
-																				   .Verbose		   = m_VerboseHttp},
-																[]() { return AbortFlag.load(); });
+				Result.CacheHttp = std::make_unique<HttpClient>(
+					m_ZenCacheHost,
+					HttpClientSettings{.LogCategory					= "httpcacheclient",
+									   .ConnectTimeout				= std::chrono::milliseconds{3000},
+									   .Timeout						= std::chrono::milliseconds{30000},
+									   .AssumeHttp2					= m_AssumeHttp2,
+									   .AllowResume					= true,
+									   .RetryCount					= 0,
+									   .Verbose						= m_VerboseHttp,
+									   .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(m_BoostWorkerMemory)},
+					[]() { return AbortFlag.load(); });
 				Result.BuildCacheStorage =
 					CreateZenBuildStorageCache(*Result.CacheHttp,
 											   StorageCacheStats,
@@ -3719,8 +3731,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PrimeCacheOnly				   = m_PrimeCacheOnly,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging && !m_Force,
 									   .EnableTargetFolderScavenging   = !m_Force,
+									   .AllowFileClone				   = m_AllowFileClone,
 									   .IncludeWildcards			   = IncludeWildcards,
-									   .ExcludeWildcards			   = ExcludeWildcards});
+									   .ExcludeWildcards			   = ExcludeWildcards,
+									   .MaximumInMemoryPayloadSize	   = GetMaxMemoryBufferSize(m_BoostWorkerMemory)});
 
 		if (AbortFlag)
 		{
@@ -4037,7 +4051,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 										   .PostDownloadVerify			   = true,
 										   .PrimeCacheOnly				   = false,
 										   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-										   .EnableTargetFolderScavenging   = false});
+										   .EnableTargetFolderScavenging   = false,
+										   .AllowFileClone				   = m_AllowFileClone});
 			if (AbortFlag)
 			{
 				throw std::runtime_error("Multitest aborted");
@@ -4221,7 +4236,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = false});
+									   .EnableTargetFolderScavenging   = false,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Download build)");
@@ -4243,7 +4259,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = true});
+									   .EnableTargetFolderScavenging   = true,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Re-download identical target)");
@@ -4357,7 +4374,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = true});
+									   .EnableTargetFolderScavenging   = true,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Re-download scrambled target)");
@@ -4412,7 +4430,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = true});
+									   .EnableTargetFolderScavenging   = true,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Download original)");
@@ -4433,7 +4452,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = true});
+									   .EnableTargetFolderScavenging   = true,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Download scrambled)");
@@ -4454,7 +4474,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = true});
+									   .EnableTargetFolderScavenging   = true,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Re-download scrambled)");
@@ -4475,7 +4496,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = true});
+									   .EnableTargetFolderScavenging   = true,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Download original)");
@@ -4496,7 +4518,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = true});
+									   .EnableTargetFolderScavenging   = true,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Download original)");