add `--boost-worker-memory` option to zen builds (#639)

6 files changed, 82 insertions, 56 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 213b1bffa..e67792672 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -263,10 +263,9 @@ namespace {
 	const bool SingleThreaded = false;
 	bool	   UseSparseFiles = false;
 
-	static bool				 IsVerbose		= false;
-	static bool				 IsQuiet		= false;
-	static ProgressBar::Mode ProgressMode	= ProgressBar::Mode::Pretty;
-	static bool				 AllowFileClone = true;
+	static bool				 IsVerbose	  = false;
+	static bool				 IsQuiet	  = false;
+	static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty;
 
 #define ZEN_CONSOLE_VERBOSE(fmtstr, ...)                                   \
 	if (IsVerbose)                                                         \
@@ -284,6 +283,11 @@ namespace {
 
 	);
 
+	static uint64_t GetMaxMemoryBufferSize(bool BoostWorkerMemory)
+	{
+		return BoostWorkerMemory ? DefaultSplitAndCompressChunkSize : 1024u * 1024u;
+	}
+
 	bool IncludePath(std::span<const std::string> IncludeWildcards,
 					 std::span<const std::string> ExcludeWildcards,
 					 const std::filesystem::path& Path)
@@ -1749,8 +1753,10 @@ namespace {
 		bool					 PrimeCacheOnly					= false;
 		bool					 EnableOtherDownloadsScavenging = true;
 		bool					 EnableTargetFolderScavenging	= true;
+		bool					 AllowFileClone					= true;
 		std::vector<std::string> IncludeWildcards;
 		std::vector<std::string> ExcludeWildcards;
+		uint64_t				 MaximumInMemoryPayloadSize = 512u * 1024u;
 	};
 
 	void DownloadFolder(ThreadWorkers&				 Workers,
@@ -1906,7 +1912,7 @@ namespace {
 				LooseChunkHashes,
 				BuildsOperationUpdateFolder::Options{.IsQuiet						 = IsQuiet,
 															 .IsVerbose						 = IsVerbose,
-															 .AllowFileClone				 = AllowFileClone,
+															 .AllowFileClone				 = Options.AllowFileClone,
 															 .UseSparseFiles				 = UseSparseFiles,
 															 .SystemRootDir					 = Options.SystemRootDir,
 															 .ZenFolderPath					 = Options.ZenFolderPath,
@@ -1919,7 +1925,8 @@ namespace {
 															 .EnableTargetFolderScavenging	 = Options.EnableTargetFolderScavenging,
 															 .ValidateCompletedSequences	 = Options.PostDownloadVerify,
 															 .ExcludeFolders				 = DefaultExcludeFolders,
-															 .ExcludeExtensions				 = DefaultExcludeExtensions});
+															 .ExcludeExtensions				 = DefaultExcludeExtensions,
+															 .MaximumInMemoryPayloadSize	 = Options.MaximumInMemoryPayloadSize});
 			{
 				ProgressBar::PushLogOperation(ProgressMode, "Download");
 				auto _ = MakeGuard([]() { ProgressBar::PopLogOperation(ProgressMode); });
@@ -2387,6 +2394,13 @@ BuildsCommand::BuildsCommand()
 					   "Increase the number of worker threads - may cause computer to be less responsive",
 					   cxxopts::value(m_BoostWorkerThreads),
 					   "<boostworkers>");
+		Ops.add_option("",
+					   "",
+					   "boost-worker-memory",
+					   "Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to "
+					   "be less responsive due to high memory usage",
+					   cxxopts::value(m_BoostWorkerMemory),
+					   "<boostworkermemory>");
 	};
 
 	auto AddZenFolderOptions = [this](cxxopts::Options& Ops) {
@@ -2964,14 +2978,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	};
 	ParseOutputOptions();
 
-	AllowFileClone = m_AllowFileClone;
-
 	std::unique_ptr<AuthMgr> Auth;
-	HttpClientSettings		 ClientSettings{.LogCategory = "httpbuildsclient",
-											.AssumeHttp2 = m_AssumeHttp2,
-											.AllowResume = true,
-											.RetryCount	 = 2,
-											.Verbose	 = m_VerboseHttp};
 
 	auto CreateBuildStorage = [&](BuildStorageBase::Statistics&	 StorageStats,
 								  BuildStorageCache::Statistics& StorageCacheStats,
@@ -2981,11 +2988,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 								  bool							 BoostCacheBackgroundWorkerPool) -> StorageInstance {
 		ParseStorageOptions(RequireNamespace, RequireBucket);
 
-		HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient",
-										  .AssumeHttp2 = m_AssumeHttp2,
-										  .AllowResume = true,
-										  .RetryCount  = 2,
-										  .Verbose	   = m_VerboseHttp};
+		HttpClientSettings ClientSettings{.LogCategory				   = "httpbuildsclient",
+										  .AssumeHttp2				   = m_AssumeHttp2,
+										  .AllowResume				   = true,
+										  .RetryCount				   = 2,
+										  .Verbose					   = m_VerboseHttp,
+										  .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(m_BoostWorkerMemory)};
 
 		std::unique_ptr<AuthMgr> Auth;
 
@@ -3037,15 +3045,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 				if (!ResolveRes.CacheUrl.empty())
 				{
-					Result.CacheHttp = std::make_unique<HttpClient>(ResolveRes.CacheUrl,
-																	HttpClientSettings{.LogCategory	   = "httpcacheclient",
-																					   .ConnectTimeout = std::chrono::milliseconds{3000},
-																					   .Timeout		   = std::chrono::milliseconds{30000},
-																					   .AssumeHttp2	   = ResolveRes.CacheAssumeHttp2,
-																					   .AllowResume	   = true,
-																					   .RetryCount	   = 0,
-																					   .Verbose		   = m_VerboseHttp},
-																	[]() { return AbortFlag.load(); });
+					Result.CacheHttp = std::make_unique<HttpClient>(
+						ResolveRes.CacheUrl,
+						HttpClientSettings{.LogCategory					= "httpcacheclient",
+										   .ConnectTimeout				= std::chrono::milliseconds{3000},
+										   .Timeout						= std::chrono::milliseconds{30000},
+										   .AssumeHttp2					= ResolveRes.CacheAssumeHttp2,
+										   .AllowResume					= true,
+										   .RetryCount					= 0,
+										   .Verbose						= m_VerboseHttp,
+										   .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(m_BoostWorkerMemory)},
+						[]() { return AbortFlag.load(); });
 					Result.BuildCacheStorage =
 						CreateZenBuildStorageCache(*Result.CacheHttp,
 												   StorageCacheStats,
@@ -3081,15 +3091,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 			if (!m_ZenCacheHost.empty())
 			{
-				Result.CacheHttp = std::make_unique<HttpClient>(m_ZenCacheHost,
-																HttpClientSettings{.LogCategory	   = "httpcacheclient",
-																				   .ConnectTimeout = std::chrono::milliseconds{3000},
-																				   .Timeout		   = std::chrono::milliseconds{30000},
-																				   .AssumeHttp2	   = m_AssumeHttp2,
-																				   .AllowResume	   = true,
-																				   .RetryCount	   = 0,
-																				   .Verbose		   = m_VerboseHttp},
-																[]() { return AbortFlag.load(); });
+				Result.CacheHttp = std::make_unique<HttpClient>(
+					m_ZenCacheHost,
+					HttpClientSettings{.LogCategory					= "httpcacheclient",
+									   .ConnectTimeout				= std::chrono::milliseconds{3000},
+									   .Timeout						= std::chrono::milliseconds{30000},
+									   .AssumeHttp2					= m_AssumeHttp2,
+									   .AllowResume					= true,
+									   .RetryCount					= 0,
+									   .Verbose						= m_VerboseHttp,
+									   .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(m_BoostWorkerMemory)},
+					[]() { return AbortFlag.load(); });
 				Result.BuildCacheStorage =
 					CreateZenBuildStorageCache(*Result.CacheHttp,
 											   StorageCacheStats,
@@ -3719,8 +3731,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PrimeCacheOnly				   = m_PrimeCacheOnly,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging && !m_Force,
 									   .EnableTargetFolderScavenging   = !m_Force,
+									   .AllowFileClone				   = m_AllowFileClone,
 									   .IncludeWildcards			   = IncludeWildcards,
-									   .ExcludeWildcards			   = ExcludeWildcards});
+									   .ExcludeWildcards			   = ExcludeWildcards,
+									   .MaximumInMemoryPayloadSize	   = GetMaxMemoryBufferSize(m_BoostWorkerMemory)});
 
 		if (AbortFlag)
 		{
@@ -4037,7 +4051,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 										   .PostDownloadVerify			   = true,
 										   .PrimeCacheOnly				   = false,
 										   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-										   .EnableTargetFolderScavenging   = false});
+										   .EnableTargetFolderScavenging   = false,
+										   .AllowFileClone				   = m_AllowFileClone});
 			if (AbortFlag)
 			{
 				throw std::runtime_error("Multitest aborted");
@@ -4221,7 +4236,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = false});
+									   .EnableTargetFolderScavenging   = false,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Download build)");
@@ -4243,7 +4259,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = true});
+									   .EnableTargetFolderScavenging   = true,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Re-download identical target)");
@@ -4357,7 +4374,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = true});
+									   .EnableTargetFolderScavenging   = true,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Re-download scrambled target)");
@@ -4412,7 +4430,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = true});
+									   .EnableTargetFolderScavenging   = true,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Download original)");
@@ -4433,7 +4452,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = true});
+									   .EnableTargetFolderScavenging   = true,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Download scrambled)");
@@ -4454,7 +4474,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = true});
+									   .EnableTargetFolderScavenging   = true,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Re-download scrambled)");
@@ -4475,7 +4496,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = true});
+									   .EnableTargetFolderScavenging   = true,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Download original)");
@@ -4496,7 +4518,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .PostDownloadVerify			   = true,
 									   .PrimeCacheOnly				   = false,
 									   .EnableOtherDownloadsScavenging = m_EnableScavenging,
-									   .EnableTargetFolderScavenging   = true});
+									   .EnableTargetFolderScavenging   = true,
+									   .AllowFileClone				   = m_AllowFileClone});
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Download original)");
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index d05d23760..964c369c5 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -33,6 +33,7 @@ private:
 	bool m_LogProgress		  = false;
 	bool m_Verbose			  = false;
 	bool m_BoostWorkerThreads = false;
+	bool m_BoostWorkerMemory  = false;
 	bool m_UseSparseFiles	  = true;
 	bool m_Quiet			  = false;
 	bool m_AllowFileClone	  = true;
diff --git a/src/zenhttp/clients/httpclientcpr.cpp b/src/zenhttp/clients/httpclientcpr.cpp
index 333fd20b0..66a4ce16f 100644
--- a/src/zenhttp/clients/httpclientcpr.cpp
+++ b/src/zenhttp/clients/httpclientcpr.cpp
@@ -979,7 +979,7 @@ CprHttpClient::Download(std::string_view Url, const std::filesystem::path& TempF
 						   std::optional<size_t> ContentLength = ParseInt<size_t>(Header.second);
 						   if (ContentLength.has_value())
 						   {
-							   if (ContentLength.value() > 1024 * 1024)
+							   if (ContentLength.value() > m_ConnectionSettings.MaximumInMemoryDownloadSize)
 							   {
 								   PayloadFile = std::make_unique<detail::TempPayloadFile>();
 								   std::error_code Ec = PayloadFile->Open(TempFolderPath, ContentLength.value());
diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h
index c9b6d4dee..9a9b74d72 100644
--- a/src/zenhttp/include/zenhttp/httpclient.h
+++ b/src/zenhttp/include/zenhttp/httpclient.h
@@ -52,11 +52,12 @@ struct HttpClientSettings
 	std::chrono::milliseconds							  ConnectTimeout{3000};
 	std::chrono::milliseconds							  Timeout{};
 	std::optional<std::function<HttpClientAccessToken()>> AccessTokenProvider;
-	bool												  AssumeHttp2 = false;
-	bool												  AllowResume = false;
-	uint8_t												  RetryCount  = 0;
-	Oid													  SessionId	  = Oid::Zero;
-	bool												  Verbose	  = false;
+	bool												  AssumeHttp2				  = false;
+	bool												  AllowResume				  = false;
+	uint8_t												  RetryCount				  = 0;
+	Oid													  SessionId					  = Oid::Zero;
+	bool												  Verbose					  = false;
+	uint64_t											  MaximumInMemoryDownloadSize = 1024u * 1024u;
 };
 
 class HttpClientError : public std::runtime_error
diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp
index fb1b3018a..ca54d2ce7 100644
--- a/src/zenremotestore/builds/buildstorageoperations.cpp
+++ b/src/zenremotestore/builds/buildstorageoperations.cpp
@@ -1812,7 +1812,7 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
 										}
 									}
 
-									if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u))
+									if (BlockChunkPath.empty() && (BlockSize > m_Options.MaximumInMemoryPayloadSize))
 									{
 										ZEN_TRACE_CPU("WriteTempFullBlock");
 										// Could not be moved and rather large, lets store it on disk
@@ -3707,7 +3707,7 @@ BuildsOperationUpdateFolder::DownloadPartialBlock(
 			}
 		}
 
-		if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u))
+		if (BlockChunkPath.empty() && (BlockSize > m_Options.MaximumInMemoryPayloadSize))
 		{
 			ZEN_TRACE_CPU("WriteTempPartialBlock");
 			// Could not be moved and rather large, lets store it on disk
@@ -3847,7 +3847,7 @@ BuildsOperationUpdateFolder::WriteLocalChunkToCache(CloneQueryInterface*					  C
 					break;
 				}
 				const uint64_t NextChunkLength = m_RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex];
-				if (ReadLength + NextChunkLength > 512u * 1024u)
+				if (ReadLength + NextChunkLength > m_Options.MaximumInMemoryPayloadSize)
 				{
 					break;
 				}
@@ -4438,7 +4438,7 @@ BuildsOperationUpdateFolder::AsyncWriteDownloadedChunk(const std::filesystem::pa
 		}
 	}
 
-	if (CompressedChunkPath.empty() && (Size > 512u * 1024u))
+	if (CompressedChunkPath.empty() && (Size > m_Options.MaximumInMemoryPayloadSize))
 	{
 		ZEN_TRACE_CPU("WriteTempChunk");
 		// Could not be moved and rather large, lets store it on disk
diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
index c3d8b4c27..6a6b8e616 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
@@ -142,6 +142,7 @@ public:
 		bool					 ValidateCompletedSequences		= true;
 		std::vector<std::string> ExcludeFolders;
 		std::vector<std::string> ExcludeExtensions;
+		uint64_t				 MaximumInMemoryPayloadSize = 512u * 1024u;
 	};
 
 	BuildsOperationUpdateFolder(OperationLogOutput&						  OperationLogOutput,