control cache upload (#646)

* add option to enable/disable upload to builds cache

11 files changed, 65 insertions, 24 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index ab991fe6a..16b623453 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -465,7 +465,8 @@ namespace {
 					  bool						   AllowMultiparts,
 					  const CbObject&			   MetaData,
 					  bool						   CreateBuild,
-					  bool						   IgnoreExistingBlocks)
+					  bool						   IgnoreExistingBlocks,
+					  bool						   UploadToZenCache)
 	{
 		ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder");
 		{
@@ -497,7 +498,8 @@ namespace {
 													 .ExcludeFolders			= DefaultExcludeFolders,
 													 .ExcludeExtensions			= DefaultExcludeExtensions,
 													 .ZenExcludeManifestName	= ZenExcludeManifestName,
-													 .NonCompressableExtensions = DefaultSplitOnlyExtensions});
+													 .NonCompressableExtensions = DefaultSplitOnlyExtensions,
+													 .PopulateCache				= UploadToZenCache});
 			UploadOp.Execute();
 			if (AbortFlag)
 			{
@@ -1757,6 +1759,7 @@ namespace {
 		std::vector<std::string> IncludeWildcards;
 		std::vector<std::string> ExcludeWildcards;
 		uint64_t				 MaximumInMemoryPayloadSize = 512u * 1024u;
+		bool					 PopulateCache				= true;
 	};
 
 	void DownloadFolder(ThreadWorkers&				 Workers,
@@ -1926,7 +1929,8 @@ namespace {
 															 .ValidateCompletedSequences	 = Options.PostDownloadVerify,
 															 .ExcludeFolders				 = DefaultExcludeFolders,
 															 .ExcludeExtensions				 = DefaultExcludeExtensions,
-															 .MaximumInMemoryPayloadSize	 = Options.MaximumInMemoryPayloadSize});
+															 .MaximumInMemoryPayloadSize	 = Options.MaximumInMemoryPayloadSize,
+															 .PopulateCache					 = Options.PopulateCache});
 			{
 				ProgressBar::PushLogOperation(ProgressMode, "Download");
 				auto _ = MakeGuard([]() { ProgressBar::PopLogOperation(ProgressMode); });
@@ -2575,6 +2579,12 @@ BuildsCommand::BuildsCommand()
 							   "Percent of an existing block that must be relevant for it to be resused. Defaults to 85.",
 							   cxxopts::value(m_BlockReuseMinPercentLimit),
 							   "<minreuse>");
+	m_UploadOptions.add_option("cache",
+							   "",
+							   "zen-cache-upload",
+							   "Upload data downloaded from remote host to zen cache",
+							   cxxopts::value(m_UploadToZenCache),
+							   "<uploadtozencache>");
 
 	AddMultipartOptions(m_UploadOptions);
 
@@ -2642,6 +2652,12 @@ BuildsCommand::BuildsCommand()
 								 "Force download of all content by ignoring any existing local content",
 								 cxxopts::value(m_Force),
 								 "<force>");
+	m_DownloadOptions.add_option("cache",
+								 "",
+								 "zen-cache-upload",
+								 "Upload data downloaded from remote host to zen cache",
+								 cxxopts::value(m_UploadToZenCache),
+								 "<uploadtozencache>");
 	AddMultipartOptions(m_DownloadOptions);
 
 	AddPartialBlockRequestOptions(m_DownloadOptions);
@@ -3594,7 +3610,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 					 m_AllowMultiparts,
 					 MetaData,
 					 m_CreateBuild,
-					 m_Clean);
+					 m_Clean,
+					 m_UploadToZenCache);
 
 		if (!AbortFlag)
 		{
@@ -3729,7 +3746,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 									   .AllowFileClone				   = m_AllowFileClone,
 									   .IncludeWildcards			   = IncludeWildcards,
 									   .ExcludeWildcards			   = ExcludeWildcards,
-									   .MaximumInMemoryPayloadSize	   = GetMaxMemoryBufferSize(m_BoostWorkerMemory)});
+									   .MaximumInMemoryPayloadSize	   = GetMaxMemoryBufferSize(m_BoostWorkerMemory),
+									   .PopulateCache				   = m_UploadToZenCache});
 
 		if (AbortFlag)
 		{
@@ -4207,7 +4225,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 					 m_AllowMultiparts,
 					 MetaData,
 					 true,
-					 false);
+					 false,
+					 m_UploadToZenCache);
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Upload build)");
@@ -4401,7 +4420,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 					 m_AllowMultiparts,
 					 MetaData2,
 					 true,
-					 false);
+					 false,
+					 m_UploadToZenCache);
 		if (AbortFlag)
 		{
 			throw std::runtime_error("Test aborted. (Upload scrambled)");
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index 964c369c5..ec752e581 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -56,7 +56,8 @@ private:
 
 	// cache
 	std::string m_ZenCacheHost;
-	bool		m_PrimeCacheOnly = false;
+	bool		m_UploadToZenCache = true;
+	bool		m_PrimeCacheOnly   = false;
 
 	std::string			  m_BuildId;
 	bool				  m_CreateBuild = false;
diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp
index 5791e6fda..4fdbd3b0d 100644
--- a/src/zen/cmds/projectstore_cmd.cpp
+++ b/src/zen/cmds/projectstore_cmd.cpp
@@ -1603,6 +1603,7 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 					Writer.AddString("namespace"sv, m_JupiterNamespace);
 					Writer.AddString("bucket"sv, m_JupiterBucket);
 					Writer.AddString("buildsid"sv, m_BuildsId);
+					Writer.AddBool("populateCache"sv, m_UploadToZenCache);
 					WriteAuthOptions(Writer,
 									 m_JupiterOpenIdProvider,
 									 m_JupiterAccessToken,
@@ -2238,6 +2239,12 @@ OplogDownloadCommand::OplogDownloadCommand()
 
 	auto AddCacheOptions = [this](cxxopts::Options& Ops) {
 		Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>");
+		Ops.add_option("cache",
+					   "",
+					   "zen-cache-upload",
+					   "Upload data downloaded from remote host to zen cache",
+					   cxxopts::value(m_UploadToZenCache),
+					   "<uploadtozencache>");
 	};
 
 	AddCloudOptions(m_Options);
@@ -2552,8 +2559,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
 													 .ForceDownload			= m_ForceDownload,
 													 .DecompressAttachments = m_DecompressAttachments,
 													 .TempFolderPath		= StorageTempPath,
-													 .AttachmentOutputPath	= m_AttachmentsPath});
-
+													 .AttachmentOutputPath	= m_AttachmentsPath,
+													 .PopulateCache			= m_UploadToZenCache});
 		Op.Execute();
 	}
 	else
diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h
index 6141a7bce..16852d270 100644
--- a/src/zen/cmds/projectstore_cmd.h
+++ b/src/zen/cmds/projectstore_cmd.h
@@ -173,6 +173,7 @@ private:
 
 	std::string m_CloudUrl;
 	std::string m_ZenCacheHost;
+	bool		m_UploadToZenCache = true;
 	std::string m_CloudKey;
 
 	std::string m_BuildsHost;
@@ -308,6 +309,7 @@ private:
 	std::string m_BuildId;
 	bool		m_ForceDownload		 = false;
 	bool		m_BoostWorkerThreads = false;
+	bool		m_UploadToZenCache	 = true;
 
 	std::filesystem::path m_OplogOutputPath;
 	bool				  m_DecompressAttachments = true;
diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp
index ca54d2ce7..798ef4dae 100644
--- a/src/zenremotestore/builds/buildstorageoperations.cpp
+++ b/src/zenremotestore/builds/buildstorageoperations.cpp
@@ -1757,7 +1757,7 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
 							if (!BlockBuffer)
 							{
 								BlockBuffer = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash);
-								if (BlockBuffer && m_Storage.BuildCacheStorage)
+								if (BlockBuffer && m_Storage.BuildCacheStorage && m_Options.PopulateCache)
 								{
 									m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId,
 																			  BlockDescription.BlockHash,
@@ -3338,7 +3338,7 @@ BuildsOperationUpdateFolder::DownloadBuildBlob(uint32_t									 RemoteChunkInde
 					m_DownloadStats.DownloadedChunkCount++;
 					m_DownloadStats.RequestsCompleteCount++;
 
-					if (Payload && m_Storage.BuildCacheStorage)
+					if (Payload && m_Storage.BuildCacheStorage && m_Options.PopulateCache)
 					{
 						m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId,
 																  ChunkHash,
@@ -3352,7 +3352,7 @@ BuildsOperationUpdateFolder::DownloadBuildBlob(uint32_t									 RemoteChunkInde
 		else
 		{
 			BuildBlob = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, ChunkHash);
-			if (BuildBlob && m_Storage.BuildCacheStorage)
+			if (BuildBlob && m_Storage.BuildCacheStorage && m_Options.PopulateCache)
 			{
 				m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId,
 														  ChunkHash,
@@ -5446,7 +5446,7 @@ BuildsOperationUploadFolder::Execute()
 					{
 						const CbObject BlockMetaData =
 							BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]);
-						if (m_Storage.BuildCacheStorage)
+						if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
 						{
 							m_Storage.BuildCacheStorage->PutBlobMetadatas(m_BuildId,
 																		  std::vector<IoHash>({BlockHash}),
@@ -5994,7 +5994,7 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent&
 												const IoHash&  BlockHash		   = OutBlocks.BlockDescriptions[BlockIndex].BlockHash;
 												const uint64_t CompressedBlockSize = Payload.GetCompressedSize();
 
-												if (m_Storage.BuildCacheStorage)
+												if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
 												{
 													m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId,
 																							  BlockHash,
@@ -6017,7 +6017,7 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent&
 																		   OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size());
 												}
 
-												if (m_Storage.BuildCacheStorage)
+												if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
 												{
 													m_Storage.BuildCacheStorage->PutBlobMetadatas(m_BuildId,
 																								  std::vector<IoHash>({BlockHash}),
@@ -6472,7 +6472,7 @@ BuildsOperationUploadFolder::UploadPartBlobs(const ChunkedFolderContent&			   Co
 						const CbObject BlockMetaData =
 							BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]);
 
-						if (m_Storage.BuildCacheStorage)
+						if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
 						{
 							m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload);
 						}
@@ -6488,7 +6488,7 @@ BuildsOperationUploadFolder::UploadPartBlobs(const ChunkedFolderContent&			   Co
 						UploadedBlockSize += PayloadSize;
 						TempUploadStats.BlocksBytes += PayloadSize;
 
-						if (m_Storage.BuildCacheStorage)
+						if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
 						{
 							m_Storage.BuildCacheStorage->PutBlobMetadatas(m_BuildId,
 																		  std::vector<IoHash>({BlockHash}),
@@ -6555,7 +6555,7 @@ BuildsOperationUploadFolder::UploadPartBlobs(const ChunkedFolderContent&			   Co
 
 						const uint64_t PayloadSize = Payload.GetSize();
 
-						if (m_Storage.BuildCacheStorage)
+						if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
 						{
 							m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
 						}
diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
index 6a6b8e616..eba0966f7 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
@@ -143,6 +143,7 @@ public:
 		std::vector<std::string> ExcludeFolders;
 		std::vector<std::string> ExcludeExtensions;
 		uint64_t				 MaximumInMemoryPayloadSize = 512u * 1024u;
+		bool					 PopulateCache				= true;
 	};
 
 	BuildsOperationUpdateFolder(OperationLogOutput&						  OperationLogOutput,
@@ -515,6 +516,8 @@ public:
 		std::string				 ZenExcludeManifestName = ".zen_exclude_manifest.txt";
 
 		std::vector<std::string> NonCompressableExtensions;
+
+		bool PopulateCache = true;
 	};
 	BuildsOperationUploadFolder(OperationLogOutput&			 OperationLogOutput,
 								StorageInstance&			 Storage,
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h
index ecd157af8..bb26f55a0 100644
--- a/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h
+++ b/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h
@@ -23,6 +23,7 @@ struct BuildsRemoteStoreOptions : RemoteStoreOptions
 	bool				  ForceDisableBlocks	 = false;
 	bool				  ForceDisableTempBlocks = false;
 	bool				  AssumeHttp2			 = false;
+	bool				  PopulateCache			 = true;
 	IoBuffer			  MetaData;
 };
 
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/projectstoreoperations.h b/src/zenremotestore/include/zenremotestore/projectstore/projectstoreoperations.h
index 2ddd0f11d..044436509 100644
--- a/src/zenremotestore/include/zenremotestore/projectstore/projectstoreoperations.h
+++ b/src/zenremotestore/include/zenremotestore/projectstore/projectstoreoperations.h
@@ -76,6 +76,7 @@ public:
 		bool				  DecompressAttachments = true;
 		std::filesystem::path TempFolderPath;
 		std::filesystem::path AttachmentOutputPath;
+		bool				  PopulateCache = true;
 	};
 
 	ProjectStoreOperationDownloadAttachments(OperationLogOutput&			  OperationLogOutput,
diff --git a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp
index db2192a49..e27ab0c1f 100644
--- a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp
+++ b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp
@@ -35,7 +35,8 @@ public:
 					  const Oid&				   BuildId,
 					  const IoBuffer&			   MetaData,
 					  bool						   ForceDisableBlocks,
-					  bool						   ForceDisableTempBlocks)
+					  bool						   ForceDisableTempBlocks,
+					  bool						   PopulateCache)
 	: m_Log(InLog)
 	, m_BuildStorageHttp(HostUrl, ClientSettings)
 	, m_BuildStorage(CreateJupiterBuildStorage(Log(),
@@ -51,6 +52,7 @@ public:
 	, m_MetaData(MetaData)
 	, m_EnableBlocks(!ForceDisableBlocks)
 	, m_UseTempBlocks(!ForceDisableTempBlocks)
+	, m_PopulateCache(PopulateCache)
 	{
 		m_MetaData.MakeOwned();
 		if (OptionalCacheClientSettings)
@@ -465,7 +467,7 @@ public:
 			if (!Result.Bytes)
 			{
 				Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash);
-				if (m_BuildCacheStorage && Result.Bytes)
+				if (m_BuildCacheStorage && Result.Bytes && m_PopulateCache)
 				{
 					m_BuildCacheStorage->PutBuildBlob(m_BuildId,
 													  RawHash,
@@ -546,7 +548,7 @@ public:
 				return LoadAttachmentsResult{ChunkResult};
 			}
 			ZEN_DEBUG("Loaded attachment in {}", NiceTimeSpanMs(static_cast<uint64_t>(ChunkResult.ElapsedSeconds * 1000)));
-			if (m_BuildCacheStorage && ChunkResult.Bytes)
+			if (m_BuildCacheStorage && ChunkResult.Bytes && m_PopulateCache)
 			{
 				m_BuildCacheStorage->PutBuildBlob(m_BuildId,
 												  Hash,
@@ -595,6 +597,7 @@ private:
 	const bool		  m_EnableBlocks	 = true;
 	const bool		  m_UseTempBlocks	 = true;
 	const bool		  m_AllowRedirect	 = false;
+	const bool		  m_PopulateCache	 = true;
 };
 
 std::shared_ptr<RemoteProjectStore>
@@ -700,7 +703,8 @@ CreateJupiterBuildsRemoteStore(LoggerRef					   InLog,
 																						  Options.BuildId,
 																						  Options.MetaData,
 																						  Options.ForceDisableBlocks,
-																						  Options.ForceDisableTempBlocks);
+																						  Options.ForceDisableTempBlocks,
+																						  Options.PopulateCache);
 	return RemoteStore;
 }
 
diff --git a/src/zenremotestore/projectstore/projectstoreoperations.cpp b/src/zenremotestore/projectstore/projectstoreoperations.cpp
index 7dd85531c..becac3d4c 100644
--- a/src/zenremotestore/projectstore/projectstoreoperations.cpp
+++ b/src/zenremotestore/projectstore/projectstoreoperations.cpp
@@ -433,7 +433,7 @@ ProjectStoreOperationDownloadAttachments::Execute()
 		if (!Payload)
 		{
 			Payload = m_Storage.BuildStorage->GetBuildBlob(m_State.GetBuildId(), RawHash);
-			if (m_Storage.BuildCacheStorage)
+			if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
 			{
 				m_Storage.BuildCacheStorage->PutBuildBlob(m_State.GetBuildId(),
 														  RawHash,
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp
index 8f2b9df9d..9661f1b49 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp
@@ -464,6 +464,7 @@ namespace {
 			bool ForceDisableBlocks		= Builds["disableblocks"sv].AsBool(false);
 			bool ForceDisableTempBlocks = Builds["disabletempblocks"sv].AsBool(false);
 			bool AssumeHttp2			= Builds["assumehttp2"sv].AsBool(false);
+			bool PopulateCache			= Builds["populateCache"sv].AsBool(true);
 
 			MemoryView MetaDataSection = Builds["metadata"sv].AsBinaryView();
 			IoBuffer   MetaData(IoBuffer::Wrap, MetaDataSection.GetData(), MetaDataSection.GetSize());
@@ -483,6 +484,7 @@ namespace {
 				ForceDisableBlocks,
 				ForceDisableTempBlocks,
 				AssumeHttp2,
+				PopulateCache,
 				MetaData};
 			RemoteStore = CreateJupiterBuildsRemoteStore(Log(),
 														 Options,