add boost-worker oplog import export options (#693)

- Feature: `zen oplog-export`, `zen oplog-import` and `zen oplog-download` now has options to boost workers
  - `--boost-worker-count` - Increase the number of worker threads - may cause computer to be less responsive
  - `--boost-worker-memory` - Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to be less responsive due to high memory usage
  - `--boost-workers` - Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive
 - Improvement: Refactored boost options for `zen builds` operations `upload`, `download`, `diff`, `prime-cache`, `fetch-blob` and `validate-part`
  - `--boost-worker-count` - Increase the number of worker threads - may cause computer to be less responsive
  - `--boost-worker-memory` - Increase the limit where we write downloaded data to temporary storage to conserve space - may cause computer to be less responsive due to high memory usage
  - `--boost-workers` - Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive

3 files changed, 57 insertions, 3 deletions

diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
index 223c668cd..eca654223 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
@@ -477,14 +477,18 @@ struct GenerateBlocksStatistics
 	}
 };
 
+static constexpr size_t DefaultMaxChunkBlockSize	  = 64u * 1024u * 1024u;
+static constexpr size_t DefaultMaxChunksPerChunkBlock = 4u * 1000u;
+static constexpr size_t DefaultMaxChunkBlockEmbedSize = 3u * 512u * 1024u;
+
 class BuildsOperationUploadFolder
 {
 public:
 	struct ChunksBlockParameters
 	{
-		size_t MaxBlockSize		 = 64u * 1024u * 1024u;
-		size_t MaxChunksPerBlock = 4u * 1000u;
-		size_t MaxChunkEmbedSize = 3u * 512u * 1024u;
+		size_t MaxBlockSize		 = DefaultMaxChunkBlockSize;
+		size_t MaxChunksPerBlock = DefaultMaxChunksPerChunkBlock;
+		size_t MaxChunkEmbedSize = DefaultMaxChunkBlockEmbedSize;
 	};
 
 	struct Options
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h
index bb26f55a0..e8b7c15c0 100644
--- a/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h
+++ b/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h
@@ -25,6 +25,7 @@ struct BuildsRemoteStoreOptions : RemoteStoreOptions
 	bool				  AssumeHttp2			 = false;
 	bool				  PopulateCache			 = true;
 	IoBuffer			  MetaData;
+	size_t				  MaximumInMemoryDownloadSize = 1024u * 1024u;
 };
 
 std::shared_ptr<RemoteProjectStore> CreateJupiterBuildsRemoteStore(LoggerRef					   InLog,
diff --git a/src/zenremotestore/include/zenremotestore/transferthreadworkers.h b/src/zenremotestore/include/zenremotestore/transferthreadworkers.h
new file mode 100644
index 000000000..a7faacfd5
--- /dev/null
+++ b/src/zenremotestore/include/zenremotestore/transferthreadworkers.h
@@ -0,0 +1,49 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenbase/refcount.h>
+#include <zencore/timer.h>
+#include <zencore/zencore.h>
+
+#include <memory>
+#include <string>
+
+namespace zen {
+
+class WorkerThreadPool;
+
+class TransferThreadWorkers : public RefCounted
+{
+public:
+	TransferThreadWorkers(bool BoostWorkers, bool SingleThreaded);
+
+	WorkerThreadPool& GetIOWorkerPool()
+	{
+		ZEN_ASSERT(m_IOPool);
+		return *m_IOPool;
+	}
+	WorkerThreadPool& GetNetworkPool()
+	{
+		ZEN_ASSERT(m_NetworkPool);
+		return *m_NetworkPool;
+	}
+
+	const std::string& GetWorkersInfo() const { return WorkersInfo; }
+
+	bool IsBoostWorkers() const { return BoostWorkers; }
+	bool IsSingleThreaded() const { return SingleThreaded; }
+
+private:
+	WorkerThreadPool* m_NetworkPool = nullptr;
+	WorkerThreadPool* m_IOPool		= nullptr;
+
+	std::unique_ptr<WorkerThreadPool> m_NetworkPoolInstance;
+	std::unique_ptr<WorkerThreadPool> m_IOPoolInstance;
+
+	std::string WorkersInfo;
+	const bool	BoostWorkers;
+	const bool	SingleThreaded;
+};
+
+}  // namespace zen