builds multipart upload (#722)

- Feature: `zen builds upload` now support structure manifest input for `--manifest-path` when the path has a `.json` extension
  - The structured manifest supports splitting a build into multiple parts

        {
            "parts": {
                "default" : {
                    "partId": "f939f3939939fff3f3202", # optional - used to control the id of each part
                    "files": [
                        "foo/bar",
                        "baz.exe"
                    ]
                },
                "symbols": {
                    "files": [
                        "baz.pdb"
                    ]
                }
            }
        }

3 files changed, 137 insertions, 9 deletions

diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
index 32c8bda01..f9fe949a6 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
@@ -11,6 +11,7 @@
 #include <zenutil/bufferedwritefilecache.h>
 
 #include <atomic>
+#include <future>
 #include <memory>
 
 ZEN_THIRD_PARTY_INCLUDES_START
@@ -421,6 +422,21 @@ struct FindBlocksStatistics
 	uint64_t NewBlocksCount			 = 0;
 	uint64_t NewBlocksChunkCount	 = 0;
 	uint64_t NewBlocksChunkByteCount = 0;
+
+	FindBlocksStatistics& operator+=(const FindBlocksStatistics& Rhs)
+	{
+		FindBlockTimeMS += Rhs.FindBlockTimeMS;
+		PotentialChunkCount += Rhs.PotentialChunkCount;
+		PotentialChunkByteCount += Rhs.PotentialChunkByteCount;
+		FoundBlockCount += Rhs.FoundBlockCount;
+		FoundBlockChunkCount += Rhs.FoundBlockChunkCount;
+		FoundBlockByteCount += Rhs.FoundBlockByteCount;
+		AcceptedBlockCount += Rhs.AcceptedBlockCount;
+		NewBlocksCount += Rhs.NewBlocksCount;
+		NewBlocksChunkCount += Rhs.NewBlocksChunkCount;
+		NewBlocksChunkByteCount += Rhs.NewBlocksChunkByteCount;
+		return *this;
+	}
 };
 
 struct UploadStatistics
@@ -528,15 +544,14 @@ public:
 								WorkerThreadPool&			 IOWorkerPool,
 								WorkerThreadPool&			 NetworkPool,
 								const Oid&					 BuildId,
-								const Oid&					 BuildPartId,
-								const std::string_view		 BuildPartName,
 								const std::filesystem::path& Path,
-								const std::filesystem::path& ManifestPath,
 								bool						 CreateBuild,
 								const CbObject&				 MetaData,
 								const Options&				 Options);
 
-	void Execute();
+	std::vector<std::pair<Oid, std::string>> Execute(const Oid&					  BuildPartId,
+													 const std::string_view		  BuildPartName,
+													 const std::filesystem::path& ManifestPath);
 
 	DiskStatistics			   m_DiskStats;
 	GetFolderContentStatistics m_LocalFolderScanStats;
@@ -548,7 +563,42 @@ public:
 	LooseChunksStatistics	   m_LooseChunksStats;
 
 private:
-	std::vector<std::filesystem::path> ParseManifest(const std::filesystem::path& Path, const std::filesystem::path& ManifestPath);
+	struct PrepareBuildResult
+	{
+		std::vector<ChunkBlockDescription> KnownBlocks;
+		uint64_t						   PreferredMultipartChunkSize = 0;
+		uint64_t						   PayloadSize				   = 0;
+		uint64_t						   PrepareBuildTimeMs		   = 0;
+		uint64_t						   FindBlocksTimeMs			   = 0;
+		uint64_t						   ElapsedTimeMs			   = 0;
+	};
+
+	PrepareBuildResult PrepareBuild();
+
+	struct PartManifest
+	{
+		struct Part
+		{
+			Oid								   PartId;
+			std::string						   PartName;
+			std::vector<std::filesystem::path> Files;
+		};
+		std::vector<Part> Parts;
+	};
+
+	struct UploadPart
+	{
+		Oid						   PartId = Oid::Zero;
+		std::string				   PartName;
+		FolderContent			   Content;
+		uint64_t				   TotalRawSize = 0;
+		GetFolderContentStatistics LocalFolderScanStats;
+	};
+
+	std::vector<BuildsOperationUploadFolder::UploadPart> ReadFolder();
+	std::vector<UploadPart>								 ReadManifestParts(const std::filesystem::path& ManifestPath);
+
+	PartManifest ParseManifest(const std::filesystem::path& Path, const std::filesystem::path& ManifestPath);
 
 	bool IsAcceptedFolder(const std::string_view& RelativePath) const;
 	bool IsAcceptedFile(const std::string_view& RelativePath) const;
@@ -571,7 +621,9 @@ private:
 	void GenerateBuildBlocks(const ChunkedFolderContent&			   Content,
 							 const ChunkedContentLookup&			   Lookup,
 							 const std::vector<std::vector<uint32_t>>& NewBlockChunks,
-							 GeneratedBlocks&						   OutBlocks);
+							 GeneratedBlocks&						   OutBlocks,
+							 GenerateBlocksStatistics&				   GenerateBlocksStats,
+							 UploadStatistics&						   UploadStats);
 
 	std::vector<uint32_t> CalculateAbsoluteChunkOrders(const std::span<const IoHash>						   LocalChunkHashes,
 													   const std::span<const uint32_t>						   LocalChunkOrder,
@@ -594,6 +646,24 @@ private:
 								  CompositeBuffer&&			   HeaderBuffer,
 								  const std::vector<uint32_t>& ChunksInBlock);
 
+	enum class PartTaskSteps : uint32_t
+	{
+		ChunkPartContent = 0,
+		CalculateDelta,
+		GenerateBlocks,
+		BuildPartManifest,
+		UploadBuildPart,
+		UploadAttachments,
+		PutBuildPartStats,
+		StepCount
+	};
+
+	void UploadBuildPart(ChunkingController& ChunkController,
+						 uint32_t			 PartIndex,
+						 const UploadPart&	 Part,
+						 uint32_t			 PartStepOffset,
+						 uint32_t			 StepCount);
+
 	void UploadPartBlobs(const ChunkedFolderContent&			   Content,
 						 const ChunkedContentLookup&			   Lookup,
 						 std::span<IoHash>						   RawHashes,
@@ -617,16 +687,16 @@ private:
 	WorkerThreadPool&	m_IOWorkerPool;
 	WorkerThreadPool&	m_NetworkPool;
 	const Oid			m_BuildId;
-	const Oid			m_BuildPartId;
-	const std::string	m_BuildPartName;
 
 	const std::filesystem::path m_Path;
-	const std::filesystem::path m_ManifestPath;
 	const bool					m_CreateBuild;	// ?? Member?
 	const CbObject				m_MetaData;		// ?? Member
 	const Options				m_Options;
 
 	tsl::robin_set<uint32_t> m_NonCompressableExtensionHashes;
+
+	std::future<PrepareBuildResult> m_PrepBuildResultFuture;
+	PrepareBuildResult				m_PrepBuildResult;
 };
 
 struct ValidateStatistics
@@ -730,4 +800,30 @@ CompositeBuffer ValidateBlob(std::atomic<bool>& AbortFlag,
 							 uint64_t&			OutCompressedSize,
 							 uint64_t&			OutDecompressedSize);
 
+std::vector<std::pair<Oid, std::string>> ResolveBuildPartNames(CbObjectView					BuildObject,
+															   const Oid&					BuildId,
+															   const std::vector<Oid>&		BuildPartIds,
+															   std::span<const std::string> BuildPartNames,
+															   std::uint64_t&				OutPreferredMultipartChunkSize);
+
+ChunkedFolderContent GetRemoteContent(OperationLogOutput&							  Output,
+									  StorageInstance&								  Storage,
+									  const Oid&									  BuildId,
+									  const std::vector<std::pair<Oid, std::string>>& BuildParts,
+									  std::span<const std::string>					  IncludeWildcards,
+									  std::span<const std::string>					  ExcludeWildcards,
+									  std::unique_ptr<ChunkingController>&			  OutChunkController,
+									  std::vector<ChunkedFolderContent>&			  OutPartContents,
+									  std::vector<ChunkBlockDescription>&			  OutBlockDescriptions,
+									  std::vector<IoHash>&							  OutLooseChunkHashes,
+									  bool											  IsQuiet,
+									  bool											  IsVerbose,
+									  bool											  DoExtraContentVerify);
+
+std::string GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix);
+
+#if ZEN_WITH_TESTS
+void buildstorageoperations_forcelink();
+#endif	// ZEN_WITH_TESTS
+
 }  // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h
index 295d275d1..d339b0f94 100644
--- a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h
+++ b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h
@@ -47,6 +47,19 @@ struct ReuseBlocksStatistics
 	uint64_t RejectedByteCount			 = 0;
 	uint64_t AcceptedReduntantChunkCount = 0;
 	uint64_t AcceptedReduntantByteCount	 = 0;
+
+	ReuseBlocksStatistics& operator+=(const ReuseBlocksStatistics& Rhs)
+	{
+		AcceptedChunkCount += Rhs.AcceptedChunkCount;
+		AcceptedByteCount += Rhs.AcceptedByteCount;
+		AcceptedRawByteCount += Rhs.AcceptedRawByteCount;
+		RejectedBlockCount += Rhs.RejectedBlockCount;
+		RejectedChunkCount += Rhs.RejectedChunkCount;
+		RejectedByteCount += Rhs.RejectedByteCount;
+		AcceptedReduntantChunkCount += Rhs.AcceptedReduntantChunkCount;
+		AcceptedReduntantByteCount += Rhs.AcceptedReduntantByteCount;
+		return *this;
+	}
 };
 
 class OperationLogOutput;
diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h
index 78f20a727..1e8e878df 100644
--- a/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h
+++ b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h
@@ -55,11 +55,30 @@ FolderContent LoadFolderContentToCompactBinary(CbObjectView Input);
 
 struct GetFolderContentStatistics
 {
+	GetFolderContentStatistics() {}
+	GetFolderContentStatistics(GetFolderContentStatistics&& Rhs)
+	: FoundFileCount(Rhs.FoundFileCount.load())
+	, FoundFileByteCount(Rhs.FoundFileByteCount.load())
+	, AcceptedFileCount(Rhs.AcceptedFileCount.load())
+	, AcceptedFileByteCount(Rhs.AcceptedFileByteCount.load())
+	, ElapsedWallTimeUS(Rhs.ElapsedWallTimeUS)
+	{
+	}
 	std::atomic<uint64_t> FoundFileCount		= 0;
 	std::atomic<uint64_t> FoundFileByteCount	= 0;
 	std::atomic<uint64_t> AcceptedFileCount		= 0;
 	std::atomic<uint64_t> AcceptedFileByteCount = 0;
 	uint64_t			  ElapsedWallTimeUS		= 0;
+
+	inline GetFolderContentStatistics& operator+=(const GetFolderContentStatistics& Rhs)
+	{
+		FoundFileCount += Rhs.FoundFileCount;
+		FoundFileByteCount += Rhs.FoundFileByteCount;
+		AcceptedFileCount += Rhs.AcceptedFileCount;
+		AcceptedFileByteCount += Rhs.AcceptedFileByteCount;
+		ElapsedWallTimeUS += Rhs.ElapsedWallTimeUS;
+		return *this;
+	}
 };
 
 FolderContent GetFolderContent(GetFolderContentStatistics&																Stats,