refactor builds cmd part4 (#579)

* move lambdas to class functions

1 files changed, 95 insertions, 21 deletions

diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
index be6720d5d..fcf880e48 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
@@ -12,6 +12,10 @@
 #include <atomic>
 #include <memory>
 
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_set.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
 namespace zen {
 
 class CloneQueryInterface;
@@ -240,6 +244,61 @@ private:
 		std::filesystem::path Path;
 	};
 
+	struct ScavengedSequenceCopyOperation
+	{
+		uint32_t ScavengedContentIndex = (uint32_t)-1;
+		uint32_t ScavengedPathIndex	   = (uint32_t)-1;
+		uint32_t RemoteSequenceIndex   = (uint32_t)-1;
+		uint64_t RawSize			   = (uint32_t)-1;
+	};
+
+	struct CopyChunkData
+	{
+		uint32_t														ScavengeSourceIndex = (uint32_t)-1;
+		uint32_t														SourceSequenceIndex = (uint32_t)-1;
+		std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> TargetChunkLocationPtrs;
+		struct ChunkTarget
+		{
+			uint32_t TargetChunkLocationCount = (uint32_t)-1;
+			uint32_t RemoteChunkIndex		  = (uint32_t)-1;
+			uint64_t CacheFileOffset		  = (uint64_t)-1;
+		};
+		std::vector<ChunkTarget> ChunkTargets;
+	};
+
+	struct BlobsExistsResult
+	{
+		tsl::robin_set<IoHash> ExistingBlobs;
+		uint64_t			   ElapsedTimeMs = 0;
+	};
+
+	struct BlockRangeDescriptor
+	{
+		uint32_t BlockIndex			  = (uint32_t)-1;
+		uint64_t RangeStart			  = 0;
+		uint64_t RangeLength		  = 0;
+		uint32_t ChunkBlockIndexStart = 0;
+		uint32_t ChunkBlockIndexCount = 0;
+	};
+
+	struct BlockRangeLimit
+	{
+		uint16_t SizePercent;
+		uint16_t MaxRangeCount;
+	};
+
+	static constexpr uint16_t FullBlockRangePercentLimit = 95;
+
+	static constexpr BuildsOperationUpdateFolder::BlockRangeLimit ForceMergeLimits[] = {
+		{.SizePercent = FullBlockRangePercentLimit, .MaxRangeCount = 1},
+		{.SizePercent = 90, .MaxRangeCount = 2},
+		{.SizePercent = 85, .MaxRangeCount = 8},
+		{.SizePercent = 80, .MaxRangeCount = 16},
+		{.SizePercent = 70, .MaxRangeCount = 32},
+		{.SizePercent = 60, .MaxRangeCount = 48},
+		{.SizePercent = 2, .MaxRangeCount = 56},
+		{.SizePercent = 0, .MaxRangeCount = 64}};
+
 	void				  ScanCacheFolder(tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& OutCachedChunkHashesFound,
 										  tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& OutCachedSequenceHashesFound);
 	void				  ScanTempBlocksFolder(tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& OutCachedBlocksFound);
@@ -265,31 +324,46 @@ private:
 
 	uint64_t GetChunkWriteCount(std::span<const std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters, uint32_t ChunkIndex);
 
-	struct ScavengedSequenceCopyOperation
-	{
-		uint32_t ScavengedContentIndex = (uint32_t)-1;
-		uint32_t ScavengedPathIndex	   = (uint32_t)-1;
-		uint32_t RemoteSequenceIndex   = (uint32_t)-1;
-		uint64_t RawSize			   = (uint32_t)-1;
-	};
-
 	void WriteScavengedSequenceToCache(const std::filesystem::path&			 ScavengeRootPath,
 									   const ChunkedFolderContent&			 ScavengedContent,
 									   const ScavengedSequenceCopyOperation& ScavengeOp);
 
-	struct CopyChunkData
-	{
-		uint32_t														ScavengeSourceIndex = (uint32_t)-1;
-		uint32_t														SourceSequenceIndex = (uint32_t)-1;
-		std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> TargetChunkLocationPtrs;
-		struct ChunkTarget
-		{
-			uint32_t TargetChunkLocationCount = (uint32_t)-1;
-			uint32_t RemoteChunkIndex		  = (uint32_t)-1;
-			uint64_t CacheFileOffset		  = (uint64_t)-1;
-		};
-		std::vector<ChunkTarget> ChunkTargets;
-	};
+	void WriteLooseChunk(const uint32_t													   RemoteChunkIndex,
+						 const BlobsExistsResult&										   ExistsResult,
+						 std::span<std::atomic<uint32_t>>								   SequenceIndexChunksLeftToWriteCounters,
+						 std::atomic<uint64_t>&											   WritePartsComplete,
+						 std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>&& ChunkTargetPtrs,
+						 BufferedWriteFileCache&										   WriteCache,
+						 ParallelWork&													   Work,
+						 uint64_t														   TotalRequestCount,
+						 uint64_t														   TotalPartWriteCount,
+						 FilteredRate&													   FilteredDownloadedBytesPerSecond,
+						 FilteredRate&													   FilteredWrittenBytesPerSecond);
+
+	void DownloadBuildBlob(uint32_t									 RemoteChunkIndex,
+						   const BlobsExistsResult&					 ExistsResult,
+						   ParallelWork&							 Work,
+						   std::function<void(IoBuffer&& Payload)>&& OnDownloaded);
+
+	BlockRangeDescriptor							 MergeBlockRanges(std::span<const BlockRangeDescriptor> Ranges);
+	std::optional<std::vector<BlockRangeDescriptor>> MakeOptionalBlockRangeVector(uint64_t					  TotalBlockSize,
+																				  const BlockRangeDescriptor& Range);
+	const BlockRangeLimit*							 GetBlockRangeLimitForRange(std::span<const BlockRangeLimit>	  Limits,
+																				uint64_t							  TotalBlockSize,
+																				std::span<const BlockRangeDescriptor> Ranges);
+	std::vector<BlockRangeDescriptor>				 CollapseBlockRanges(const uint64_t						   AlwaysAcceptableGap,
+																		 std::span<const BlockRangeDescriptor> BlockRanges);
+	uint64_t										 CalculateNextGap(std::span<const BlockRangeDescriptor> BlockRanges);
+	std::optional<std::vector<BlockRangeDescriptor>> CalculateBlockRanges(uint32_t					   BlockIndex,
+																		  const ChunkBlockDescription& BlockDescription,
+																		  std::span<const uint32_t>	   BlockChunkIndexNeeded,
+																		  bool						   LimitToSingleRange,
+																		  const uint64_t			   ChunkStartOffsetInBlock,
+																		  const uint64_t			   TotalBlockSize,
+																		  uint64_t&					   OutTotalWantedChunksSize);
+	void											 DownloadPartialBlock(const BlockRangeDescriptor																BlockRange,
+																		  const BlobsExistsResult&																	ExistsResult,
+																		  std::function<void(IoBuffer&& InMemoryBuffer, const std::filesystem::path& OnDiskPath)>&& OnDownloaded);
 
 	std::vector<uint32_t> WriteLocalChunkToCache(CloneQueryInterface*					   CloneQuery,
 												 const CopyChunkData&					   CopyData,