move partial chunk block anailsys to chunkblock.h/cpp (#767)

2 files changed, 98 insertions, 46 deletions

diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
index 9e5bf8d91..6800444e0 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
@@ -7,6 +7,7 @@
 #include <zencore/uid.h>
 #include <zencore/zencore.h>
 #include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/chunking/chunkblock.h>
 #include <zenremotestore/chunking/chunkedcontent.h>
 #include <zenutil/bufferedwritefilecache.h>
 
@@ -218,33 +219,6 @@ private:
 		uint64_t			   ElapsedTimeMs = 0;
 	};
 
-	struct BlockRangeDescriptor
-	{
-		uint32_t BlockIndex			  = (uint32_t)-1;
-		uint64_t RangeStart			  = 0;
-		uint64_t RangeLength		  = 0;
-		uint32_t ChunkBlockIndexStart = 0;
-		uint32_t ChunkBlockIndexCount = 0;
-	};
-
-	struct BlockRangeLimit
-	{
-		uint16_t SizePercent;
-		uint16_t MaxRangeCount;
-	};
-
-	static constexpr uint16_t FullBlockRangePercentLimit = 95;
-
-	static constexpr BuildsOperationUpdateFolder::BlockRangeLimit ForceMergeLimits[] = {
-		{.SizePercent = FullBlockRangePercentLimit, .MaxRangeCount = 1},
-		{.SizePercent = 90, .MaxRangeCount = 2},
-		{.SizePercent = 85, .MaxRangeCount = 8},
-		{.SizePercent = 80, .MaxRangeCount = 16},
-		{.SizePercent = 70, .MaxRangeCount = 32},
-		{.SizePercent = 60, .MaxRangeCount = 48},
-		{.SizePercent = 2, .MaxRangeCount = 56},
-		{.SizePercent = 0, .MaxRangeCount = 64}};
-
 	void				  ScanCacheFolder(tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& OutCachedChunkHashesFound,
 										  tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& OutCachedSequenceHashesFound);
 	void				  ScanTempBlocksFolder(tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& OutCachedBlocksFound);
@@ -299,25 +273,9 @@ private:
 						   ParallelWork&							 Work,
 						   std::function<void(IoBuffer&& Payload)>&& OnDownloaded);
 
-	BlockRangeDescriptor							 MergeBlockRanges(std::span<const BlockRangeDescriptor> Ranges);
-	std::optional<std::vector<BlockRangeDescriptor>> MakeOptionalBlockRangeVector(uint64_t					  TotalBlockSize,
-																				  const BlockRangeDescriptor& Range);
-	const BlockRangeLimit*							 GetBlockRangeLimitForRange(std::span<const BlockRangeLimit>	  Limits,
-																				uint64_t							  TotalBlockSize,
-																				std::span<const BlockRangeDescriptor> Ranges);
-	std::vector<BlockRangeDescriptor>				 CollapseBlockRanges(const uint64_t						   AlwaysAcceptableGap,
-																		 std::span<const BlockRangeDescriptor> BlockRanges);
-	uint64_t										 CalculateNextGap(std::span<const BlockRangeDescriptor> BlockRanges);
-	std::optional<std::vector<BlockRangeDescriptor>> CalculateBlockRanges(uint32_t					   BlockIndex,
-																		  const ChunkBlockDescription& BlockDescription,
-																		  std::span<const uint32_t>	   BlockChunkIndexNeeded,
-																		  bool						   LimitToSingleRange,
-																		  const uint64_t			   ChunkStartOffsetInBlock,
-																		  const uint64_t			   TotalBlockSize,
-																		  uint64_t&					   OutTotalWantedChunksSize);
-	void											 DownloadPartialBlock(const BlockRangeDescriptor																BlockRange,
-																		  const BlobsExistsResult&																	ExistsResult,
-																		  std::function<void(IoBuffer&& InMemoryBuffer, const std::filesystem::path& OnDiskPath)>&& OnDownloaded);
+	void DownloadPartialBlock(const ChunkBlockAnalyser::BlockRangeDescriptor											BlockRange,
+							  const BlobsExistsResult&																	ExistsResult,
+							  std::function<void(IoBuffer&& InMemoryBuffer, const std::filesystem::path& OnDiskPath)>&& OnDownloaded);
 
 	std::vector<uint32_t> WriteLocalChunkToCache(CloneQueryInterface*					   CloneQuery,
 												 const CopyChunkData&					   CopyData,
diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h
index d339b0f94..57710fcf5 100644
--- a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h
+++ b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h
@@ -7,6 +7,10 @@
 #include <zencore/compactbinary.h>
 #include <zencore/compress.h>
 
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
 #include <optional>
 #include <vector>
 
@@ -73,6 +77,96 @@ std::vector<size_t> FindReuseBlocks(OperationLogOutput&						  Output,
 									std::span<const uint32_t>				  ChunkIndexes,
 									std::vector<uint32_t>&					  OutUnusedChunkIndexes);
 
+class ChunkBlockAnalyser
+{
+public:
+	struct Options
+	{
+		bool IsQuiet   = false;
+		bool IsVerbose = false;
+	};
+
+	ChunkBlockAnalyser(OperationLogOutput& LogOutput, std::span<const ChunkBlockDescription> BlockDescriptions, const Options& Options);
+
+	struct BlockRangeDescriptor
+	{
+		uint32_t BlockIndex			  = (uint32_t)-1;
+		uint64_t RangeStart			  = 0;
+		uint64_t RangeLength		  = 0;
+		uint32_t ChunkBlockIndexStart = 0;
+		uint32_t ChunkBlockIndexCount = 0;
+	};
+
+	struct NeededBlock
+	{
+		uint32_t			  BlockIndex;
+		std::vector<uint32_t> ChunkIndexes;
+	};
+
+	std::vector<NeededBlock> GetNeeded(const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex,
+									   std::function<bool(uint32_t ChunkIndex)>&&			   NeedsBlockChunk);
+
+	enum EPartialBlockDownloadMode
+	{
+		Off,
+		SingleRange,
+		On,
+		Exact
+	};
+
+	struct BlockResult
+	{
+		std::vector<BlockRangeDescriptor> BlockRanges;
+		std::vector<uint32_t>			  FullBlockIndexes;
+	};
+
+	BlockResult CalculatePartialBlockDownloads(std::span<const NeededBlock>				  NeededBlocks,
+											   std::span<const EPartialBlockDownloadMode> BlockPartialDownloadModes);
+
+private:
+	struct BlockRangeLimit
+	{
+		uint16_t SizePercent;
+		uint16_t MaxRangeCount;
+	};
+
+	static constexpr uint16_t FullBlockRangePercentLimit = 95;
+
+	static constexpr BlockRangeLimit ForceMergeLimits[] = {{.SizePercent = FullBlockRangePercentLimit, .MaxRangeCount = 1},
+														   {.SizePercent = 90, .MaxRangeCount = 2},
+														   {.SizePercent = 85, .MaxRangeCount = 8},
+														   {.SizePercent = 80, .MaxRangeCount = 16},
+														   {.SizePercent = 75, .MaxRangeCount = 32},
+														   {.SizePercent = 70, .MaxRangeCount = 48},
+														   {.SizePercent = 4, .MaxRangeCount = 82},
+														   {.SizePercent = 0, .MaxRangeCount = 96}};
+
+	BlockRangeDescriptor							 MergeBlockRanges(std::span<const BlockRangeDescriptor> Ranges);
+	std::optional<std::vector<BlockRangeDescriptor>> MakeOptionalBlockRangeVector(uint64_t					  TotalBlockSize,
+																				  const BlockRangeDescriptor& Range);
+	const BlockRangeLimit*							 GetBlockRangeLimitForRange(std::span<const BlockRangeLimit>	  Limits,
+																				uint64_t							  TotalBlockSize,
+																				std::span<const BlockRangeDescriptor> Ranges);
+	std::vector<BlockRangeDescriptor>				 CollapseBlockRanges(const uint64_t						   AlwaysAcceptableGap,
+																		 std::span<const BlockRangeDescriptor> BlockRanges);
+	uint64_t										 CalculateNextGap(std::span<const BlockRangeDescriptor> BlockRanges);
+	std::optional<std::vector<BlockRangeDescriptor>> CalculateBlockRanges(uint32_t					   BlockIndex,
+																		  const ChunkBlockDescription& BlockDescription,
+																		  std::span<const uint32_t>	   BlockChunkIndexNeeded,
+																		  EPartialBlockDownloadMode	   PartialBlockDownloadMode,
+																		  const uint64_t			   ChunkStartOffsetInBlock,
+																		  const uint64_t			   TotalBlockSize,
+																		  uint64_t&					   OutTotalWantedChunksSize);
+
+	OperationLogOutput&							 m_LogOutput;
+	const std::span<const ChunkBlockDescription> m_BlockDescriptions;
+	const Options								 m_Options;
+};
+
+#if ZEN_WITH_TESTS
+
+class CbWriter;
 void chunkblock_forcelink();
+#endif	// ZEN_WITH_TESTS
 
 }  // namespace zen