oplog download size (#690)

- Bugfix: Upload of oplogs could reference multiple blocks for the same chunk causing redundant downloads of blocks
- Improvement: Use the improved block reuse selection function from zen builds upload in zen oplog-export to reduce oplog download size

3 files changed, 39 insertions, 27 deletions

diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
index 3c4535d9c..223c668cd 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
@@ -406,24 +406,16 @@ private:
 
 struct FindBlocksStatistics
 {
-	uint64_t FindBlockTimeMS			 = 0;
-	uint64_t PotentialChunkCount		 = 0;
-	uint64_t PotentialChunkByteCount	 = 0;
-	uint64_t FoundBlockCount			 = 0;
-	uint64_t FoundBlockChunkCount		 = 0;
-	uint64_t FoundBlockByteCount		 = 0;
-	uint64_t AcceptedBlockCount			 = 0;
-	uint64_t AcceptedChunkCount			 = 0;
-	uint64_t AcceptedByteCount			 = 0;
-	uint64_t AcceptedRawByteCount		 = 0;
-	uint64_t RejectedBlockCount			 = 0;
-	uint64_t RejectedChunkCount			 = 0;
-	uint64_t RejectedByteCount			 = 0;
-	uint64_t AcceptedReduntantChunkCount = 0;
-	uint64_t AcceptedReduntantByteCount	 = 0;
-	uint64_t NewBlocksCount				 = 0;
-	uint64_t NewBlocksChunkCount		 = 0;
-	uint64_t NewBlocksChunkByteCount	 = 0;
+	uint64_t FindBlockTimeMS		 = 0;
+	uint64_t PotentialChunkCount	 = 0;
+	uint64_t PotentialChunkByteCount = 0;
+	uint64_t FoundBlockCount		 = 0;
+	uint64_t FoundBlockChunkCount	 = 0;
+	uint64_t FoundBlockByteCount	 = 0;
+	uint64_t AcceptedBlockCount		 = 0;
+	uint64_t NewBlocksCount			 = 0;
+	uint64_t NewBlocksChunkCount	 = 0;
+	uint64_t NewBlocksChunkByteCount = 0;
 };
 
 struct UploadStatistics
@@ -541,6 +533,7 @@ public:
 	GetFolderContentStatistics m_LocalFolderScanStats;
 	ChunkingStatistics		   m_ChunkingStats;
 	FindBlocksStatistics	   m_FindBlocksStats;
+	ReuseBlocksStatistics	   m_ReuseBlocksStats;
 	UploadStatistics		   m_UploadStats;
 	GenerateBlocksStatistics   m_GenerateBlocksStats;
 	LooseChunksStatistics	   m_LooseChunksStats;
@@ -551,14 +544,10 @@ private:
 	bool IsAcceptedFolder(const std::string_view& RelativePath) const;
 	bool IsAcceptedFile(const std::string_view& RelativePath) const;
 
-	std::vector<size_t> FindReuseBlocks(const std::vector<ChunkBlockDescription>& KnownBlocks,
-										std::span<const IoHash>					  ChunkHashes,
-										std::span<const uint32_t>				  ChunkIndexes,
-										std::vector<uint32_t>&					  OutUnusedChunkIndexes);
-	void				ArrangeChunksIntoBlocks(const ChunkedFolderContent&			Content,
-												const ChunkedContentLookup&			Lookup,
-												std::vector<uint32_t>&				ChunkIndexes,
-												std::vector<std::vector<uint32_t>>& OutBlocks);
+	void ArrangeChunksIntoBlocks(const ChunkedFolderContent&		 Content,
+								 const ChunkedContentLookup&		 Lookup,
+								 std::vector<uint32_t>&				 ChunkIndexes,
+								 std::vector<std::vector<uint32_t>>& OutBlocks);
 	struct GeneratedBlocks
 	{
 		std::vector<ChunkBlockDescription> BlockDescriptions;
diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h
index b0d8ef24c..295d275d1 100644
--- a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h
+++ b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h
@@ -37,6 +37,29 @@ bool				  IterateChunkBlock(const SharedBuffer&															BlockPayload,
 										uint64_t&																	OutHeaderSize);
 std::vector<uint32_t> ReadChunkBlockHeader(const MemoryView BlockView, uint64_t& OutHeaderSize);
 
+struct ReuseBlocksStatistics
+{
+	uint64_t AcceptedChunkCount			 = 0;
+	uint64_t AcceptedByteCount			 = 0;
+	uint64_t AcceptedRawByteCount		 = 0;
+	uint64_t RejectedBlockCount			 = 0;
+	uint64_t RejectedChunkCount			 = 0;
+	uint64_t RejectedByteCount			 = 0;
+	uint64_t AcceptedReduntantChunkCount = 0;
+	uint64_t AcceptedReduntantByteCount	 = 0;
+};
+
+class OperationLogOutput;
+
+std::vector<size_t> FindReuseBlocks(OperationLogOutput&						  Output,
+									const uint8_t							  BlockReuseMinPercentLimit,
+									const bool								  IsVerbose,
+									ReuseBlocksStatistics&					  Stats,
+									const std::vector<ChunkBlockDescription>& KnownBlocks,
+									std::span<const IoHash>					  ChunkHashes,
+									std::span<const uint32_t>				  ChunkIndexes,
+									std::vector<uint32_t>&					  OutUnusedChunkIndexes);
+
 void chunkblock_forcelink();
 
 }  // namespace zen
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h
index 182b64609..008f94351 100644
--- a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h
+++ b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h
@@ -70,7 +70,7 @@ public:
 
 	struct GetKnownBlocksResult : public Result
 	{
-		std::vector<ThinChunkBlockDescription> Blocks;
+		std::vector<ChunkBlockDescription> Blocks;
 	};
 
 	struct RemoteStoreInfo