async find blocks (#300)

* put/get build and find blocks while scanning local folder when uploading

* changelog

* remove redundant move

1 files changed, 80 insertions, 42 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 56c3c3c4f..e03175256 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -81,6 +81,8 @@ namespace {
 	const ChunksBlockParameters DefaultChunksBlockParams{.MaxBlockSize		= 32u * 1024u * 1024u,
 														 .MaxChunkEmbedSize = DefaultChunkedParams.MaxSize};
 
+	const uint64_t DefaultPreferredMultipartChunkSize = 32u * 1024u * 1024u;
+
 	const double DefaultLatency		  = 0;	// .0010;
 	const double DefaultDelayPerKBSec = 0;	// 0.00005;
 
@@ -2309,6 +2311,61 @@ namespace {
 
 		CbObject ChunkerParameters;
 
+		struct PrepareBuildResult
+		{
+			std::vector<ChunkBlockDescription> KnownBlocks;
+			uint64_t						   PreferredMultipartChunkSize = DefaultPreferredMultipartChunkSize;
+			uint64_t						   PayloadSize				   = 0;
+			uint64_t						   PrepareBuildTimeMs		   = 0;
+			uint64_t						   FindBlocksTimeMs			   = 0;
+			uint64_t						   ElapsedTimeMs			   = 0;
+		};
+
+		FindBlocksStatistics FindBlocksStats;
+
+		std::future<PrepareBuildResult> PrepBuildResultFuture =
+			GetSmallWorkerPool(EWorkloadType::Burst)
+				.EnqueueTask(std::packaged_task<PrepareBuildResult()>{
+					[&Storage, BuildId, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] {
+						PrepareBuildResult Result;
+						Stopwatch		   Timer;
+						if (CreateBuild)
+						{
+							Stopwatch PutBuildTimer;
+							CbObject  PutBuildResult		   = Storage.PutBuild(BuildId, MetaData);
+							Result.PrepareBuildTimeMs		   = PutBuildTimer.GetElapsedTimeMs();
+							Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize);
+							Result.PayloadSize				   = MetaData.GetSize();
+						}
+						else
+						{
+							Stopwatch GetBuildTimer;
+							CbObject  Build			  = Storage.GetBuild(BuildId);
+							Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs();
+							Result.PayloadSize		  = Build.GetSize();
+							if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0)
+							{
+								Result.PreferredMultipartChunkSize = ChunkSize;
+							}
+							else if (AllowMultiparts)
+							{
+								ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'",
+										 NiceBytes(Result.PreferredMultipartChunkSize));
+							}
+						}
+
+						if (!IgnoreExistingBlocks)
+						{
+							Stopwatch KnownBlocksTimer;
+							Result.KnownBlocks				= Storage.FindBlocks(BuildId);
+							FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs();
+							FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size();
+							Result.FindBlocksTimeMs			= KnownBlocksTimer.GetElapsedTimeMs();
+						}
+						Result.ElapsedTimeMs = Timer.GetElapsedTimeMs();
+						return Result;
+					}});
+
 		ChunkedFolderContent LocalContent;
 
 		GetFolderContentStatistics LocalFolderScanStats;
@@ -2517,43 +2574,29 @@ namespace {
 						NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
 		}
 
-		const ChunkedContentLookup LocalLookup				   = BuildChunkedContentLookup(LocalContent);
-		std::uint64_t			   PreferredMultipartChunkSize = 32u * 1024u * 1024u;
-
-		if (CreateBuild)
-		{
-			Stopwatch PutBuildTimer;
-			CbObject  PutBuildResult = Storage.PutBuild(BuildId, MetaData);
-			ZEN_CONSOLE("PutBuild took {}. Payload size: {}",
-						NiceTimeSpanMs(PutBuildTimer.GetElapsedTimeMs()),
-						NiceBytes(MetaData.GetSize()));
-			PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(PreferredMultipartChunkSize);
-		}
-		else
-		{
-			Stopwatch GetBuildTimer;
-			CbObject  Build = Storage.GetBuild(BuildId);
-			ZEN_CONSOLE("GetBuild took {}. Payload size: {}", NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), NiceBytes(Build.GetSize()));
-			if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0)
-			{
-				PreferredMultipartChunkSize = ChunkSize;
-			}
-			else if (AllowMultiparts)
-			{
-				ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", NiceBytes(PreferredMultipartChunkSize));
-			}
-		}
-
-		const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
+		const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent);
 
-		FindBlocksStatistics	 FindBlocksStats;
 		GenerateBlocksStatistics GenerateBlocksStats;
 		LooseChunksStatistics	 LooseChunksStats;
 
-		std::vector<ChunkBlockDescription> KnownBlocks;
-		std::vector<size_t>				   ReuseBlockIndexes;
-		std::vector<uint32_t>			   NewBlockChunkIndexes;
-		Stopwatch						   BlockArrangeTimer;
+		std::vector<size_t>	  ReuseBlockIndexes;
+		std::vector<uint32_t> NewBlockChunkIndexes;
+
+		PrepareBuildResult PrepBuildResult = PrepBuildResultFuture.get();
+
+		ZEN_CONSOLE("Build prepare took {}. {} took {}, payload size {}{}",
+					NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs),
+					CreateBuild ? "PutBuild" : "GetBuild",
+					NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs),
+					NiceBytes(PrepBuildResult.PayloadSize),
+					IgnoreExistingBlocks ? ""
+										 : fmt::format(". Found {} blocks in {}",
+													   PrepBuildResult.KnownBlocks.size(),
+													   NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs)));
+
+		const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PrepBuildResult.PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
+
+		Stopwatch BlockArrangeTimer;
 
 		std::vector<std::uint32_t> LooseChunkIndexes;
 		{
@@ -2583,12 +2626,7 @@ namespace {
 			}
 			else
 			{
-				Stopwatch KnownBlocksTimer;
-				KnownBlocks						= Storage.FindBlocks(BuildId);
-				FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs();
-				FindBlocksStats.FoundBlockCount = KnownBlocks.size();
-
-				ReuseBlockIndexes				   = FindReuseBlocks(KnownBlocks,
+				ReuseBlockIndexes				   = FindReuseBlocks(PrepBuildResult.KnownBlocks,
 													 LocalContent.ChunkedContent.ChunkHashes,
 													 BlockChunkIndexes,
 													 BlockReuseMinPercentLimit,
@@ -2596,7 +2634,7 @@ namespace {
 													 FindBlocksStats);
 				FindBlocksStats.AcceptedBlockCount = ReuseBlockIndexes.size();
 
-				for (const ChunkBlockDescription& Description : KnownBlocks)
+				for (const ChunkBlockDescription& Description : PrepBuildResult.KnownBlocks)
 				{
 					for (uint32_t ChunkRawLength : Description.ChunkRawLengths)
 					{
@@ -2708,8 +2746,8 @@ namespace {
 			AllChunkBlockDescriptions.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size());
 			for (size_t ReuseBlockIndex : ReuseBlockIndexes)
 			{
-				AllChunkBlockDescriptions.push_back(KnownBlocks[ReuseBlockIndex]);
-				AllChunkBlockHashes.push_back(KnownBlocks[ReuseBlockIndex].BlockHash);
+				AllChunkBlockDescriptions.push_back(PrepBuildResult.KnownBlocks[ReuseBlockIndex]);
+				AllChunkBlockHashes.push_back(PrepBuildResult.KnownBlocks[ReuseBlockIndex].BlockHash);
 			}
 			AllChunkBlockDescriptions.insert(AllChunkBlockDescriptions.end(),
 											 NewBlocks.BlockDescriptions.begin(),