make sure new blocks generated by a part is accessible to following parts (#732)

2 files changed, 92 insertions, 81 deletions

diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp
index 306e24b9a..a8169538b 100644
--- a/src/zenremotestore/builds/buildstorageoperations.cpp
+++ b/src/zenremotestore/builds/buildstorageoperations.cpp
@@ -4598,10 +4598,13 @@ BuildsOperationUploadFolder::PrepareBuild()
 		ZEN_TRACE_CPU("CreateBuild");
 
 		Stopwatch PutBuildTimer;
-		CbObject  PutBuildResult		   = m_Storage.BuildStorage->PutBuild(m_BuildId, m_MetaData);
-		Result.PrepareBuildTimeMs		   = PutBuildTimer.GetElapsedTimeMs();
-		Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize);
-		Result.PayloadSize				   = m_MetaData.GetSize();
+		CbObject  PutBuildResult  = m_Storage.BuildStorage->PutBuild(m_BuildId, m_MetaData);
+		Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs();
+		if (auto ChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(); ChunkSize != 0)
+		{
+			Result.PreferredMultipartChunkSize = ChunkSize;
+		}
+		Result.PayloadSize = m_MetaData.GetSize();
 	}
 	else
 	{
@@ -5577,30 +5580,36 @@ BuildsOperationUploadFolder::UploadBuildPart(ChunkingController& ChunkController
 
 	if (PartIndex == 0)
 	{
-		m_PrepBuildResult = m_PrepBuildResultFuture.get();
+		const PrepareBuildResult PrepBuildResult = m_PrepBuildResultFuture.get();
 
-		m_FindBlocksStats.FindBlockTimeMS = m_PrepBuildResult.ElapsedTimeMs;
-		m_FindBlocksStats.FoundBlockCount = m_PrepBuildResult.KnownBlocks.size();
-	}
+		m_FindBlocksStats.FindBlockTimeMS = PrepBuildResult.ElapsedTimeMs;
+		m_FindBlocksStats.FoundBlockCount = PrepBuildResult.KnownBlocks.size();
 
-	if (!m_Options.IsQuiet)
-	{
-		ZEN_OPERATION_LOG_INFO(m_LogOutput,
-							   "Build prepare took {}. {} took {}, payload size {}{}",
-							   NiceTimeSpanMs(m_PrepBuildResult.ElapsedTimeMs),
-							   m_CreateBuild ? "PutBuild" : "GetBuild",
-							   NiceTimeSpanMs(m_PrepBuildResult.PrepareBuildTimeMs),
-							   NiceBytes(m_PrepBuildResult.PayloadSize),
-							   m_Options.IgnoreExistingBlocks ? ""
-															  : fmt::format(". Found {} blocks in {}",
-																			m_PrepBuildResult.KnownBlocks.size(),
-																			NiceTimeSpanMs(m_PrepBuildResult.FindBlocksTimeMs)));
+		if (!m_Options.IsQuiet)
+		{
+			ZEN_OPERATION_LOG_INFO(m_LogOutput,
+								   "Build prepare took {}. {} took {}, payload size {}{}",
+								   NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs),
+								   m_CreateBuild ? "PutBuild" : "GetBuild",
+								   NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs),
+								   NiceBytes(PrepBuildResult.PayloadSize),
+								   m_Options.IgnoreExistingBlocks ? ""
+																  : fmt::format(". Found {} blocks in {}",
+																				PrepBuildResult.KnownBlocks.size(),
+																				NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs)));
+		}
+
+		m_PreferredMultipartChunkSize = PrepBuildResult.PreferredMultipartChunkSize;
+
+		m_LargeAttachmentSize = m_Options.AllowMultiparts ? m_PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
+
+		m_KnownBlocks = std::move(PrepBuildResult.KnownBlocks);
 	}
 
-	m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::CalculateDelta, StepCount);
+	ZEN_ASSERT(m_PreferredMultipartChunkSize != 0);
+	ZEN_ASSERT(m_LargeAttachmentSize != 0);
 
-	const std::uint64_t LargeAttachmentSize =
-		m_Options.AllowMultiparts ? m_PrepBuildResult.PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
+	m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::CalculateDelta, StepCount);
 
 	Stopwatch BlockArrangeTimer;
 
@@ -5639,13 +5648,13 @@ BuildsOperationUploadFolder::UploadBuildPart(ChunkingController& ChunkController
 												m_Options.BlockReuseMinPercentLimit,
 												m_Options.IsVerbose,
 												ReuseBlocksStats,
-												m_PrepBuildResult.KnownBlocks,
+												m_KnownBlocks,
 												LocalContent.ChunkedContent.ChunkHashes,
 												BlockChunkIndexes,
 												NewBlockChunkIndexes);
 			FindBlocksStats.AcceptedBlockCount += ReuseBlockIndexes.size();
 
-			for (const ChunkBlockDescription& Description : m_PrepBuildResult.KnownBlocks)
+			for (const ChunkBlockDescription& Description : m_KnownBlocks)
 			{
 				for (uint32_t ChunkRawLength : Description.ChunkRawLengths)
 				{
@@ -5767,8 +5776,8 @@ BuildsOperationUploadFolder::UploadBuildPart(ChunkingController& ChunkController
 		AllChunkBlockDescriptions.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size());
 		for (size_t ReuseBlockIndex : ReuseBlockIndexes)
 		{
-			AllChunkBlockDescriptions.push_back(m_PrepBuildResult.KnownBlocks[ReuseBlockIndex]);
-			AllChunkBlockHashes.push_back(m_PrepBuildResult.KnownBlocks[ReuseBlockIndex].BlockHash);
+			AllChunkBlockDescriptions.push_back(m_KnownBlocks[ReuseBlockIndex]);
+			AllChunkBlockHashes.push_back(m_KnownBlocks[ReuseBlockIndex].BlockHash);
 		}
 		AllChunkBlockDescriptions.insert(AllChunkBlockDescriptions.end(),
 										 NewBlocks.BlockDescriptions.begin(),
@@ -5777,6 +5786,7 @@ BuildsOperationUploadFolder::UploadBuildPart(ChunkingController& ChunkController
 		{
 			AllChunkBlockHashes.push_back(BlockDescription.BlockHash);
 		}
+
 		std::vector<IoHash> AbsoluteChunkHashes;
 		if (m_Options.DoExtraContentValidation)
 		{
@@ -5920,58 +5930,54 @@ BuildsOperationUploadFolder::UploadBuildPart(ChunkingController& ChunkController
 	}
 	IoHash PartHash = PutBuildPartResult.first;
 
-	auto UploadAttachments = [this,
-							  &LooseChunksStats,
-							  &UploadStats,
-							  &LocalContent,
-							  &LocalLookup,
-							  &NewBlockChunks,
-							  &NewBlocks,
-							  &LooseChunkIndexes,
-							  &LargeAttachmentSize](std::span<IoHash> RawHashes, std::vector<IoHash>& OutUnknownChunks) {
-		if (!m_AbortFlag)
-		{
-			UploadStatistics	  TempUploadStats;
-			LooseChunksStatistics TempLooseChunksStats;
-
-			Stopwatch TempUploadTimer;
-			auto	  __ = MakeGuard([&]() {
-				 if (!m_Options.IsQuiet)
-				 {
-					 uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs();
-					 ZEN_OPERATION_LOG_INFO(m_LogOutput,
-											"Uploaded {} ({}) blocks. "
-												"Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. "
-												"Transferred {} ({}bits/s) in {}",
-											TempUploadStats.BlockCount.load(),
-											NiceBytes(TempUploadStats.BlocksBytes),
-
-											TempLooseChunksStats.CompressedChunkCount.load(),
-											NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()),
-											NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS,
-																	  TempLooseChunksStats.ChunkByteCount)),
-											TempUploadStats.ChunkCount.load(),
-											NiceBytes(TempUploadStats.ChunksBytes),
-
-											NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes),
-											NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)),
-											NiceTimeSpanMs(TempChunkUploadTimeUs / 1000));
-				 }
-			 });
-			UploadPartBlobs(LocalContent,
-							LocalLookup,
-							RawHashes,
-							NewBlockChunks,
-							NewBlocks,
-							LooseChunkIndexes,
-							LargeAttachmentSize,
-							TempUploadStats,
-							TempLooseChunksStats,
-							OutUnknownChunks);
-			UploadStats += TempUploadStats;
-			LooseChunksStats += TempLooseChunksStats;
-		}
-	};
+	auto UploadAttachments =
+		[this, &LooseChunksStats, &UploadStats, &LocalContent, &LocalLookup, &NewBlockChunks, &NewBlocks, &LooseChunkIndexes](
+			std::span<IoHash>	 RawHashes,
+			std::vector<IoHash>& OutUnknownChunks) {
+			if (!m_AbortFlag)
+			{
+				UploadStatistics	  TempUploadStats;
+				LooseChunksStatistics TempLooseChunksStats;
+
+				Stopwatch TempUploadTimer;
+				auto	  __ = MakeGuard([&]() {
+					 if (!m_Options.IsQuiet)
+					 {
+						 uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs();
+						 ZEN_OPERATION_LOG_INFO(
+							 m_LogOutput,
+							 "Uploaded {} ({}) blocks. "
+								 "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. "
+								 "Transferred {} ({}bits/s) in {}",
+							 TempUploadStats.BlockCount.load(),
+							 NiceBytes(TempUploadStats.BlocksBytes),
+
+							 TempLooseChunksStats.CompressedChunkCount.load(),
+							 NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()),
+							 NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS,
+													   TempLooseChunksStats.ChunkByteCount)),
+							 TempUploadStats.ChunkCount.load(),
+							 NiceBytes(TempUploadStats.ChunksBytes),
+
+							 NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes),
+							 NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)),
+							 NiceTimeSpanMs(TempChunkUploadTimeUs / 1000));
+					 }
+				 });
+				UploadPartBlobs(LocalContent,
+								LocalLookup,
+								RawHashes,
+								NewBlockChunks,
+								NewBlocks,
+								LooseChunkIndexes,
+								m_LargeAttachmentSize,
+								TempUploadStats,
+								TempLooseChunksStats,
+								OutUnknownChunks);
+				UploadStats += TempUploadStats;
+				LooseChunksStats += TempLooseChunksStats;
+			}
+		};
 
 	m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::UploadAttachments, StepCount);
 
@@ -6125,6 +6131,9 @@ BuildsOperationUploadFolder::UploadBuildPart(ChunkingController& ChunkController
 									   NiceTimeSpanMs(ElapsedUS / 1000));
 			}
 		}
+
+		// The newly generated blocks are now known blocks so the next part upload can use those blocks as well
+		m_KnownBlocks.insert(m_KnownBlocks.end(), NewBlocks.BlockDescriptions.begin(), NewBlocks.BlockDescriptions.end());
 	}
 
 	m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::PutBuildPartStats, StepCount);
diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
index f9fe949a6..057ad1a10 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
@@ -695,8 +695,10 @@ private:
 
 	tsl::robin_set<uint32_t> m_NonCompressableExtensionHashes;
 
-	std::future<PrepareBuildResult> m_PrepBuildResultFuture;
-	PrepareBuildResult				m_PrepBuildResult;
+	std::future<PrepareBuildResult>	   m_PrepBuildResultFuture;
+	std::vector<ChunkBlockDescription> m_KnownBlocks;
+	uint64_t						   m_PreferredMultipartChunkSize = 0;
+	uint64_t						   m_LargeAttachmentSize		 = 0;
 };
 
 struct ValidateStatistics