add retry for failed block metadata upload (#445)

* add retry for failed block metadata upload

4 files changed, 60 insertions, 32 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index be640fe31..cd27daa9e 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -2779,14 +2779,18 @@ namespace {
 																									std::vector<CbObject>({BlockMetaData}));
 													}
 
-													Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData);
-													ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})",
-																		BlockHash,
-																		NiceBytes(BlockMetaData.GetSize()));
+													bool MetadataSucceeded =
+														Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData);
+													if (MetadataSucceeded)
+													{
+														ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})",
+																			BlockHash,
+																			NiceBytes(BlockMetaData.GetSize()));
 
-													OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
+														OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
+														UploadStats.BlocksBytes += BlockMetaData.GetSize();
+													}
 
-													UploadStats.BlocksBytes += BlockMetaData.GetSize();
 													UploadStats.BlockCount++;
 													if (UploadStats.BlockCount == NewBlockCount)
 													{
@@ -2988,13 +2992,16 @@ namespace {
 																			std::vector<IoHash>({BlockHash}),
 																			std::vector<CbObject>({BlockMetaData}));
 							}
-							Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData);
-							ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize()));
+							bool MetadataSucceeded = Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData);
+							if (MetadataSucceeded)
+							{
+								ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize()));
 
-							NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
+								NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
+								UploadStats.BlocksBytes += BlockMetaData.GetSize();
+							}
 
 							UploadStats.BlockCount++;
-							UploadStats.BlocksBytes += BlockMetaData.GetSize();
 
 							UploadedBlockCount++;
 							if (UploadedBlockCount == UploadBlockCount && UploadedChunkCount == UploadChunkCount)
@@ -4204,29 +4211,44 @@ namespace {
 
 		if (!NewBlocks.BlockDescriptions.empty() && !AbortFlag)
 		{
-			uint64_t			UploadBlockMetadataCount = 0;
-			std::vector<IoHash> BlockHashes;
-			BlockHashes.reserve(NewBlocks.BlockDescriptions.size());
+			uint64_t  UploadBlockMetadataCount = 0;
 			Stopwatch UploadBlockMetadataTimer;
-			for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++)
+
+			uint32_t FailedMetadataUploadCount = 1;
+			int32_t	 MetadataUploadRetryCount  = 3;
+			while ((MetadataUploadRetryCount-- > 0) && (FailedMetadataUploadCount > 0))
 			{
-				const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash;
-				if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex])
+				FailedMetadataUploadCount = 0;
+				for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++)
 				{
-					const CbObject BlockMetaData =
-						BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]);
-					if (Storage.BuildCacheStorage)
+					if (AbortFlag)
+					{
+						break;
+					}
+					const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash;
+					if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex])
 					{
-						Storage.BuildCacheStorage->PutBlobMetadatas(BuildId,
-																	std::vector<IoHash>({BlockHash}),
-																	std::vector<CbObject>({BlockMetaData}));
+						const CbObject BlockMetaData =
+							BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]);
+						if (Storage.BuildCacheStorage)
+						{
+							Storage.BuildCacheStorage->PutBlobMetadatas(BuildId,
+																		std::vector<IoHash>({BlockHash}),
+																		std::vector<CbObject>({BlockMetaData}));
+						}
+						bool MetadataSucceeded = Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData);
+						if (MetadataSucceeded)
+						{
+							UploadStats.BlocksBytes += BlockMetaData.GetSize();
+							NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
+							UploadBlockMetadataCount++;
+						}
+						else
+						{
+							FailedMetadataUploadCount++;
+						}
 					}
-					Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData);
-					UploadStats.BlocksBytes += BlockMetaData.GetSize();
-					NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
-					UploadBlockMetadataCount++;
 				}
-				BlockHashes.push_back(BlockHash);
 			}
 			if (UploadBlockMetadataCount > 0)
 			{
diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp
index c389d16c5..c2cc5ab3c 100644
--- a/src/zenutil/filebuildstorage.cpp
+++ b/src/zenutil/filebuildstorage.cpp
@@ -450,7 +450,7 @@ public:
 		return {};
 	}
 
-	virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
+	virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
 	{
 		ZEN_TRACE_CPU("FileBuildStorage::PutBlockMetadata");
 		ZEN_UNUSED(BuildId);
@@ -467,6 +467,7 @@ public:
 		m_Stats.TotalBytesWritten += MetaData.GetSize();
 		WriteAsJson(BlockMetaDataPath, MetaData);
 		SimulateLatency(0, 0);
+		return true;
 	}
 
 	virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override
diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h
index 5422c837c..03496f0f9 100644
--- a/src/zenutil/include/zenutil/buildstorage.h
+++ b/src/zenutil/include/zenutil/buildstorage.h
@@ -55,9 +55,9 @@ public:
 																 std::function<void(uint64_t Offset, const IoBuffer& Chunk)>&& OnReceive,
 																 std::function<void()>&& OnComplete) = 0;
 
-	virtual void	 PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0;
-	virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount)										= 0;
-	virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes)					= 0;
+	virtual [[nodiscard]] bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0;
+	virtual CbObject		   FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount)									  = 0;
+	virtual CbObject		   GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes)				  = 0;
 
 	virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map<std::string, double>& FloatStats) = 0;
 };
diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
index 53cad78da..9974725ff 100644
--- a/src/zenutil/jupiter/jupiterbuildstorage.cpp
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -334,7 +334,7 @@ public:
 		return WorkList;
 	}
 
-	virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
+	virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
 	{
 		ZEN_TRACE_CPU("Jupiter::PutBlockMetadata");
 
@@ -346,9 +346,14 @@ public:
 		AddStatistic(PutMetaResult);
 		if (!PutMetaResult.Success)
 		{
+			if (PutMetaResult.ErrorCode == int32_t(HttpResponseCode::NotFound))
+			{
+				return false;
+			}
 			throw std::runtime_error(
 				fmt::format("Failed putting build block metadata: {} ({})", PutMetaResult.Reason, PutMetaResult.ErrorCode));
 		}
+		return true;
 	}
 
 	virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override