convert ZEN_ASSERTs to exception to handle corrupt data gracefully (#760)

* convert ZEN_ASSERTs to exception to handle corrupt data gracefully

3 files changed, 59 insertions, 13 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c2fe710a9..16a6b7fb1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -30,6 +30,7 @@
 
 - Improvement: Reduced time project and project oplogs are locked during GC and Validation
 - Improvement: `zen` now supports additional configuration of logging options, such as `--log-warn=...` for configuring log levels, etc (see `zen --help`)
+- Bugfix: If a corrupted block (or partial block) is downloaded, handle it gracefully and end the download instead of causing an assert
 
 ## 5.7.20
 - Improvement: When validating cache records read from disk we now do a limited validation of the payload to reduce overhead
diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp
index 2319ad66d..ade431393 100644
--- a/src/zenremotestore/builds/buildstorageoperations.cpp
+++ b/src/zenremotestore/builds/buildstorageoperations.cpp
@@ -4083,7 +4083,8 @@ BuildsOperationUpdateFolder::WriteSequenceChunkToCache(BufferedWriteFileCache::L
 }
 
 bool
-BuildsOperationUpdateFolder::GetBlockWriteOps(std::span<const IoHash>		   ChunkRawHashes,
+BuildsOperationUpdateFolder::GetBlockWriteOps(const IoHash&					   BlockRawHash,
+											  std::span<const IoHash>		   ChunkRawHashes,
 											  std::span<const uint32_t>		   ChunkCompressedLengths,
 											  std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
 											  std::span<std::atomic<bool>>	   RemoteChunkIndexNeedsCopyFromSourceFlags,
@@ -4115,9 +4116,34 @@ BuildsOperationUpdateFolder::GetBlockWriteOps(std::span<const IoHash>		   ChunkR
 					uint64_t		 VerifyChunkSize;
 					CompressedBuffer CompressedChunk =
 						CompressedBuffer::FromCompressed(SharedBuffer::MakeView(ChunkMemoryView), VerifyChunkHash, VerifyChunkSize);
-					ZEN_ASSERT(CompressedChunk);
-					ZEN_ASSERT(VerifyChunkHash == ChunkHash);
-					ZEN_ASSERT(VerifyChunkSize == m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]);
+					if (!CompressedChunk)
+					{
+						throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} is not a valid compressed buffer",
+															 ChunkHash,
+															 OffsetInBlock,
+															 ChunkCompressedSize,
+															 BlockRawHash));
+					}
+					if (VerifyChunkHash != ChunkHash)
+					{
+						throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} has a mismatching content hash {}",
+															 ChunkHash,
+															 OffsetInBlock,
+															 ChunkCompressedSize,
+															 BlockRawHash,
+															 VerifyChunkHash));
+					}
+					if (VerifyChunkSize != m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex])
+					{
+						throw std::runtime_error(
+							fmt::format("Chunk {} at {}, size {} in block {} has a mismatching raw size {}, expected {}",
+										ChunkHash,
+										OffsetInBlock,
+										ChunkCompressedSize,
+										BlockRawHash,
+										VerifyChunkSize,
+										m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]));
+					}
 
 					OodleCompressor		  ChunkCompressor;
 					OodleCompressionLevel ChunkCompressionLevel;
@@ -4138,7 +4164,18 @@ BuildsOperationUpdateFolder::GetBlockWriteOps(std::span<const IoHash>		   ChunkR
 					{
 						Decompressed = CompressedChunk.Decompress().AsIoBuffer();
 					}
-					ZEN_ASSERT(Decompressed.GetSize() == m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]);
+
+					if (Decompressed.GetSize() != m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex])
+					{
+						throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} decompressed to size {}, expected {}",
+															 ChunkHash,
+															 OffsetInBlock,
+															 ChunkCompressedSize,
+															 BlockRawHash,
+															 Decompressed.GetSize(),
+															 m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]));
+					}
+
 					ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed));
 					for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs)
 					{
@@ -4237,7 +4274,8 @@ BuildsOperationUpdateFolder::WriteChunksBlockToCache(const ChunkBlockDescription
 		const std::vector<uint32_t> ChunkCompressedLengths =
 			ReadChunkBlockHeader(BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder()), HeaderSize);
 
-		if (GetBlockWriteOps(BlockDescription.ChunkRawHashes,
+		if (GetBlockWriteOps(BlockDescription.BlockHash,
+							 BlockDescription.ChunkRawHashes,
 							 ChunkCompressedLengths,
 							 SequenceIndexChunksLeftToWriteCounters,
 							 RemoteChunkIndexNeedsCopyFromSourceFlags,
@@ -4252,7 +4290,8 @@ BuildsOperationUpdateFolder::WriteChunksBlockToCache(const ChunkBlockDescription
 		return false;
 	}
 
-	if (GetBlockWriteOps(BlockDescription.ChunkRawHashes,
+	if (GetBlockWriteOps(BlockDescription.BlockHash,
+						 BlockDescription.ChunkRawHashes,
 						 BlockDescription.ChunkCompressedLengths,
 						 SequenceIndexChunksLeftToWriteCounters,
 						 RemoteChunkIndexNeedsCopyFromSourceFlags,
@@ -4283,7 +4322,8 @@ BuildsOperationUpdateFolder::WritePartialBlockChunksToCache(const ChunkBlockDesc
 	const MemoryView BlockView		   = BlockMemoryBuffer.GetView();
 
 	BlockWriteOps Ops;
-	if (GetBlockWriteOps(BlockDescription.ChunkRawHashes,
+	if (GetBlockWriteOps(BlockDescription.BlockHash,
+						 BlockDescription.ChunkRawHashes,
 						 BlockDescription.ChunkCompressedLengths,
 						 SequenceIndexChunksLeftToWriteCounters,
 						 RemoteChunkIndexNeedsCopyFromSourceFlags,
@@ -5334,6 +5374,13 @@ BuildsOperationUploadFolder::FetchChunk(const ChunkedFolderContent& Content,
 	ZEN_ASSERT(!ChunkLocations.empty());
 	CompositeBuffer Chunk =
 		OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, ChunkLocations[0].Offset, Content.ChunkedContent.ChunkRawSizes[ChunkIndex]);
+	if (!Chunk)
+	{
+		throw std::runtime_error(fmt::format("Unable to read chunk at {}, size {} from '{}'",
+											 ChunkLocations[0].Offset,
+											 Content.ChunkedContent.ChunkRawSizes[ChunkIndex],
+											 Content.Paths[Lookup.SequenceIndexFirstPathIndex[ChunkLocations[0].SequenceIndex]]));
+	}
 	ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == ChunkHash);
 	return Chunk;
 };
@@ -5362,10 +5409,7 @@ BuildsOperationUploadFolder::GenerateBlock(const ChunkedFolderContent&	Content,
 			Content.ChunkedContent.ChunkHashes[ChunkIndex],
 			[this, &Content, &Lookup, &OpenFileCache, ChunkIndex](const IoHash& ChunkHash) -> std::pair<uint64_t, CompressedBuffer> {
 				CompositeBuffer Chunk = FetchChunk(Content, Lookup, ChunkHash, OpenFileCache);
-				if (!Chunk)
-				{
-					ZEN_ASSERT(false);
-				}
+				ZEN_ASSERT(Chunk);
 				uint64_t RawSize = Chunk.GetSize();
 
 				const bool ShouldCompressChunk = RawSize >= m_Options.MinimumSizeForCompressInBlock &&
diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
index 6304159ae..9e5bf8d91 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
@@ -339,7 +339,8 @@ private:
 								   const uint64_t				  FileOffset,
 								   const uint32_t				  PathIndex);
 
-	bool GetBlockWriteOps(std::span<const IoHash>		   ChunkRawHashes,
+	bool GetBlockWriteOps(const IoHash&					   BlockRawHash,
+						  std::span<const IoHash>		   ChunkRawHashes,
 						  std::span<const uint32_t>		   ChunkCompressedLengths,
 						  std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
 						  std::span<std::atomic<bool>>	   RemoteChunkIndexNeedsCopyFromSourceFlags,