gracefully handle missing chunks when exporting an oplog (#526)

- Improvement: Gracefully handle missing chunks when exporting an oplog
- Bugfix: Add object validation when replaying oplog to avoid crash on malformed data

2 files changed, 66 insertions, 28 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 690f697d0..284a32baa 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1043,37 +1043,61 @@ struct ProjectStore::OplogStorage : public RefCounted
 			MemoryView	   OpBufferView = OpBlobsBuffer.MakeView(Entry.Address.Size, OpFileOffset);
 			if (OpBufferView.GetSize() == Entry.Address.Size)
 			{
-				CbObjectView OpView(OpBufferView.GetData());
-				if (OpView.GetSize() != OpBufferView.GetSize())
+				if (CbValidateError Error = ValidateCompactBinary(OpBufferView, CbValidateMode::Default); Error == CbValidateError::None)
 				{
-					ZEN_WARN("oplog '{}/{}': skipping invalid format op - {}. Object payload size {} does not match op data size {}",
-							 m_OwnerOplog->GetOuterProjectIdentifier(),
-							 m_OwnerOplog->OplogId(),
-							 Entry.Lsn.Number,
-							 OpView.GetSize(),
-							 OpBufferView.GetSize());
+					CbObjectView OpView(OpBufferView.GetData());
+					if (OpView.GetSize() != OpBufferView.GetSize())
+					{
+						ZEN_WARN("oplog '{}/{}': skipping invalid format op - {}. Object payload size {} does not match op data size {}",
+								 m_OwnerOplog->GetOuterProjectIdentifier(),
+								 m_OwnerOplog->OplogId(),
+								 Entry.Lsn.Number,
+								 OpView.GetSize(),
+								 OpBufferView.GetSize());
+					}
+					else
+					{
+						Handler(Entry.Lsn, OpView);
+					}
 				}
 				else
 				{
-					Handler(Entry.Lsn, OpView);
+					ZEN_WARN("oplog '{}/{}': skipping invalid format op - {}. Validation error: {}",
+							 m_OwnerOplog->GetOuterProjectIdentifier(),
+							 m_OwnerOplog->OplogId(),
+							 Entry.Lsn.Number,
+							 ToString(Error));
 				}
-				continue;
-			}
-			IoBuffer OpBuffer(Entry.Address.Size);
-			OpBlobsBuffer.Read((void*)OpBuffer.Data(), Entry.Address.Size, OpFileOffset);
-			CbObjectView OpView(OpBuffer.Data());
-			if (OpView.GetSize() != OpBuffer.GetSize())
-			{
-				ZEN_WARN("oplog '{}/{}': skipping invalid format op - {}. Object payload size {} does not match op data size {}",
-						 m_OwnerOplog->GetOuterProjectIdentifier(),
-						 m_OwnerOplog->OplogId(),
-						 Entry.Lsn.Number,
-						 OpView.GetSize(),
-						 OpBuffer.GetSize());
 			}
 			else
 			{
-				Handler(Entry.Lsn, OpView);
+				IoBuffer OpBuffer(Entry.Address.Size);
+				OpBlobsBuffer.Read((void*)OpBuffer.Data(), Entry.Address.Size, OpFileOffset);
+				if (CbValidateError Error = ValidateCompactBinary(OpBufferView, CbValidateMode::Default); Error == CbValidateError::None)
+				{
+					CbObjectView OpView(OpBuffer.Data());
+					if (OpView.GetSize() != OpBuffer.GetSize())
+					{
+						ZEN_WARN("oplog '{}/{}': skipping invalid format op - {}. Object payload size {} does not match op data size {}",
+								 m_OwnerOplog->GetOuterProjectIdentifier(),
+								 m_OwnerOplog->OplogId(),
+								 Entry.Lsn.Number,
+								 OpView.GetSize(),
+								 OpBuffer.GetSize());
+					}
+					else
+					{
+						Handler(Entry.Lsn, OpView);
+					}
+				}
+				else
+				{
+					ZEN_WARN("oplog '{}/{}': skipping invalid format op - {}. Validation error: {}",
+							 m_OwnerOplog->GetOuterProjectIdentifier(),
+							 m_OwnerOplog->OplogId(),
+							 Entry.Lsn.Number,
+							 ToString(Error));
+				}
 			}
 		}
 	}
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index 407a284c1..3d66670dd 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -2011,11 +2011,25 @@ BuildContainer(CidStore&																	ChunkStore,
 					{
 						ChunksInBlock.emplace_back(
 							std::make_pair(RawHash, [&ChunkStore](const IoHash& RawHash) -> std::pair<uint64_t, CompressedBuffer> {
-								IoBuffer		 Chunk = ChunkStore.FindChunkByCid(RawHash);
-								IoHash			 _;
-								uint64_t		 RawSize	= 0;
-								CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunk)), _, RawSize);
-								ZEN_ASSERT(Compressed);
+								IoBuffer Chunk = ChunkStore.FindChunkByCid(RawHash);
+								if (!Chunk)
+								{
+									throw std::runtime_error(fmt::format("Failed to find chunk {} in cid store", RawHash));
+								}
+								IoHash			 ValidateRawHash;
+								uint64_t		 RawSize = 0;
+								CompressedBuffer Compressed =
+									CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunk)), ValidateRawHash, RawSize);
+								if (!Compressed)
+								{
+									throw std::runtime_error(
+										fmt::format("Chunk {} in cid store is malformed (not a compressed buffer)", RawHash));
+								}
+								if (RawHash != ValidateRawHash)
+								{
+									throw std::runtime_error(
+										fmt::format("Chunk {} in cid store is malformed (mismatching raw hash)", RawHash));
+								}
 								return {RawSize, Compressed};
 							}));
 					}