gracefully handle missing chunks when exporting an oplog (#526)

- Improvement: Gracefully handle missing chunks when exporting an oplog
- Bugfix: Add object validation when replaying oplog to avoid crash on malformed data

3 files changed, 68 insertions, 28 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4fe434ba2..41de81d21 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,10 +1,12 @@
 ##
 - Feature: `zen builds download` and `zen builds ls` now allows multiple wildcards for `--wildcard` and `--exclude-wildcard` separated by semicolons (;)
+- Improvement: Gracefully handle missing chunks when exporting an oplog
 - Bugfix: Fixed invalid namespace+bucket regexes in BuildStore (only fired with new MSVC compiler)
 - Bugfix: `GcScheduler` could delay shutdown in some situations
 - Bugfix: On exit, trace shutdown can happen before all threads completed their TLS cleanup
 - Bugfix: When requesting a value via the GetCacheChunks rpc method, if the request specified a raw hash, only return a hit if the raw hash matches what was in the cache
 - Bugfix: Flush of blockstore file could sometimes cause an error due to a race condition
+- Bugfix: Add object validation when replaying oplog to avoid crash on malformed data
 
 ## 5.7.2
 - Feature: Added `--force` option to `zen builds download` command that forces download of all content ignoring any local state
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 690f697d0..284a32baa 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1043,37 +1043,61 @@ struct ProjectStore::OplogStorage : public RefCounted
 			MemoryView	   OpBufferView = OpBlobsBuffer.MakeView(Entry.Address.Size, OpFileOffset);
 			if (OpBufferView.GetSize() == Entry.Address.Size)
 			{
-				CbObjectView OpView(OpBufferView.GetData());
-				if (OpView.GetSize() != OpBufferView.GetSize())
+				if (CbValidateError Error = ValidateCompactBinary(OpBufferView, CbValidateMode::Default); Error == CbValidateError::None)
 				{
-					ZEN_WARN("oplog '{}/{}': skipping invalid format op - {}. Object payload size {} does not match op data size {}",
-							 m_OwnerOplog->GetOuterProjectIdentifier(),
-							 m_OwnerOplog->OplogId(),
-							 Entry.Lsn.Number,
-							 OpView.GetSize(),
-							 OpBufferView.GetSize());
+					CbObjectView OpView(OpBufferView.GetData());
+					if (OpView.GetSize() != OpBufferView.GetSize())
+					{
+						ZEN_WARN("oplog '{}/{}': skipping invalid format op - {}. Object payload size {} does not match op data size {}",
+								 m_OwnerOplog->GetOuterProjectIdentifier(),
+								 m_OwnerOplog->OplogId(),
+								 Entry.Lsn.Number,
+								 OpView.GetSize(),
+								 OpBufferView.GetSize());
+					}
+					else
+					{
+						Handler(Entry.Lsn, OpView);
+					}
 				}
 				else
 				{
-					Handler(Entry.Lsn, OpView);
+					ZEN_WARN("oplog '{}/{}': skipping invalid format op - {}. Validation error: {}",
+							 m_OwnerOplog->GetOuterProjectIdentifier(),
+							 m_OwnerOplog->OplogId(),
+							 Entry.Lsn.Number,
+							 ToString(Error));
 				}
-				continue;
-			}
-			IoBuffer OpBuffer(Entry.Address.Size);
-			OpBlobsBuffer.Read((void*)OpBuffer.Data(), Entry.Address.Size, OpFileOffset);
-			CbObjectView OpView(OpBuffer.Data());
-			if (OpView.GetSize() != OpBuffer.GetSize())
-			{
-				ZEN_WARN("oplog '{}/{}': skipping invalid format op - {}. Object payload size {} does not match op data size {}",
-						 m_OwnerOplog->GetOuterProjectIdentifier(),
-						 m_OwnerOplog->OplogId(),
-						 Entry.Lsn.Number,
-						 OpView.GetSize(),
-						 OpBuffer.GetSize());
 			}
 			else
 			{
-				Handler(Entry.Lsn, OpView);
+				IoBuffer OpBuffer(Entry.Address.Size);
+				OpBlobsBuffer.Read((void*)OpBuffer.Data(), Entry.Address.Size, OpFileOffset);
+				if (CbValidateError Error = ValidateCompactBinary(OpBufferView, CbValidateMode::Default); Error == CbValidateError::None)
+				{
+					CbObjectView OpView(OpBuffer.Data());
+					if (OpView.GetSize() != OpBuffer.GetSize())
+					{
+						ZEN_WARN("oplog '{}/{}': skipping invalid format op - {}. Object payload size {} does not match op data size {}",
+								 m_OwnerOplog->GetOuterProjectIdentifier(),
+								 m_OwnerOplog->OplogId(),
+								 Entry.Lsn.Number,
+								 OpView.GetSize(),
+								 OpBuffer.GetSize());
+					}
+					else
+					{
+						Handler(Entry.Lsn, OpView);
+					}
+				}
+				else
+				{
+					ZEN_WARN("oplog '{}/{}': skipping invalid format op - {}. Validation error: {}",
+							 m_OwnerOplog->GetOuterProjectIdentifier(),
+							 m_OwnerOplog->OplogId(),
+							 Entry.Lsn.Number,
+							 ToString(Error));
+				}
 			}
 		}
 	}
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index 407a284c1..3d66670dd 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -2011,11 +2011,25 @@ BuildContainer(CidStore&																	ChunkStore,
 					{
 						ChunksInBlock.emplace_back(
 							std::make_pair(RawHash, [&ChunkStore](const IoHash& RawHash) -> std::pair<uint64_t, CompressedBuffer> {
-								IoBuffer		 Chunk = ChunkStore.FindChunkByCid(RawHash);
-								IoHash			 _;
-								uint64_t		 RawSize	= 0;
-								CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunk)), _, RawSize);
-								ZEN_ASSERT(Compressed);
+								IoBuffer Chunk = ChunkStore.FindChunkByCid(RawHash);
+								if (!Chunk)
+								{
+									throw std::runtime_error(fmt::format("Failed to find chunk {} in cid store", RawHash));
+								}
+								IoHash			 ValidateRawHash;
+								uint64_t		 RawSize = 0;
+								CompressedBuffer Compressed =
+									CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunk)), ValidateRawHash, RawSize);
+								if (!Compressed)
+								{
+									throw std::runtime_error(
+										fmt::format("Chunk {} in cid store is malformed (not a compressed buffer)", RawHash));
+								}
+								if (RawHash != ValidateRawHash)
+								{
+									throw std::runtime_error(
+										fmt::format("Chunk {} in cid store is malformed (mismatching raw hash)", RawHash));
+								}
 								return {RawSize, Compressed};
 							}));
 					}