add --ignore-missing-attachments to oplog-import command (#637)

1 files changed, 59 insertions, 34 deletions

diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index 826c8ff51..a8f4c5106 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -1517,6 +1517,7 @@ LoadOplog(CidStore&			   ChunkStore,
 		  RemoteProjectStore&  RemoteStore,
 		  ProjectStore::Oplog& Oplog,
 		  bool				   ForceDownload,
+		  bool				   IgnoreMissingAttachments,
 		  JobContext*		   OptionalContext)
 {
 	using namespace std::literals;
@@ -1547,43 +1548,55 @@ LoadOplog(CidStore&			   ChunkStore,
 	auto HasAttachment = [&ChunkStore, ForceDownload](const IoHash& RawHash) {
 		return !ForceDownload && ChunkStore.ContainsChunk(RawHash);
 	};
-	auto OnNeedBlock = [&RemoteStore, &ChunkStore, &WorkerPool, &ChunksInBlocks, &AttachmentsWorkLatch, &AttachmentCount, &RemoteResult](
-						   const IoHash&		 BlockHash,
-						   std::vector<IoHash>&& Chunks) {
+	auto OnNeedBlock = [&RemoteStore,
+						&ChunkStore,
+						&WorkerPool,
+						&ChunksInBlocks,
+						&AttachmentsWorkLatch,
+						&AttachmentCount,
+						&RemoteResult,
+						IgnoreMissingAttachments](const IoHash& BlockHash, std::vector<IoHash>&& Chunks) {
+		if (RemoteResult.IsError())
+		{
+			return;
+		}
 		if (BlockHash == IoHash::Zero)
 		{
 			AttachmentsWorkLatch.AddCount(1);
 			AttachmentCount.fetch_add(1);
-			WorkerPool.ScheduleWork([&RemoteStore, &ChunkStore, &AttachmentsWorkLatch, &RemoteResult, Chunks = std::move(Chunks)]() {
-				auto _ = MakeGuard([&AttachmentsWorkLatch] { AttachmentsWorkLatch.CountDown(); });
-				if (RemoteResult.IsError())
-				{
-					return;
-				}
-
-				RemoteProjectStore::LoadAttachmentsResult Result = RemoteStore.LoadAttachments(Chunks);
-				if (Result.ErrorCode)
-				{
-					RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
-					ZEN_WARN("Failed to load attachments with {} chunks ({}). Reason: '{}'",
-							 Chunks.size(),
-							 RemoteResult.GetError(),
-							 RemoteResult.GetErrorReason());
-					return;
-				}
-				ZEN_DEBUG("Loaded {} bulk attachments in {}",
-						  Chunks.size(),
-						  NiceTimeSpanMs(static_cast<uint64_t>(Result.ElapsedSeconds * 1000)));
-				for (const auto& It : Result.Chunks)
-				{
-					ChunkStore.AddChunk(It.second.GetCompressed().Flatten().AsIoBuffer(), It.first, CidStore::InsertMode::kCopyOnly);
-				}
-			});
+			WorkerPool.ScheduleWork(
+				[&RemoteStore, &ChunkStore, &AttachmentsWorkLatch, &RemoteResult, Chunks = std::move(Chunks), IgnoreMissingAttachments]() {
+					auto _ = MakeGuard([&AttachmentsWorkLatch] { AttachmentsWorkLatch.CountDown(); });
+					if (RemoteResult.IsError())
+					{
+						return;
+					}
+					RemoteProjectStore::LoadAttachmentsResult Result = RemoteStore.LoadAttachments(Chunks);
+					if (Result.ErrorCode)
+					{
+						ZEN_WARN("Failed to load attachments with {} chunks ({}). Reason: '{}'",
+								 Chunks.size(),
+								 RemoteResult.GetError(),
+								 RemoteResult.GetErrorReason());
+						if (!IgnoreMissingAttachments)
+						{
+							RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
+						}
+						return;
+					}
+					ZEN_DEBUG("Loaded {} bulk attachments in {}",
+							  Chunks.size(),
+							  NiceTimeSpanMs(static_cast<uint64_t>(Result.ElapsedSeconds * 1000)));
+					for (const auto& It : Result.Chunks)
+					{
+						ChunkStore.AddChunk(It.second.GetCompressed().Flatten().AsIoBuffer(), It.first, CidStore::InsertMode::kCopyOnly);
+					}
+				});
 			return;
 		}
 		AttachmentsWorkLatch.AddCount(1);
 		AttachmentCount.fetch_add(1);
-		WorkerPool.ScheduleWork([&AttachmentsWorkLatch, &ChunkStore, &RemoteStore, BlockHash, &RemoteResult]() {
+		WorkerPool.ScheduleWork([&AttachmentsWorkLatch, &ChunkStore, &RemoteStore, BlockHash, &RemoteResult, IgnoreMissingAttachments]() {
 			auto _ = MakeGuard([&AttachmentsWorkLatch] { AttachmentsWorkLatch.CountDown(); });
 			if (RemoteResult.IsError())
 			{
@@ -1592,11 +1605,14 @@ LoadOplog(CidStore&			   ChunkStore,
 			RemoteProjectStore::LoadAttachmentResult BlockResult = RemoteStore.LoadAttachment(BlockHash);
 			if (BlockResult.ErrorCode)
 			{
-				RemoteResult.SetError(BlockResult.ErrorCode, BlockResult.Reason, BlockResult.Text);
 				ZEN_WARN("Failed to load oplog container, missing attachment {} ({}). Reason: '{}'",
 						 BlockHash,
 						 RemoteResult.GetError(),
 						 RemoteResult.GetErrorReason());
+				if (!IgnoreMissingAttachments)
+				{
+					RemoteResult.SetError(BlockResult.ErrorCode, BlockResult.Reason, BlockResult.Text);
+				}
 				return;
 			}
 			ZEN_DEBUG("Loaded block attachment in {}", NiceTimeSpanMs(static_cast<uint64_t>(BlockResult.ElapsedSeconds * 1000)));
@@ -1617,8 +1633,14 @@ LoadOplog(CidStore&			   ChunkStore,
 		});
 	};
 
-	auto OnNeedAttachment = [&RemoteStore, &ChunkStore, &WorkerPool, &AttachmentsWorkLatch, &RemoteResult, &Attachments, &AttachmentCount](
-								const IoHash& RawHash) {
+	auto OnNeedAttachment = [&RemoteStore,
+							 &ChunkStore,
+							 &WorkerPool,
+							 &AttachmentsWorkLatch,
+							 &RemoteResult,
+							 &Attachments,
+							 &AttachmentCount,
+							 IgnoreMissingAttachments](const IoHash& RawHash) {
 		if (!Attachments.insert(RawHash).second)
 		{
 			return;
@@ -1626,7 +1648,7 @@ LoadOplog(CidStore&			   ChunkStore,
 
 		AttachmentsWorkLatch.AddCount(1);
 		AttachmentCount.fetch_add(1);
-		WorkerPool.ScheduleWork([&RemoteStore, &ChunkStore, &RemoteResult, &AttachmentsWorkLatch, RawHash]() {
+		WorkerPool.ScheduleWork([&RemoteStore, &ChunkStore, &RemoteResult, &AttachmentsWorkLatch, RawHash, IgnoreMissingAttachments]() {
 			auto _ = MakeGuard([&AttachmentsWorkLatch] { AttachmentsWorkLatch.CountDown(); });
 			if (RemoteResult.IsError())
 			{
@@ -1635,11 +1657,14 @@ LoadOplog(CidStore&			   ChunkStore,
 			RemoteProjectStore::LoadAttachmentResult AttachmentResult = RemoteStore.LoadAttachment(RawHash);
 			if (AttachmentResult.ErrorCode)
 			{
-				RemoteResult.SetError(AttachmentResult.ErrorCode, AttachmentResult.Reason, AttachmentResult.Text);
 				ZEN_WARN("Failed to download attachment {}, reason: '{}', error code: {}",
 						 RawHash,
 						 AttachmentResult.Reason,
 						 AttachmentResult.ErrorCode);
+				if (!IgnoreMissingAttachments)
+				{
+					RemoteResult.SetError(AttachmentResult.ErrorCode, AttachmentResult.Reason, AttachmentResult.Text);
+				}
 				return;
 			}
 			ZEN_DEBUG("Loaded attachment in {}", NiceTimeSpanMs(static_cast<uint64_t>(AttachmentResult.ElapsedSeconds * 1000)));