add ignore-missing-attachments option to oplog export (debugging tool) (#641)

* add ignore-missing-attachments option to oplog export (debugging tool)
* add more status codes to do retry for in http client
* add missing X-Jupiter-IoHash header for jupiter PutRef
* reduce oplog block size to reduce amount of redundant chunks to download
* improved logging

1 files changed, 34 insertions, 21 deletions

diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index 83cec4725..099b9ceb3 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -230,6 +230,7 @@ BuildContainer(CidStore&																	ChunkStore,
 			   size_t																		MaxBlockSize,
 			   size_t																		MaxChunkEmbedSize,
 			   bool																			BuildBlocks,
+			   bool																			IgnoreMissingAttachments,
 			   const std::vector<Block>&													KnownBlocks,
 			   WorkerThreadPool&															WorkerPool,
 			   const std::function<void(CompressedBuffer&&, const IoHash&)>&				AsyncOnBlock,
@@ -399,7 +400,7 @@ BuildContainer(CidStore&																	ChunkStore,
 			CB(RewrittenOp);
 		};
 
-		ReportMessage(OptionalContext, "Building exported oplog and fetching attachments");
+		ReportMessage(OptionalContext, "Building exported oplog and collecting attachments");
 
 		tsl::robin_map<int, std::string> OpLSNToKey;
 
@@ -558,17 +559,22 @@ BuildContainer(CidStore&																	ChunkStore,
 				Sb.Append("' for op: \n");
 				Op.value().ToJson(Sb);
 
-				RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::NotFound), Sb.ToString(), {});
-				ReportMessage(OptionalContext,
-							  fmt::format("Failed to build container ({}): {}", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
+				ReportMessage(OptionalContext, fmt::format("Missing attachment '{}': {}", AttachmentHash, Sb.ToView()));
 
-				BlockCreateLatch.CountDown();
-				while (!BlockCreateLatch.Wait(1000))
+				if (IgnoreMissingAttachments)
 				{
-					ZEN_INFO("Aborting, {} blocks remaining...", BlockCreateLatch.Remaining());
+					continue;
+				}
+				else
+				{
+					RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::NotFound), Sb.ToString(), {});
+					BlockCreateLatch.CountDown();
+					while (!BlockCreateLatch.Wait(1000))
+					{
+						ZEN_INFO("Aborting, {} blocks remaining...", BlockCreateLatch.Remaining());
+					}
+					return {};
 				}
-
-				return {};
 			}
 
 			uint64_t PayloadSize = Payload.GetSize();
@@ -798,6 +804,7 @@ BuildContainer(CidStore&																	ChunkStore,
 			   size_t																		MaxBlockSize,
 			   size_t																		MaxChunkEmbedSize,
 			   bool																			BuildBlocks,
+			   bool																			IgnoreMissingAttachments,
 			   const std::function<void(CompressedBuffer&&, const IoHash&)>&				AsyncOnBlock,
 			   const std::function<void(const IoHash&)>&									OnLargeAttachment,
 			   const std::function<void(const std::unordered_set<IoHash, IoHash::Hasher>)>& OnBlockChunks,
@@ -812,6 +819,7 @@ BuildContainer(CidStore&																	ChunkStore,
 											   MaxBlockSize,
 											   MaxChunkEmbedSize,
 											   BuildBlocks,
+											   IgnoreMissingAttachments,
 											   {},
 											   WorkerPool,
 											   AsyncOnBlock,
@@ -983,10 +991,10 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 										  RemoteResult.GetErrorReason()));
 				return;
 			}
-			ZEN_DEBUG("Saved attachment {}, {} in {}",
-					  RawHash,
-					  NiceBytes(Payload.GetSize()),
-					  NiceTimeSpanMs(static_cast<uint64_t>(Result.ElapsedSeconds * 1000)));
+			ZEN_INFO("Saved large attachment '{}' in {} ({})",
+					 RawHash,
+					 NiceTimeSpanMs(static_cast<uint64_t>(Result.ElapsedSeconds * 1000)),
+					 NiceBytes(Payload.GetSize()));
 			return;
 		});
 	}
@@ -1037,10 +1045,10 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 					return;
 				}
 
-				ZEN_DEBUG("Saved attachment {}, {} in {}",
-						  RawHash,
-						  NiceBytes(Payload.GetSize()),
-						  NiceTimeSpanMs(static_cast<uint64_t>(Result.ElapsedSeconds * 1000)));
+				ZEN_INFO("Saved block attachment '{}' in {} ({})",
+						 RawHash,
+						 NiceTimeSpanMs(static_cast<uint64_t>(Result.ElapsedSeconds * 1000)),
+						 NiceBytes(Payload.GetSize()));
 				return;
 			});
 	}
@@ -1085,7 +1093,8 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 								 NeededChunks = std::move(NeededChunks),
 								 &BulkAttachmentCountToUpload,
 								 OptionalContext]() {
-			auto					  _ = MakeGuard([&SaveAttachmentsLatch] { SaveAttachmentsLatch.CountDown(); });
+			auto					  _			 = MakeGuard([&SaveAttachmentsLatch] { SaveAttachmentsLatch.CountDown(); });
+			size_t					  ChunksSize = 0;
 			std::vector<SharedBuffer> ChunkBuffers;
 			ChunkBuffers.reserve(NeededChunks.size());
 			for (const IoHash& Chunk : NeededChunks)
@@ -1099,6 +1108,7 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 					ChunkBuffers.clear();
 					break;
 				}
+				ChunksSize += ChunkPayload.GetSize();
 				ChunkBuffers.emplace_back(SharedBuffer(std::move(ChunkPayload)));
 			}
 			RemoteProjectStore::SaveAttachmentsResult Result = RemoteStore.SaveAttachments(ChunkBuffers);
@@ -1112,9 +1122,10 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 										  RemoteResult.GetErrorReason()));
 				return;
 			}
-			ZEN_DEBUG("Saved {} bulk attachments in {}",
-					  Chunks.size(),
-					  NiceTimeSpanMs(static_cast<uint64_t>(Result.ElapsedSeconds * 1000)));
+			ZEN_INFO("Saved {} bulk attachments in {} ({})",
+					 Chunks.size(),
+					 NiceTimeSpanMs(static_cast<uint64_t>(Result.ElapsedSeconds * 1000)),
+					 NiceBytes(ChunksSize));
 			BulkAttachmentCountToUpload.fetch_sub(Chunks.size());
 		});
 	}
@@ -1153,6 +1164,7 @@ SaveOplog(CidStore&				 ChunkStore,
 		  bool					 BuildBlocks,
 		  bool					 UseTempBlocks,
 		  bool					 ForceUpload,
+		  bool					 IgnoreMissingAttachments,
 		  JobContext*			 OptionalContext)
 {
 	using namespace std::literals;
@@ -1307,6 +1319,7 @@ SaveOplog(CidStore&				 ChunkStore,
 												   MaxBlockSize,
 												   MaxChunkEmbedSize,
 												   BuildBlocks,
+												   IgnoreMissingAttachments,
 												   KnownBlocks,
 												   WorkerPool,
 												   OnBlock,