add ignore-missing-attachments option to oplog export (debugging tool) (#641)

* add ignore-missing-attachments option to oplog export (debugging tool)
* add more status codes to do retry for in http client
* add missing X-Jupiter-IoHash header for jupiter PutRef
* reduce oplog block size to reduce amount of redundant chunks to download
* improved logging

1 files changed, 12 insertions, 14 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index f117a4203..caf405066 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -2755,7 +2755,8 @@ ProjectStore::ReadOplog(const std::string_view				  ProjectId,
 		*Oplog,
 		MaxBlockSize,
 		MaxChunkEmbedSize,
-		false,
+		/* BuildBlocks */ false,
+		/* IgnoreMissingAttachemnts */ false,
 		[](CompressedBuffer&&, const IoHash) {},
 		[](const IoHash&) {},
 		[](const std::unordered_set<IoHash, IoHash::Hasher>) {},
@@ -3108,10 +3109,11 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
 
 	using namespace std::literals;
 
-	size_t MaxBlockSize		 = Params["maxblocksize"sv].AsUInt64(128u * 1024u * 1024u);
-	size_t MaxChunkEmbedSize = Params["maxchunkembedsize"sv].AsUInt64(1024u * 1024u);
-	bool   Force			 = Params["force"sv].AsBool(false);
-	bool   EmbedLooseFile	 = Params["embedloosefiles"sv].AsBool(false);
+	size_t MaxBlockSize				= Params["maxblocksize"sv].AsUInt64(64u * 1024u * 1024u);
+	size_t MaxChunkEmbedSize		= Params["maxchunkembedsize"sv].AsUInt64(1024u * 1024u);
+	bool   Force					= Params["force"sv].AsBool(false);
+	bool   IgnoreMissingAttachments = Params["ignoremissingattachments"sv].AsBool(false);
+	bool   EmbedLooseFile			= Params["embedloosefiles"sv].AsBool(false);
 
 	CreateRemoteStoreResult RemoteStoreResult = CreateRemoteStore(Params, AuthManager, MaxBlockSize, MaxChunkEmbedSize, Oplog.TempPath());
 
@@ -3140,7 +3142,8 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
 		 EmbedLooseFile,
 		 CreateBlocks	   = StoreInfo.CreateBlocks,
 		 UseTempBlockFiles = StoreInfo.UseTempBlockFiles,
-		 Force](JobContext& Context) {
+		 Force,
+		 IgnoreMissingAttachments](JobContext& Context) {
 			RemoteProjectStore::Result Result	= SaveOplog(m_CidStore,
 															*ActualRemoteStore,
 															*Project.Get(),
@@ -3151,6 +3154,7 @@ ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Op
 															CreateBlocks,
 															UseTempBlockFiles,
 															Force,
+															IgnoreMissingAttachments,
 															&Context);
 			auto					   Response = ConvertResult(Result);
 			ZEN_INFO("SaveOplog: Status: {} '{}'", ToString(Response.first), Response.second);
@@ -3170,7 +3174,7 @@ ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog,
 
 	using namespace std::literals;
 
-	size_t MaxBlockSize				= Params["maxblocksize"sv].AsUInt64(128u * 1024u * 1024u);
+	size_t MaxBlockSize				= Params["maxblocksize"sv].AsUInt64(64u * 1024u * 1024u);
 	size_t MaxChunkEmbedSize		= Params["maxchunkembedsize"sv].AsUInt64(1024u * 1024u);
 	bool   Force					= Params["force"sv].AsBool(false);
 	bool   IgnoreMissingAttachments = Params["ignoremissingattachments"sv].AsBool(false);
@@ -3187,13 +3191,7 @@ ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog,
 	ZEN_INFO("Loading oplog '{}/{}' from {}", Project.Identifier, Oplog.OplogId(), StoreInfo.Description);
 	JobId JobId = m_JobQueue.QueueJob(
 		fmt::format("Import oplog '{}/{}' from {}", Project.Identifier, Oplog.OplogId(), StoreInfo.Description),
-		[this,
-		 ActualRemoteStore = std::move(RemoteStore),
-		 OplogPtr		   = &Oplog,
-		 MaxBlockSize,
-		 MaxChunkEmbedSize,
-		 Force,
-		 IgnoreMissingAttachments](JobContext& Context) {
+		[this, ActualRemoteStore = std::move(RemoteStore), OplogPtr = &Oplog, Force, IgnoreMissingAttachments](JobContext& Context) {
 			RemoteProjectStore::Result Result =
 				LoadOplog(m_CidStore, *ActualRemoteStore, *OplogPtr, Force, IgnoreMissingAttachments, &Context);
 			auto Response = ConvertResult(Result);