add requested item in oplog remote op (#340)

* add more context for oplog import/export errors

6 files changed, 163 insertions, 60 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cf686699e..418fbe926 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
 - Feature: Added `zen serve` command for establishing a link to a directory tree for use with staged UE builds and the `-Mount=` option
 - Bugfix: Log error instead of hard crash if GC scheduler thread throws exception
 - Improvement: Keep reason and status code when parsing responses from jupiter remote requests
+- Improvement: Add additional context for errors when importing/exporting oplogs
 - Improvement: Added `ZenServerInstance::SpawnServerAndWait` and improved logic around process termination when using `ZenServerInstance::AttachToRunningServer`
 
 ## 0.2.13
diff --git a/src/zen/cmds/projectstore.cpp b/src/zen/cmds/projectstore.cpp
index cca155e19..ed59a8f98 100644
--- a/src/zen/cmds/projectstore.cpp
+++ b/src/zen/cmds/projectstore.cpp
@@ -549,7 +549,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 				}
 			}
 			Writer.EndObject();	 // "file"
-			TargetDescription = fmt::format("[file] '{}/{}'", m_FileDirectoryPath, m_FileName);
+			TargetDescription = fmt::format("[file] {}/{}", m_FileDirectoryPath, m_FileName);
 		}
 		if (!m_CloudUrl.empty())
 		{
@@ -581,7 +581,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 				}
 			}
 			Writer.EndObject();	 // "cloud"
-			TargetDescription = fmt::format("[cloud] '{}/{}/{}/{}'", m_CloudUrl, m_CloudNamespace, m_CloudBucket, m_CloudKey);
+			TargetDescription = fmt::format("[cloud] {}/{}/{}/{}", m_CloudUrl, m_CloudNamespace, m_CloudBucket, m_CloudKey);
 		}
 		if (!m_ZenUrl.empty())
 		{
@@ -593,7 +593,7 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 			}
 			Writer.EndObject();	 // "zen"
 
-			TargetDescription = fmt::format("[zen] '{}/{}/{}'", m_ZenUrl, m_ZenProjectName, m_ZenOplogName);
+			TargetDescription = fmt::format("[zen] {}/{}/{}", m_ZenUrl, m_ZenProjectName, m_ZenOplogName);
 		}
 	}
 	Writer.EndObject();	 // "params"
@@ -793,7 +793,7 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 				Writer.AddString("name"sv, m_FileName);
 			}
 			Writer.EndObject();	 // "file"
-			SourceDescription = fmt::format("[file] '{}/{}'", m_FileDirectoryPath, m_FileName);
+			SourceDescription = fmt::format("[file] {}/{}", m_FileDirectoryPath, m_FileName);
 		}
 		if (!m_CloudUrl.empty())
 		{
@@ -817,7 +817,7 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 				}
 			}
 			Writer.EndObject();	 // "cloud"
-			SourceDescription = fmt::format("[cloud] '{}/{}/{}/{}'", m_CloudUrl, m_CloudNamespace, m_CloudBucket, m_CloudKey);
+			SourceDescription = fmt::format("[cloud] {}/{}/{}/{}", m_CloudUrl, m_CloudNamespace, m_CloudBucket, m_CloudKey);
 		}
 		if (!m_ZenUrl.empty())
 		{
@@ -828,7 +828,7 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 				Writer.AddString("oplog"sv, m_ZenOplogName);
 			}
 			Writer.EndObject();	 // "zen"
-			SourceDescription = fmt::format("[zen] '{}'", m_ZenUrl);
+			SourceDescription = fmt::format("[zen] {}", m_ZenUrl);
 		}
 	}
 	Writer.EndObject();	 // "params"
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index a2ab31254..0dc86a50f 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -99,7 +99,7 @@ ZenCmdBase::FormatHttpResponse(const cpr::Response& Response)
 		zen::HttpContentType ContentType = zen::ParseContentType(It->second);
 		if (ContentType == zen::HttpContentType::kText)
 		{
-			Content = fmt::format("'{}'", Response.text);
+			Content = Response.text;
 		}
 		else if (ContentType == zen::HttpContentType::kJSON)
 		{
diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp
index d7a34a6c2..1279a294b 100644
--- a/src/zenserver/projectstore/fileremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp
@@ -60,17 +60,24 @@ public:
 		std::filesystem::path ContainerPath = m_OutputPath;
 		ContainerPath.append(m_Name);
 
-		CreateDirectories(m_OutputPath);
-		BasicFile ContainerFile;
-		ContainerFile.Open(ContainerPath, BasicFile::Mode::kTruncate);
-		std::error_code Ec;
-		ContainerFile.WriteAll(Payload, Ec);
-		if (Ec)
+		try
+		{
+			CreateDirectories(m_OutputPath);
+			BasicFile ContainerFile;
+			ContainerFile.Open(ContainerPath, BasicFile::Mode::kTruncate);
+			std::error_code Ec;
+			ContainerFile.WriteAll(Payload, Ec);
+			if (Ec)
+			{
+				throw std::system_error(Ec, Ec.message());
+			}
+			Result.RawHash = IoHash::HashBuffer(Payload);
+		}
+		catch (std::exception& Ex)
 		{
 			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
-			Result.Reason	 = Ec.message();
+			Result.Reason	 = fmt::format("Failed saving oplog container to '{}'. Reason: {}", ContainerPath, Ex.what());
 		}
-		Result.RawHash		  = IoHash::HashBuffer(Payload);
 		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
 		return Result;
 	}
@@ -98,7 +105,7 @@ public:
 			catch (std::exception& Ex)
 			{
 				Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
-				Result.Reason	 = Ex.what();
+				Result.Reason	 = fmt::format("Failed saving oplog attachment to '{}'. Reason: {}", ChunkPath, Ex.what());
 			}
 		}
 		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
@@ -134,8 +141,9 @@ public:
 		ContainerPath.append(m_Name);
 		if (!std::filesystem::is_regular_file(ContainerPath))
 		{
-			Result.ErrorCode	  = gsl::narrow<int>(HttpResponseCode::NotFound);
-			Result.Reason		  = fmt::format("The file {} does not exist"sv, ContainerPath.string());
+			Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound);
+			Result.Reason =
+				fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", ContainerPath.string());
 			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
 			return Result;
 		}
@@ -149,7 +157,7 @@ public:
 		if (!Result.ContainerObject)
 		{
 			Result.ErrorCode	  = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
-			Result.Reason		  = fmt::format("The file {} is not formatted as a compact binary object"sv, ContainerPath.string());
+			Result.Reason		  = fmt::format("The file {} is not formatted as a compact binary object", ContainerPath.string());
 			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
 			return Result;
 		}
@@ -163,8 +171,8 @@ public:
 		std::filesystem::path ChunkPath = GetAttachmentPath(RawHash);
 		if (!std::filesystem::is_regular_file(ChunkPath))
 		{
-			Result.ErrorCode	  = gsl::narrow<int>(HttpResponseCode::NotFound);
-			Result.Reason		  = fmt::format("The file {} does not exist"sv, ChunkPath.string());
+			Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound);
+			Result.Reason = fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string());
 			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
 			return Result;
 		}
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index 53455beef..08c8aa0e6 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -46,31 +46,50 @@ public:
 	virtual SaveResult SaveContainer(const IoBuffer& Payload) override
 	{
 		const int32_t MaxAttempts = 3;
-		PutRefResult  Result;
+		PutRefResult  PutResult;
 		{
 			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
+			for (int32_t Attempt = 0; Attempt < MaxAttempts && !PutResult.Success; Attempt++)
 			{
-				Result = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject);
+				PutResult = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject);
 			}
 		}
 
-		return SaveResult{ConvertResult(Result), {Result.Needs.begin(), Result.Needs.end()} /*, {}*/, IoHash::HashBuffer(Payload)};
+		SaveResult Result{ConvertResult(PutResult), {PutResult.Needs.begin(), PutResult.Needs.end()}, IoHash::HashBuffer(Payload)};
+		if (Result.ErrorCode)
+		{
+			Result.Reason = fmt::format("Failed saving oplog container to {}/{}/{}/{}. Reason: '{}'",
+										m_CloudClient->ServiceUrl(),
+										m_Namespace,
+										m_Bucket,
+										m_Key,
+										Result.Reason);
+		}
+		return Result;
 	}
 
 	virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash) override
 	{
 		const int32_t	 MaxAttempts = 3;
-		CloudCacheResult Result;
+		CloudCacheResult PutResult;
 		{
 			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
+			for (int32_t Attempt = 0; Attempt < MaxAttempts && !PutResult.Success; Attempt++)
 			{
-				Result = Session.PutCompressedBlob(m_Namespace, RawHash, Payload);
+				PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload);
 			}
 		}
 
-		return SaveAttachmentResult{ConvertResult(Result)};
+		SaveAttachmentResult Result{ConvertResult(PutResult)};
+		if (Result.ErrorCode)
+		{
+			Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}. Reason: '{}'",
+										m_CloudClient->ServiceUrl(),
+										m_Namespace,
+										RawHash,
+										Result.Reason);
+		}
+		return Result;
 	}
 
 	virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Chunks) override
@@ -91,60 +110,88 @@ public:
 	virtual Result FinalizeContainer(const IoHash& RawHash) override
 	{
 		const int32_t	 MaxAttempts = 3;
-		CloudCacheResult Result;
+		CloudCacheResult FinalizeResult;
 		{
 			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
+			for (int32_t Attempt = 0; Attempt < MaxAttempts && !FinalizeResult.Success; Attempt++)
 			{
-				Result = Session.FinalizeRef(m_Namespace, m_Bucket, m_Key, RawHash);
+				FinalizeResult = Session.FinalizeRef(m_Namespace, m_Bucket, m_Key, RawHash);
 			}
 		}
-		return ConvertResult(Result);
+		Result Result{ConvertResult(FinalizeResult)};
+		if (Result.ErrorCode)
+		{
+			Result.Reason = fmt::format("Failed finalizing oplog container to {}/{}/{}/{}. Reason: '{}'",
+										m_CloudClient->ServiceUrl(),
+										m_Namespace,
+										m_Bucket,
+										m_Key,
+										Result.Reason);
+		}
+		return Result;
 	}
 
 	virtual LoadContainerResult LoadContainer() override
 	{
 		const int32_t	 MaxAttempts = 3;
-		CloudCacheResult Result;
+		CloudCacheResult GetResult;
 		{
 			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
+			for (int32_t Attempt = 0; Attempt < MaxAttempts && !GetResult.Success; Attempt++)
 			{
-				Result = Session.GetRef(m_Namespace, m_Bucket, m_Key, ZenContentType::kCbObject);
+				GetResult = Session.GetRef(m_Namespace, m_Bucket, m_Key, ZenContentType::kCbObject);
 			}
 		}
 
-		if (Result.ErrorCode || !Result.Success)
+		if (GetResult.ErrorCode || !GetResult.Success)
 		{
-			return LoadContainerResult{ConvertResult(Result)};
+			LoadContainerResult Result{ConvertResult(GetResult)};
+			Result.Reason = fmt::format("Failed fetching oplog container from {}/{}/{}/{}. Reason: '{}'",
+										m_CloudClient->ServiceUrl(),
+										m_Namespace,
+										m_Bucket,
+										m_Key,
+										Result.Reason);
+			return Result;
 		}
 
-		CbObject ContainerObject = LoadCompactBinaryObject(Result.Response);
+		CbObject ContainerObject = LoadCompactBinaryObject(GetResult.Response);
 		if (!ContainerObject)
 		{
 			return LoadContainerResult{
-				RemoteProjectStore::Result{
-					.ErrorCode		= gsl::narrow<int32_t>(HttpResponseCode::InternalServerError),
-					.ElapsedSeconds = Result.ElapsedSeconds,
-					.Reason = fmt::format("The ref {}/{}/{} is not formatted as a compact binary object"sv, m_Namespace, m_Bucket, m_Key)},
-				std::move(ContainerObject)};
+				RemoteProjectStore::Result{.ErrorCode	   = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError),
+										   .ElapsedSeconds = GetResult.ElapsedSeconds,
+										   .Reason = fmt::format("The ref {}/{}/{}/{} is not formatted as a compact binary object"sv,
+																 m_CloudClient->ServiceUrl(),
+																 m_Namespace,
+																 m_Bucket,
+																 m_Key)},
+				{}};
 		}
-
-		return LoadContainerResult{ConvertResult(Result), std::move(ContainerObject)};
+		return LoadContainerResult{ConvertResult(GetResult), std::move(ContainerObject)};
 	}
 
 	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
 	{
 		const int32_t	 MaxAttempts = 3;
-		CloudCacheResult Result;
+		CloudCacheResult GetResult;
 		{
 			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
+			for (int32_t Attempt = 0; Attempt < MaxAttempts && !GetResult.Success; Attempt++)
 			{
-				Result = Session.GetCompressedBlob(m_Namespace, RawHash);
+				GetResult = Session.GetCompressedBlob(m_Namespace, RawHash);
 			}
 		}
-		return LoadAttachmentResult{ConvertResult(Result), std::move(Result.Response)};
+		LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)};
+		if (GetResult.ErrorCode)
+		{
+			Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}. Reason: '{}'",
+										m_CloudClient->ServiceUrl(),
+										m_Namespace,
+										RawHash,
+										Result.Reason);
+		}
+		return Result;
 	}
 
 	virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) override
diff --git a/src/zenserver/projectstore/zenremoteprojectstore.cpp b/src/zenserver/projectstore/zenremoteprojectstore.cpp
index 05df1e118..d2ebc1484 100644
--- a/src/zenserver/projectstore/zenremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/zenremoteprojectstore.cpp
@@ -58,6 +58,11 @@ public:
 
 		if (Result.ErrorCode)
 		{
+			Result.Reason		  = fmt::format("Failed saving oplog container to {}/{}/{}. Reason: '{}'",
+										m_ProjectStoreUrl,
+										m_Project,
+										m_Oplog,
+										Result.Reason);
 			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
 			return Result;
 		}
@@ -107,7 +112,16 @@ public:
 		Session->SetReadCallback(cpr::ReadCallback(gsl::narrow<cpr::cpr_off_t>(SizeLeft), ReadCallback));
 		cpr::Response		 Response = Session->Post();
 		SaveAttachmentResult Result	  = SaveAttachmentResult{ConvertResult(Response)};
-		Result.ElapsedSeconds		  = Timer.GetElapsedTimeMs() / 1000.500;
+		if (Result.ErrorCode)
+		{
+			Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'",
+										m_ProjectStoreUrl,
+										m_Project,
+										m_Oplog,
+										RawHash,
+										Result.Reason);
+		}
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
 		return Result;
 	}
 
@@ -153,7 +167,16 @@ public:
 		Session->SetReadCallback(cpr::ReadCallback(gsl::narrow<cpr::cpr_off_t>(SizeLeft), ReadCallback));
 		cpr::Response		  Response = Session->Post();
 		SaveAttachmentsResult Result   = SaveAttachmentsResult{ConvertResult(Response)};
-		Result.ElapsedSeconds		   = Timer.GetElapsedTimeMs() / 1000.500;
+		if (Result.ErrorCode)
+		{
+			Result.Reason = fmt::format("Failed saving {} oplog attachments to {}/{}/{}. Reason: '{}'",
+										Chunks.size(),
+										m_ProjectStoreUrl,
+										m_Project,
+										m_Oplog,
+										Result.Reason);
+		}
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
 		return Result;
 	}
 
@@ -198,6 +221,15 @@ public:
 					std::pair<IoHash, CompressedBuffer>{Attachment.GetHash(), Attachment.AsCompressedBinary().MakeOwned()});
 			}
 		}
+		else
+		{
+			Result.Reason = fmt::format("Failed fetching {} oplog attachments from {}/{}/{}. Reason: '{}'",
+										RawHashes.size(),
+										m_ProjectStoreUrl,
+										m_Project,
+										m_Oplog,
+										Result.Reason);
+		}
 		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
 		return Result;
 	};
@@ -225,18 +257,24 @@ public:
 		cpr::Response Response = Session->Get();
 
 		LoadContainerResult Result = LoadContainerResult{ConvertResult(Response)};
-		if (!Result.ErrorCode)
+		if (Result.ErrorCode)
+		{
+			Result.Reason = fmt::format("Failed fetching oplog container from {}/{}/{}. Reason: '{}'",
+										m_ProjectStoreUrl,
+										m_Project,
+										m_Oplog,
+										Result.Reason);
+		}
+		else
 		{
 			Result.ContainerObject = LoadCompactBinaryObject(IoBuffer(IoBuffer::Clone, Response.text.data(), Response.text.size()));
 			if (!Result.ContainerObject)
 			{
-				Result.Reason		  = fmt::format("The response for {}/{}/{} is not formatted as a compact binary object"sv,
-											m_ProjectStoreUrl,
-											m_Project,
-											m_Oplog);
-				Result.ErrorCode	  = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
-				Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
-				return Result;
+				Result.Reason	 = fmt::format("The response for {}/{}/{} is not formatted as a compact binary object"sv,
+											   m_ProjectStoreUrl,
+											   m_Project,
+											   m_Oplog);
+				Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
 			}
 		}
 		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
@@ -259,6 +297,15 @@ public:
 		{
 			Result.Bytes = IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size());
 		}
+		if (!Result.ErrorCode)
+		{
+			Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}. Reason: '{}'",
+										m_ProjectStoreUrl,
+										m_Project,
+										m_Oplog,
+										RawHash,
+										Result.Reason);
+		}
 		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
 		return Result;
 	}