Add retry with optional resume logic to HttpClient::Download (#639)

- Improvement: Refactored Jupiter upstream to use HttpClient
- Improvement: Added retry and resume logic to HttpClient
- Improvement: Added authentication support to HttpClient
- Improvement: Clearer logging in GCV2 compact of FileCas/BlockStore
- Improvement: Size details in oplog import logging

1 files changed, 29 insertions, 29 deletions

diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index ddab7432d..83cec4725 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -560,7 +560,7 @@ BuildContainer(CidStore&																	ChunkStore,
 
 				RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::NotFound), Sb.ToString(), {});
 				ReportMessage(OptionalContext,
-							  fmt::format("Failed to build container ({}): '{}'", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
+							  fmt::format("Failed to build container ({}): {}", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
 
 				BlockCreateLatch.CountDown();
 				while (!BlockCreateLatch.Wait(1000))
@@ -893,11 +893,9 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 			RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::NotFound),
 								  "Invalid attachment",
 								  fmt::format("Upload requested of unknown attachment '{}'", Needed));
-			ReportMessage(OptionalContext,
-						  fmt::format("Failed to upload attachment '{}'. ({}): '{}'",
-									  Needed,
-									  RemoteResult.GetError(),
-									  RemoteResult.GetErrorReason()));
+			ReportMessage(
+				OptionalContext,
+				fmt::format("Failed to upload attachment '{}'. ({}): {}", Needed, RemoteResult.GetError(), RemoteResult.GetErrorReason()));
 			return;
 		}
 	}
@@ -969,7 +967,7 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 				RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::NotFound),
 									  fmt::format("Failed to find attachment {}", RawHash),
 									  {});
-				ZEN_WARN("Failed to save attachment '{}' ({}): '{}'", RawHash, RemoteResult.GetError(), RemoteResult.GetErrorReason());
+				ZEN_WARN("Failed to save attachment '{}' ({}): {}", RawHash, RemoteResult.GetError(), RemoteResult.GetErrorReason());
 				return;
 			}
 
@@ -978,7 +976,7 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 			{
 				RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
 				ReportMessage(OptionalContext,
-							  fmt::format("Failed to save attachment '{}', {} ({}): '{}'",
+							  fmt::format("Failed to save attachment '{}', {} ({}): {}",
 										  RawHash,
 										  NiceBytes(Payload.GetSize()),
 										  RemoteResult.GetError(),
@@ -1031,7 +1029,7 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 				{
 					RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
 					ReportMessage(OptionalContext,
-								  fmt::format("Failed to save attachment '{}', {} ({}): '{}'",
+								  fmt::format("Failed to save attachment '{}', {} ({}): {}",
 											  RawHash,
 											  NiceBytes(Payload.GetSize()),
 											  RemoteResult.GetError(),
@@ -1108,7 +1106,7 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 			{
 				RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
 				ReportMessage(OptionalContext,
-							  fmt::format("Failed to save attachments with {} chunks ({}): '{}'",
+							  fmt::format("Failed to save attachments with {} chunks ({}): {}",
 										  Chunks.size(),
 										  RemoteResult.GetError(),
 										  RemoteResult.GetErrorReason()));
@@ -1230,7 +1228,7 @@ SaveOplog(CidStore&				 ChunkStore,
 		{
 			RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
 			ReportMessage(OptionalContext,
-						  fmt::format("Failed to save attachment ({}): '{}'", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
+						  fmt::format("Failed to save attachment ({}): {}", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
 			return;
 		}
 		ZEN_DEBUG("Saved block {}, {}", BlockHash, NiceBytes(CompressedBlock.GetCompressedSize()));
@@ -1270,10 +1268,9 @@ SaveOplog(CidStore&				 ChunkStore,
 		{
 			if (BaseContainerResult.ErrorCode)
 			{
-				ReportMessage(OptionalContext,
-							  fmt::format("Failed to load oplog base container: '{}', error code: {}",
-										  BaseContainerResult.Reason,
-										  BaseContainerResult.ErrorCode));
+				ReportMessage(
+					OptionalContext,
+					fmt::format("Failed to load oplog base container ({}): {}", BaseContainerResult.ErrorCode, BaseContainerResult.Reason));
 			}
 			else
 			{
@@ -1337,7 +1334,7 @@ SaveOplog(CidStore&				 ChunkStore,
 		{
 			RemoteResult.SetError(ContainerSaveResult.ErrorCode, ContainerSaveResult.Reason, "Failed to save oplog container");
 			ReportMessage(OptionalContext,
-						  fmt::format("Failed to save oplog container ({}): '{}'", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
+						  fmt::format("Failed to save oplog container ({}): {}", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
 		}
 		else
 		{
@@ -1372,7 +1369,7 @@ SaveOplog(CidStore&				 ChunkStore,
 			{
 				RemoteResult.SetError(ContainerFinalizeResult.ErrorCode, ContainerFinalizeResult.Reason, ContainerFinalizeResult.Text);
 				ReportMessage(OptionalContext,
-							  fmt::format("Failed to finalize oplog container {} ({}): '{}'",
+							  fmt::format("Failed to finalize oplog container {} ({}): {}",
 										  ContainerSaveResult.RawHash,
 										  RemoteResult.GetError(),
 										  RemoteResult.GetErrorReason()));
@@ -1635,7 +1632,7 @@ LoadOplog(CidStore&			   ChunkStore,
 				if (Result.ErrorCode)
 				{
 					ReportMessage(OptionalContext,
-								  fmt::format("Failed to load attachments with {} chunks ({}): '{}'",
+								  fmt::format("Failed to load attachments with {} chunks ({}): {}",
 											  Chunks.size(),
 											  RemoteResult.GetError(),
 											  RemoteResult.GetErrorReason()));
@@ -1691,7 +1688,7 @@ LoadOplog(CidStore&			   ChunkStore,
 			if (BlockResult.ErrorCode)
 			{
 				ReportMessage(OptionalContext,
-							  fmt::format("Failed to download block attachment {} ({}): '{}'",
+							  fmt::format("Failed to download block attachment {} ({}): {}",
 										  BlockHash,
 										  RemoteResult.GetError(),
 										  RemoteResult.GetErrorReason()));
@@ -1706,18 +1703,20 @@ LoadOplog(CidStore&			   ChunkStore,
 				return;
 			}
 			Info.AttachmentBlocksDownloaded.fetch_add(1);
-			ZEN_INFO("Loaded block attachment '{}' in {}",
+			uint64_t BlockSize = BlockResult.Bytes.GetSize();
+			ZEN_INFO("Loaded block attachment '{}' in {} ({})",
 					 BlockHash,
-					 NiceTimeSpanMs(static_cast<uint64_t>(BlockResult.ElapsedSeconds * 1000)));
+					 NiceTimeSpanMs(static_cast<uint64_t>(BlockResult.ElapsedSeconds * 1000)),
+					 NiceBytes(BlockSize));
 			if (RemoteResult.IsError())
 			{
 				return;
 			}
+			Info.AttachmentBlockBytesDownloaded.fetch_add(BlockSize);
 
 			bool StoreChunksOK =
 				IterateBlock(std::move(BlockResult.Bytes), [&ChunkStore, &Info](CompressedBuffer&& Chunk, const IoHash& AttachmentRawHash) {
-					uint64_t ChunkSize = Chunk.GetCompressedSize();
-					Info.AttachmentBlockBytesDownloaded.fetch_add(ChunkSize);
+					uint64_t			   ChunkSize = Chunk.GetCompressedSize();
 					CidStore::InsertResult InsertResult =
 						ChunkStore.AddChunk(Chunk.GetCompressed().Flatten().AsIoBuffer(), AttachmentRawHash);
 					if (InsertResult.New)
@@ -1730,7 +1729,7 @@ LoadOplog(CidStore&			   ChunkStore,
 			if (!StoreChunksOK)
 			{
 				ReportMessage(OptionalContext,
-							  fmt::format("Block attachment {} has invalid format ({}): '{}'",
+							  fmt::format("Block attachment {} has invalid format ({}): {}",
 										  BlockHash,
 										  RemoteResult.GetError(),
 										  RemoteResult.GetErrorReason()));
@@ -1788,20 +1787,21 @@ LoadOplog(CidStore&			   ChunkStore,
 					}
 					return;
 				}
-				ZEN_INFO("Loaded large attachment '{}' in {}",
+				uint64_t AttachmentSize = AttachmentResult.Bytes.GetSize();
+				ZEN_INFO("Loaded large attachment '{}' in {} ({})",
 						 RawHash,
-						 NiceTimeSpanMs(static_cast<uint64_t>(AttachmentResult.ElapsedSeconds * 1000)));
+						 NiceTimeSpanMs(static_cast<uint64_t>(AttachmentResult.ElapsedSeconds * 1000)),
+						 NiceBytes(AttachmentSize));
 				Info.AttachmentsDownloaded.fetch_add(1);
 				if (RemoteResult.IsError())
 				{
 					return;
 				}
-				uint64_t ChunkSize = AttachmentResult.Bytes.GetSize();
-				Info.AttachmentBytesDownloaded.fetch_add(ChunkSize);
+				Info.AttachmentBytesDownloaded.fetch_add(AttachmentSize);
 				CidStore::InsertResult InsertResult = ChunkStore.AddChunk(AttachmentResult.Bytes, RawHash);
 				if (InsertResult.New)
 				{
-					Info.AttachmentBytesStored.fetch_add(ChunkSize);
+					Info.AttachmentBytesStored.fetch_add(AttachmentSize);
 					Info.AttachmentsStored.fetch_add(1);
 				}
 			});