Add retry with optional resume logic to HttpClient::Download (#639)

- Improvement: Refactored Jupiter upstream to use HttpClient
- Improvement: Added retry and resume logic to HttpClient
- Improvement: Added authentication support to HttpClient
- Improvement: Clearer logging in GCV2 compact of FileCas/BlockStore
- Improvement: Size details in oplog import logging

3 files changed, 51 insertions, 101 deletions

diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index 9d8f6c17b..c9f1f5f6f 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -54,19 +54,8 @@ public:
 
 	virtual SaveResult SaveContainer(const IoBuffer& Payload) override
 	{
-		const int32_t MaxAttempts = 3;
-		PutRefResult  PutResult;
-		{
-			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !PutResult.Success; Attempt++)
-			{
-				PutResult = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject);
-				if (!PutResult.Success)
-				{
-					Sleep(100 * (Attempt + 1));
-				}
-			}
-		}
+		CloudCacheSession Session(m_CloudClient.Get());
+		PutRefResult	  PutResult = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject);
 
 		SaveResult Result{ConvertResult(PutResult), {PutResult.Needs.begin(), PutResult.Needs.end()}, PutResult.RawHash};
 		if (Result.ErrorCode)
@@ -83,19 +72,8 @@ public:
 
 	virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash) override
 	{
-		const int32_t	 MaxAttempts = 3;
-		CloudCacheResult PutResult;
-		{
-			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !PutResult.Success; Attempt++)
-			{
-				PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload);
-				if (!PutResult.Success)
-				{
-					Sleep(100 * (Attempt + 1));
-				}
-			}
-		}
+		CloudCacheSession Session(m_CloudClient.Get());
+		CloudCacheResult  PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload);
 
 		SaveAttachmentResult Result{ConvertResult(PutResult)};
 		if (Result.ErrorCode)
@@ -126,20 +104,9 @@ public:
 
 	virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) override
 	{
-		const int32_t	  MaxAttempts = 3;
-		FinalizeRefResult FinalizeRefResult;
-		{
-			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !FinalizeRefResult.Success; Attempt++)
-			{
-				FinalizeRefResult = Session.FinalizeRef(m_Namespace, m_Bucket, m_Key, RawHash);
-				if (!FinalizeRefResult.Success)
-				{
-					Sleep(100 * (Attempt + 1));
-				}
-			}
-		}
-		FinalizeResult Result{ConvertResult(FinalizeRefResult), {FinalizeRefResult.Needs.begin(), FinalizeRefResult.Needs.end()}};
+		CloudCacheSession Session(m_CloudClient.Get());
+		FinalizeRefResult FinalizeRefResult = Session.FinalizeRef(m_Namespace, m_Bucket, m_Key, RawHash);
+		FinalizeResult	  Result{ConvertResult(FinalizeRefResult), {FinalizeRefResult.Needs.begin(), FinalizeRefResult.Needs.end()}};
 		if (Result.ErrorCode)
 		{
 			Result.Reason = fmt::format("Failed finalizing oplog container to {}/{}/{}/{}. Reason: '{}'",
@@ -165,19 +132,8 @@ public:
 
 	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
 	{
-		const int32_t	 MaxAttempts = 3;
-		CloudCacheResult GetResult;
-		{
-			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !GetResult.Success; Attempt++)
-			{
-				GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath);
-				if (!GetResult.Success)
-				{
-					Sleep(100 * (Attempt + 1));
-				}
-			}
-		}
+		CloudCacheSession	 Session(m_CloudClient.Get());
+		CloudCacheResult	 GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath);
 		LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)};
 		if (GetResult.ErrorCode)
 		{
@@ -210,20 +166,8 @@ public:
 private:
 	LoadContainerResult LoadContainer(const IoHash& Key)
 	{
-		const int32_t	 MaxAttempts = 3;
-		CloudCacheResult GetResult;
-		{
-			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !GetResult.Success; Attempt++)
-			{
-				GetResult = Session.GetRef(m_Namespace, m_Bucket, Key, ZenContentType::kCbObject);
-				if (!GetResult.Success)
-				{
-					Sleep(100 * (Attempt + 1));
-				}
-			}
-		}
-
+		CloudCacheSession Session(m_CloudClient.Get());
+		CloudCacheResult  GetResult = Session.GetRef(m_Namespace, m_Bucket, Key, ZenContentType::kCbObject, m_TempFilePath);
 		if (GetResult.ErrorCode || !GetResult.Success)
 		{
 			LoadContainerResult Result{ConvertResult(GetResult)};
@@ -312,7 +256,9 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi
 										  .ServiceUrl	  = Url,
 										  .ConnectTimeout = std::chrono::milliseconds(2000),
 										  .Timeout		  = std::chrono::milliseconds(1800000),
-										  .AssumeHttp2	  = Options.AssumeHttp2};
+										  .AssumeHttp2	  = Options.AssumeHttp2,
+										  .AllowResume	  = true,
+										  .RetryCount	  = 2};
 	// 1) Access token as parameter in request
 	// 2) Environment variable (different win vs linux/mac)
 	// 3) openid-provider (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 42af9b79b..f117a4203 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -3593,12 +3593,16 @@ ProjectStore::CreateReferenceCheckers(GcCtx& Ctx)
 			Checkers.reserve(OpLogs.size());
 			for (const std::string& OpLogId : OpLogs)
 			{
-				ProjectStore::Oplog* Oplog		 = Project->OpenOplog(OpLogId);
-				GcClock::TimePoint	 Now		 = GcClock::Now();
-				bool				 TryPreCache = Project->LastOplogAccessTime(OpLogId) < (Now - std::chrono::minutes(5));
+				ProjectStore::Oplog* Oplog = Project->OpenOplog(OpLogId);
+				if (Oplog == nullptr)
+				{
+					continue;
+				}
+				GcClock::TimePoint Now		   = GcClock::Now();
+				bool			   TryPreCache = Project->LastOplogAccessTime(OpLogId) < (Now - std::chrono::minutes(5));
 				Checkers.emplace_back(new ProjectStoreReferenceChecker(*Oplog, TryPreCache));
+				OplogCount++;
 			}
-			OplogCount += OpLogs.size();
 		}
 	}
 	catch (std::exception&)
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index ddab7432d..83cec4725 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -560,7 +560,7 @@ BuildContainer(CidStore&																	ChunkStore,
 
 				RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::NotFound), Sb.ToString(), {});
 				ReportMessage(OptionalContext,
-							  fmt::format("Failed to build container ({}): '{}'", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
+							  fmt::format("Failed to build container ({}): {}", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
 
 				BlockCreateLatch.CountDown();
 				while (!BlockCreateLatch.Wait(1000))
@@ -893,11 +893,9 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 			RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::NotFound),
 								  "Invalid attachment",
 								  fmt::format("Upload requested of unknown attachment '{}'", Needed));
-			ReportMessage(OptionalContext,
-						  fmt::format("Failed to upload attachment '{}'. ({}): '{}'",
-									  Needed,
-									  RemoteResult.GetError(),
-									  RemoteResult.GetErrorReason()));
+			ReportMessage(
+				OptionalContext,
+				fmt::format("Failed to upload attachment '{}'. ({}): {}", Needed, RemoteResult.GetError(), RemoteResult.GetErrorReason()));
 			return;
 		}
 	}
@@ -969,7 +967,7 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 				RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::NotFound),
 									  fmt::format("Failed to find attachment {}", RawHash),
 									  {});
-				ZEN_WARN("Failed to save attachment '{}' ({}): '{}'", RawHash, RemoteResult.GetError(), RemoteResult.GetErrorReason());
+				ZEN_WARN("Failed to save attachment '{}' ({}): {}", RawHash, RemoteResult.GetError(), RemoteResult.GetErrorReason());
 				return;
 			}
 
@@ -978,7 +976,7 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 			{
 				RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
 				ReportMessage(OptionalContext,
-							  fmt::format("Failed to save attachment '{}', {} ({}): '{}'",
+							  fmt::format("Failed to save attachment '{}', {} ({}): {}",
 										  RawHash,
 										  NiceBytes(Payload.GetSize()),
 										  RemoteResult.GetError(),
@@ -1031,7 +1029,7 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 				{
 					RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
 					ReportMessage(OptionalContext,
-								  fmt::format("Failed to save attachment '{}', {} ({}): '{}'",
+								  fmt::format("Failed to save attachment '{}', {} ({}): {}",
 											  RawHash,
 											  NiceBytes(Payload.GetSize()),
 											  RemoteResult.GetError(),
@@ -1108,7 +1106,7 @@ UploadAttachments(WorkerThreadPool&											  WorkerPool,
 			{
 				RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
 				ReportMessage(OptionalContext,
-							  fmt::format("Failed to save attachments with {} chunks ({}): '{}'",
+							  fmt::format("Failed to save attachments with {} chunks ({}): {}",
 										  Chunks.size(),
 										  RemoteResult.GetError(),
 										  RemoteResult.GetErrorReason()));
@@ -1230,7 +1228,7 @@ SaveOplog(CidStore&				 ChunkStore,
 		{
 			RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text);
 			ReportMessage(OptionalContext,
-						  fmt::format("Failed to save attachment ({}): '{}'", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
+						  fmt::format("Failed to save attachment ({}): {}", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
 			return;
 		}
 		ZEN_DEBUG("Saved block {}, {}", BlockHash, NiceBytes(CompressedBlock.GetCompressedSize()));
@@ -1270,10 +1268,9 @@ SaveOplog(CidStore&				 ChunkStore,
 		{
 			if (BaseContainerResult.ErrorCode)
 			{
-				ReportMessage(OptionalContext,
-							  fmt::format("Failed to load oplog base container: '{}', error code: {}",
-										  BaseContainerResult.Reason,
-										  BaseContainerResult.ErrorCode));
+				ReportMessage(
+					OptionalContext,
+					fmt::format("Failed to load oplog base container ({}): {}", BaseContainerResult.ErrorCode, BaseContainerResult.Reason));
 			}
 			else
 			{
@@ -1337,7 +1334,7 @@ SaveOplog(CidStore&				 ChunkStore,
 		{
 			RemoteResult.SetError(ContainerSaveResult.ErrorCode, ContainerSaveResult.Reason, "Failed to save oplog container");
 			ReportMessage(OptionalContext,
-						  fmt::format("Failed to save oplog container ({}): '{}'", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
+						  fmt::format("Failed to save oplog container ({}): {}", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
 		}
 		else
 		{
@@ -1372,7 +1369,7 @@ SaveOplog(CidStore&				 ChunkStore,
 			{
 				RemoteResult.SetError(ContainerFinalizeResult.ErrorCode, ContainerFinalizeResult.Reason, ContainerFinalizeResult.Text);
 				ReportMessage(OptionalContext,
-							  fmt::format("Failed to finalize oplog container {} ({}): '{}'",
+							  fmt::format("Failed to finalize oplog container {} ({}): {}",
 										  ContainerSaveResult.RawHash,
 										  RemoteResult.GetError(),
 										  RemoteResult.GetErrorReason()));
@@ -1635,7 +1632,7 @@ LoadOplog(CidStore&			   ChunkStore,
 				if (Result.ErrorCode)
 				{
 					ReportMessage(OptionalContext,
-								  fmt::format("Failed to load attachments with {} chunks ({}): '{}'",
+								  fmt::format("Failed to load attachments with {} chunks ({}): {}",
 											  Chunks.size(),
 											  RemoteResult.GetError(),
 											  RemoteResult.GetErrorReason()));
@@ -1691,7 +1688,7 @@ LoadOplog(CidStore&			   ChunkStore,
 			if (BlockResult.ErrorCode)
 			{
 				ReportMessage(OptionalContext,
-							  fmt::format("Failed to download block attachment {} ({}): '{}'",
+							  fmt::format("Failed to download block attachment {} ({}): {}",
 										  BlockHash,
 										  RemoteResult.GetError(),
 										  RemoteResult.GetErrorReason()));
@@ -1706,18 +1703,20 @@ LoadOplog(CidStore&			   ChunkStore,
 				return;
 			}
 			Info.AttachmentBlocksDownloaded.fetch_add(1);
-			ZEN_INFO("Loaded block attachment '{}' in {}",
+			uint64_t BlockSize = BlockResult.Bytes.GetSize();
+			ZEN_INFO("Loaded block attachment '{}' in {} ({})",
 					 BlockHash,
-					 NiceTimeSpanMs(static_cast<uint64_t>(BlockResult.ElapsedSeconds * 1000)));
+					 NiceTimeSpanMs(static_cast<uint64_t>(BlockResult.ElapsedSeconds * 1000)),
+					 NiceBytes(BlockSize));
 			if (RemoteResult.IsError())
 			{
 				return;
 			}
+			Info.AttachmentBlockBytesDownloaded.fetch_add(BlockSize);
 
 			bool StoreChunksOK =
 				IterateBlock(std::move(BlockResult.Bytes), [&ChunkStore, &Info](CompressedBuffer&& Chunk, const IoHash& AttachmentRawHash) {
-					uint64_t ChunkSize = Chunk.GetCompressedSize();
-					Info.AttachmentBlockBytesDownloaded.fetch_add(ChunkSize);
+					uint64_t			   ChunkSize = Chunk.GetCompressedSize();
 					CidStore::InsertResult InsertResult =
 						ChunkStore.AddChunk(Chunk.GetCompressed().Flatten().AsIoBuffer(), AttachmentRawHash);
 					if (InsertResult.New)
@@ -1730,7 +1729,7 @@ LoadOplog(CidStore&			   ChunkStore,
 			if (!StoreChunksOK)
 			{
 				ReportMessage(OptionalContext,
-							  fmt::format("Block attachment {} has invalid format ({}): '{}'",
+							  fmt::format("Block attachment {} has invalid format ({}): {}",
 										  BlockHash,
 										  RemoteResult.GetError(),
 										  RemoteResult.GetErrorReason()));
@@ -1788,20 +1787,21 @@ LoadOplog(CidStore&			   ChunkStore,
 					}
 					return;
 				}
-				ZEN_INFO("Loaded large attachment '{}' in {}",
+				uint64_t AttachmentSize = AttachmentResult.Bytes.GetSize();
+				ZEN_INFO("Loaded large attachment '{}' in {} ({})",
 						 RawHash,
-						 NiceTimeSpanMs(static_cast<uint64_t>(AttachmentResult.ElapsedSeconds * 1000)));
+						 NiceTimeSpanMs(static_cast<uint64_t>(AttachmentResult.ElapsedSeconds * 1000)),
+						 NiceBytes(AttachmentSize));
 				Info.AttachmentsDownloaded.fetch_add(1);
 				if (RemoteResult.IsError())
 				{
 					return;
 				}
-				uint64_t ChunkSize = AttachmentResult.Bytes.GetSize();
-				Info.AttachmentBytesDownloaded.fetch_add(ChunkSize);
+				Info.AttachmentBytesDownloaded.fetch_add(AttachmentSize);
 				CidStore::InsertResult InsertResult = ChunkStore.AddChunk(AttachmentResult.Bytes, RawHash);
 				if (InsertResult.New)
 				{
-					Info.AttachmentBytesStored.fetch_add(ChunkSize);
+					Info.AttachmentBytesStored.fetch_add(AttachmentSize);
 					Info.AttachmentsStored.fetch_add(1);
 				}
 			});