Add retry with optional resume logic to HttpClient::Download (#639)

- Improvement: Refactored Jupiter upstream to use HttpClient
- Improvement: Added retry and resume logic to HttpClient
- Improvement: Added authentication support to HttpClient
- Improvement: Clearer logging in GCV2 compact of FileCas/BlockStore
- Improvement: Size details in oplog import logging

1 files changed, 14 insertions, 68 deletions

diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index 9d8f6c17b..c9f1f5f6f 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -54,19 +54,8 @@ public:
 
 	virtual SaveResult SaveContainer(const IoBuffer& Payload) override
 	{
-		const int32_t MaxAttempts = 3;
-		PutRefResult  PutResult;
-		{
-			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !PutResult.Success; Attempt++)
-			{
-				PutResult = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject);
-				if (!PutResult.Success)
-				{
-					Sleep(100 * (Attempt + 1));
-				}
-			}
-		}
+		CloudCacheSession Session(m_CloudClient.Get());
+		PutRefResult	  PutResult = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject);
 
 		SaveResult Result{ConvertResult(PutResult), {PutResult.Needs.begin(), PutResult.Needs.end()}, PutResult.RawHash};
 		if (Result.ErrorCode)
@@ -83,19 +72,8 @@ public:
 
 	virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash) override
 	{
-		const int32_t	 MaxAttempts = 3;
-		CloudCacheResult PutResult;
-		{
-			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !PutResult.Success; Attempt++)
-			{
-				PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload);
-				if (!PutResult.Success)
-				{
-					Sleep(100 * (Attempt + 1));
-				}
-			}
-		}
+		CloudCacheSession Session(m_CloudClient.Get());
+		CloudCacheResult  PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload);
 
 		SaveAttachmentResult Result{ConvertResult(PutResult)};
 		if (Result.ErrorCode)
@@ -126,20 +104,9 @@ public:
 
 	virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) override
 	{
-		const int32_t	  MaxAttempts = 3;
-		FinalizeRefResult FinalizeRefResult;
-		{
-			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !FinalizeRefResult.Success; Attempt++)
-			{
-				FinalizeRefResult = Session.FinalizeRef(m_Namespace, m_Bucket, m_Key, RawHash);
-				if (!FinalizeRefResult.Success)
-				{
-					Sleep(100 * (Attempt + 1));
-				}
-			}
-		}
-		FinalizeResult Result{ConvertResult(FinalizeRefResult), {FinalizeRefResult.Needs.begin(), FinalizeRefResult.Needs.end()}};
+		CloudCacheSession Session(m_CloudClient.Get());
+		FinalizeRefResult FinalizeRefResult = Session.FinalizeRef(m_Namespace, m_Bucket, m_Key, RawHash);
+		FinalizeResult	  Result{ConvertResult(FinalizeRefResult), {FinalizeRefResult.Needs.begin(), FinalizeRefResult.Needs.end()}};
 		if (Result.ErrorCode)
 		{
 			Result.Reason = fmt::format("Failed finalizing oplog container to {}/{}/{}/{}. Reason: '{}'",
@@ -165,19 +132,8 @@ public:
 
 	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
 	{
-		const int32_t	 MaxAttempts = 3;
-		CloudCacheResult GetResult;
-		{
-			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !GetResult.Success; Attempt++)
-			{
-				GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath);
-				if (!GetResult.Success)
-				{
-					Sleep(100 * (Attempt + 1));
-				}
-			}
-		}
+		CloudCacheSession	 Session(m_CloudClient.Get());
+		CloudCacheResult	 GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath);
 		LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)};
 		if (GetResult.ErrorCode)
 		{
@@ -210,20 +166,8 @@ public:
 private:
 	LoadContainerResult LoadContainer(const IoHash& Key)
 	{
-		const int32_t	 MaxAttempts = 3;
-		CloudCacheResult GetResult;
-		{
-			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !GetResult.Success; Attempt++)
-			{
-				GetResult = Session.GetRef(m_Namespace, m_Bucket, Key, ZenContentType::kCbObject);
-				if (!GetResult.Success)
-				{
-					Sleep(100 * (Attempt + 1));
-				}
-			}
-		}
-
+		CloudCacheSession Session(m_CloudClient.Get());
+		CloudCacheResult  GetResult = Session.GetRef(m_Namespace, m_Bucket, Key, ZenContentType::kCbObject, m_TempFilePath);
 		if (GetResult.ErrorCode || !GetResult.Success)
 		{
 			LoadContainerResult Result{ConvertResult(GetResult)};
@@ -312,7 +256,9 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi
 										  .ServiceUrl	  = Url,
 										  .ConnectTimeout = std::chrono::milliseconds(2000),
 										  .Timeout		  = std::chrono::milliseconds(1800000),
-										  .AssumeHttp2	  = Options.AssumeHttp2};
+										  .AssumeHttp2	  = Options.AssumeHttp2,
+										  .AllowResume	  = true,
+										  .RetryCount	  = 2};
 	// 1) Access token as parameter in request
 	// 2) Environment variable (different win vs linux/mac)
 	// 3) openid-provider (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider