Add retry with optional resume logic to HttpClient::Download (#639)

- Improvement: Refactored Jupiter upstream to use HttpClient
- Improvement: Added retry and resume logic to HttpClient
- Improvement: Added authentication support to HttpClient
- Improvement: Clearer logging in GCV2 compact of FileCas/BlockStore
- Improvement: Size details in oplog import logging

1 files changed, 21 insertions, 30 deletions

diff --git a/src/zenserver/upstream/jupiter.h b/src/zenserver/upstream/jupiter.h
index b5aa95ed5..93f2cc883 100644
--- a/src/zenserver/upstream/jupiter.h
+++ b/src/zenserver/upstream/jupiter.h
@@ -6,6 +6,7 @@
 #include <zencore/iohash.h>
 #include <zencore/logging.h>
 #include <zencore/thread.h>
+#include <zenhttp/httpclient.h>
 #include <zenhttp/httpserver.h>
 
 #include <atomic>
@@ -22,9 +23,6 @@ class Session;
 }
 
 namespace zen {
-namespace detail {
-	struct CloudCacheSessionState;
-}
 
 class CbObjectView;
 class CloudCacheClient;
@@ -96,7 +94,11 @@ public:
 	~CloudCacheSession();
 
 	CloudCacheResult Authenticate();
-	CloudCacheResult GetRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType RefType);
+	CloudCacheResult GetRef(std::string_view	  Namespace,
+							std::string_view	  BucketId,
+							const IoHash&		  Key,
+							ZenContentType		  RefType,
+							std::filesystem::path TempFolderPath = {});
 	CloudCacheResult GetBlob(std::string_view Namespace, const IoHash& Key);
 	CloudCacheResult GetCompressedBlob(std::string_view Namespace, const IoHash& Key, std::filesystem::path TempFolderPath = {});
 	CloudCacheResult GetObject(std::string_view Namespace, const IoHash& Key);
@@ -131,17 +133,14 @@ public:
 	CloudCacheClient& Client() { return *m_CacheClient; };
 
 private:
-	inline LoggerRef	  Log() { return m_Log; }
-	cpr::Session&		  GetSession();
-	CloudCacheAccessToken GetAccessToken(bool RefreshToken = false);
+	inline LoggerRef Log() { return m_Log; }
 
 	CloudCacheResult CacheTypeExists(std::string_view Namespace, std::string_view TypeId, const IoHash& Key);
 
 	CloudCacheExistsResult CacheTypeExists(std::string_view Namespace, std::string_view TypeId, const std::set<IoHash>& Keys);
 
-	LoggerRef						m_Log;
-	RefPtr<CloudCacheClient>		m_CacheClient;
-	detail::CloudCacheSessionState* m_SessionState;
+	LoggerRef				 m_Log;
+	RefPtr<CloudCacheClient> m_CacheClient;
 };
 
 /**
@@ -178,6 +177,8 @@ struct CloudCacheClientOptions
 	std::chrono::milliseconds ConnectTimeout{5000};
 	std::chrono::milliseconds Timeout{};
 	bool					  AssumeHttp2 = false;
+	bool					  AllowResume = false;
+	uint8_t					  RetryCount  = 0;
 };
 
 /**
@@ -189,30 +190,20 @@ public:
 	CloudCacheClient(const CloudCacheClientOptions& Options, std::unique_ptr<CloudCacheTokenProvider> TokenProvider);
 	~CloudCacheClient();
 
-	CloudCacheAccessToken AcquireAccessToken();
-	std::string_view	  DefaultDdcNamespace() const { return m_DefaultDdcNamespace; }
-	std::string_view	  DefaultBlobStoreNamespace() const { return m_DefaultBlobStoreNamespace; }
-	std::string_view	  ComputeCluster() const { return m_ComputeCluster; }
-	std::string_view	  ServiceUrl() const { return m_ServiceUrl; }
+	std::string_view DefaultDdcNamespace() const { return m_DefaultDdcNamespace; }
+	std::string_view DefaultBlobStoreNamespace() const { return m_DefaultBlobStoreNamespace; }
+	std::string_view ComputeCluster() const { return m_ComputeCluster; }
+	std::string_view ServiceUrl() const { return m_HttpClient.GetBaseUri(); }
 
 	LoggerRef Logger() { return m_Log; }
 
 private:
-	LoggerRef								 m_Log;
-	std::string								 m_ServiceUrl;
-	std::string								 m_DefaultDdcNamespace;
-	std::string								 m_DefaultBlobStoreNamespace;
-	std::string								 m_ComputeCluster;
-	std::chrono::milliseconds				 m_ConnectTimeout{};
-	std::chrono::milliseconds				 m_Timeout{};
-	std::unique_ptr<CloudCacheTokenProvider> m_TokenProvider;
-	bool									 m_AssumeHttp2;
-
-	RwLock									   m_SessionStateLock;
-	std::list<detail::CloudCacheSessionState*> m_SessionStateCache;
-
-	detail::CloudCacheSessionState* AllocSessionState();
-	void							FreeSessionState(detail::CloudCacheSessionState*);
+	LoggerRef									   m_Log;
+	const std::string							   m_DefaultDdcNamespace;
+	const std::string							   m_DefaultBlobStoreNamespace;
+	const std::string							   m_ComputeCluster;
+	const std::unique_ptr<CloudCacheTokenProvider> m_TokenProvider;
+	HttpClient									   m_HttpClient;
 
 	friend class CloudCacheSession;
 };