stream oplog attachments from jupiter (#384)

* stream large downloads from jupiter to temporary file
* rework DeleteOnClose - top level marks file for delete and if lower level parts wants to keep it it clears that flag
* changelog
* log number of attachments to download
* add delay on jupiter request failure when retrying
* make sure we upload all attachments even if Needs are empty when ForceUpload is true
release TempAttachment as soon as it is used
* sort attachments so we get predictable blocks for the same oplog

1 files changed, 33 insertions, 9 deletions

diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index 4be58256c..2bfa6851b 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -15,16 +15,18 @@ using namespace std::literals;
 class JupiterRemoteStore : public RemoteProjectStore
 {
 public:
-	JupiterRemoteStore(Ref<CloudCacheClient>&& CloudClient,
-					   std::string_view		   Namespace,
-					   std::string_view		   Bucket,
-					   const IoHash&		   Key,
-					   bool					   ForceDisableBlocks,
-					   bool					   ForceDisableTempBlocks)
+	JupiterRemoteStore(Ref<CloudCacheClient>&&		CloudClient,
+					   std::string_view				Namespace,
+					   std::string_view				Bucket,
+					   const IoHash&				Key,
+					   bool							ForceDisableBlocks,
+					   bool							ForceDisableTempBlocks,
+					   const std::filesystem::path& TempFilePath)
 	: m_CloudClient(std::move(CloudClient))
 	, m_Namespace(Namespace)
 	, m_Bucket(Bucket)
 	, m_Key(Key)
+	, m_TempFilePath(TempFilePath)
 	{
 		if (ForceDisableBlocks)
 		{
@@ -52,6 +54,10 @@ public:
 			for (int32_t Attempt = 0; Attempt < MaxAttempts && !PutResult.Success; Attempt++)
 			{
 				PutResult = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject);
+				if (!PutResult.Success)
+				{
+					Sleep(100 * (Attempt + 1));
+				}
 			}
 		}
 
@@ -77,6 +83,10 @@ public:
 			for (int32_t Attempt = 0; Attempt < MaxAttempts && !PutResult.Success; Attempt++)
 			{
 				PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload);
+				if (!PutResult.Success)
+				{
+					Sleep(100 * (Attempt + 1));
+				}
 			}
 		}
 
@@ -116,6 +126,10 @@ public:
 			for (int32_t Attempt = 0; Attempt < MaxAttempts && !FinalizeResult.Success; Attempt++)
 			{
 				FinalizeResult = Session.FinalizeRef(m_Namespace, m_Bucket, m_Key, RawHash);
+				if (!FinalizeResult.Success)
+				{
+					Sleep(100 * (Attempt + 1));
+				}
 			}
 		}
 		Result Result{ConvertResult(FinalizeResult)};
@@ -140,6 +154,10 @@ public:
 			for (int32_t Attempt = 0; Attempt < MaxAttempts && !GetResult.Success; Attempt++)
 			{
 				GetResult = Session.GetRef(m_Namespace, m_Bucket, m_Key, ZenContentType::kCbObject);
+				if (!GetResult.Success)
+				{
+					Sleep(100 * (Attempt + 1));
+				}
 			}
 		}
 
@@ -179,7 +197,11 @@ public:
 			CloudCacheSession Session(m_CloudClient.Get());
 			for (int32_t Attempt = 0; Attempt < MaxAttempts && !GetResult.Success; Attempt++)
 			{
-				GetResult = Session.GetCompressedBlob(m_Namespace, RawHash);
+				GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath);
+				if (!GetResult.Success)
+				{
+					Sleep(100 * (Attempt + 1));
+				}
 			}
 		}
 		LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)};
@@ -236,12 +258,13 @@ private:
 	const std::string	  m_Namespace;
 	const std::string	  m_Bucket;
 	const IoHash		  m_Key;
+	std::filesystem::path m_TempFilePath;
 	bool				  m_EnableBlocks  = true;
 	bool				  m_UseTempBlocks = true;
 };
 
 std::unique_ptr<RemoteProjectStore>
-CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options)
+CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath)
 {
 	std::string Url = Options.Url;
 	if (Url.find("://"sv) == std::string::npos)
@@ -281,7 +304,8 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options)
 																						   Options.Bucket,
 																						   Options.Key,
 																						   Options.ForceDisableBlocks,
-																						   Options.ForceDisableTempBlocks);
+																						   Options.ForceDisableTempBlocks,
+																						   TempFilePath);
 	return RemoteStore;
 }