incremental oplog upload for block-based targets (#392)

* add option for base container for oplog export
read base oplog and fetch known blocks
* reuse blocks if a known block has 80+ % usage
* changelog
* better logging and added base to remotestore descriptions

1 files changed, 61 insertions, 40 deletions

diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index 2bfa6851b..e1a4a9dd4 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -19,6 +19,7 @@ public:
 					   std::string_view				Namespace,
 					   std::string_view				Bucket,
 					   const IoHash&				Key,
+					   const IoHash&				OptionalBaseKey,
 					   bool							ForceDisableBlocks,
 					   bool							ForceDisableTempBlocks,
 					   const std::filesystem::path& TempFilePath)
@@ -26,6 +27,7 @@ public:
 	, m_Namespace(Namespace)
 	, m_Bucket(Bucket)
 	, m_Key(Key)
+	, m_OptionalBaseKey(OptionalBaseKey)
 	, m_TempFilePath(TempFilePath)
 	{
 		if (ForceDisableBlocks)
@@ -42,7 +44,13 @@ public:
 	{
 		return {.CreateBlocks	   = m_EnableBlocks,
 				.UseTempBlockFiles = m_UseTempBlocks,
-				.Description	   = fmt::format("[cloud] {} as {}/{}/{}"sv, m_CloudClient->ServiceUrl(), m_Namespace, m_Bucket, m_Key)};
+				.Description	   = fmt::format("[cloud] {} as {}/{}/{}{}{}"sv,
+											 m_CloudClient->ServiceUrl(),
+											 m_Namespace,
+											 m_Bucket,
+											 m_Key,
+											 m_OptionalBaseKey == IoHash::Zero ? "" : " Base: ",
+											 m_OptionalBaseKey)};
 	}
 
 	virtual SaveResult SaveContainer(const IoBuffer& Payload) override
@@ -145,48 +153,15 @@ public:
 		return Result;
 	}
 
-	virtual LoadContainerResult LoadContainer() override
-	{
-		const int32_t	 MaxAttempts = 3;
-		CloudCacheResult GetResult;
-		{
-			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !GetResult.Success; Attempt++)
-			{
-				GetResult = Session.GetRef(m_Namespace, m_Bucket, m_Key, ZenContentType::kCbObject);
-				if (!GetResult.Success)
-				{
-					Sleep(100 * (Attempt + 1));
-				}
-			}
-		}
+	virtual LoadContainerResult LoadContainer() override { return LoadContainer(m_Key); }
 
-		if (GetResult.ErrorCode || !GetResult.Success)
-		{
-			LoadContainerResult Result{ConvertResult(GetResult)};
-			Result.Reason = fmt::format("Failed fetching oplog container from {}/{}/{}/{}. Reason: '{}'",
-										m_CloudClient->ServiceUrl(),
-										m_Namespace,
-										m_Bucket,
-										m_Key,
-										Result.Reason);
-			return Result;
-		}
-
-		CbObject ContainerObject = LoadCompactBinaryObject(GetResult.Response);
-		if (!ContainerObject)
+	virtual LoadContainerResult LoadBaseContainer() override
+	{
+		if (m_OptionalBaseKey == IoHash::Zero)
 		{
-			return LoadContainerResult{
-				RemoteProjectStore::Result{.ErrorCode	   = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError),
-										   .ElapsedSeconds = GetResult.ElapsedSeconds,
-										   .Reason = fmt::format("The ref {}/{}/{}/{} is not formatted as a compact binary object"sv,
-																 m_CloudClient->ServiceUrl(),
-																 m_Namespace,
-																 m_Bucket,
-																 m_Key)},
-				{}};
+			return LoadContainerResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent)}};
 		}
-		return LoadContainerResult{ConvertResult(GetResult), std::move(ContainerObject)};
+		return LoadContainer(m_OptionalBaseKey);
 	}
 
 	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
@@ -234,6 +209,50 @@ public:
 	}
 
 private:
+	LoadContainerResult LoadContainer(const IoHash& Key)
+	{
+		const int32_t	 MaxAttempts = 3;
+		CloudCacheResult GetResult;
+		{
+			CloudCacheSession Session(m_CloudClient.Get());
+			for (int32_t Attempt = 0; Attempt < MaxAttempts && !GetResult.Success; Attempt++)
+			{
+				GetResult = Session.GetRef(m_Namespace, m_Bucket, Key, ZenContentType::kCbObject);
+				if (!GetResult.Success)
+				{
+					Sleep(100 * (Attempt + 1));
+				}
+			}
+		}
+
+		if (GetResult.ErrorCode || !GetResult.Success)
+		{
+			LoadContainerResult Result{ConvertResult(GetResult)};
+			Result.Reason = fmt::format("Failed fetching oplog container from {}/{}/{}/{}. Reason: '{}'",
+										m_CloudClient->ServiceUrl(),
+										m_Namespace,
+										m_Bucket,
+										Key,
+										Result.Reason);
+			return Result;
+		}
+
+		CbObject ContainerObject = LoadCompactBinaryObject(GetResult.Response);
+		if (!ContainerObject)
+		{
+			return LoadContainerResult{
+				RemoteProjectStore::Result{.ErrorCode	   = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError),
+										   .ElapsedSeconds = GetResult.ElapsedSeconds,
+										   .Reason = fmt::format("The ref {}/{}/{}/{} is not formatted as a compact binary object"sv,
+																 m_CloudClient->ServiceUrl(),
+																 m_Namespace,
+																 m_Bucket,
+																 Key)},
+				{}};
+		}
+		return LoadContainerResult{ConvertResult(GetResult), std::move(ContainerObject)};
+	}
+
 	static Result ConvertResult(const CloudCacheResult& Response)
 	{
 		std::string Text;
@@ -258,6 +277,7 @@ private:
 	const std::string	  m_Namespace;
 	const std::string	  m_Bucket;
 	const IoHash		  m_Key;
+	const IoHash		  m_OptionalBaseKey;
 	std::filesystem::path m_TempFilePath;
 	bool				  m_EnableBlocks  = true;
 	bool				  m_UseTempBlocks = true;
@@ -303,6 +323,7 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi
 																						   Options.Namespace,
 																						   Options.Bucket,
 																						   Options.Key,
+																						   Options.OptionalBaseKey,
 																						   Options.ForceDisableBlocks,
 																						   Options.ForceDisableTempBlocks,
 																						   TempFilePath);