incremental oplog upload for block-based targets (#392)

* add option for base container for oplog export
read base oplog and fetch known blocks
* reuse blocks if a known block has 80+ % usage
* changelog
* better logging and added base to remotestore descriptions

1 files changed, 45 insertions, 30 deletions

diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp
index 1279a294b..faa748c5d 100644
--- a/src/zenserver/projectstore/fileremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp
@@ -16,10 +16,12 @@ class LocalExportProjectStore : public RemoteProjectStore
 {
 public:
 	LocalExportProjectStore(std::string_view			 Name,
+							std::string_view			 OptionalBaseName,
 							const std::filesystem::path& FolderPath,
 							bool						 ForceDisableBlocks,
 							bool						 ForceEnableTempBlocks)
 	: m_Name(Name)
+	, m_OptionalBaseName(OptionalBaseName)
 	, m_OutputPath(FolderPath)
 	{
 		if (ForceDisableBlocks)
@@ -34,9 +36,11 @@ public:
 
 	virtual RemoteStoreInfo GetInfo() const override
 	{
-		return {.CreateBlocks	   = m_EnableBlocks,
-				.UseTempBlockFiles = m_UseTempBlocks,
-				.Description	   = fmt::format("[file] {}"sv, m_OutputPath)};
+		return {
+			.CreateBlocks	   = m_EnableBlocks,
+			.UseTempBlockFiles = m_UseTempBlocks,
+			.Description =
+				fmt::format("[file] {}/{}{}{}"sv, m_OutputPath, m_Name, m_OptionalBaseName.empty() ? "" : " Base: ", m_OptionalBaseName)};
 	}
 
 	virtual SaveResult SaveContainer(const IoBuffer& Payload) override
@@ -133,36 +137,14 @@ public:
 
 	virtual Result FinalizeContainer(const IoHash&) override { return {}; }
 
-	virtual LoadContainerResult LoadContainer() override
+	virtual LoadContainerResult LoadContainer() override { return LoadContainer(m_Name); }
+	virtual LoadContainerResult LoadBaseContainer() override
 	{
-		Stopwatch			  Timer;
-		LoadContainerResult	  Result;
-		std::filesystem::path ContainerPath = m_OutputPath;
-		ContainerPath.append(m_Name);
-		if (!std::filesystem::is_regular_file(ContainerPath))
-		{
-			Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound);
-			Result.Reason =
-				fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", ContainerPath.string());
-			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
-			return Result;
-		}
-		IoBuffer ContainerPayload;
+		if (m_OptionalBaseName.empty())
 		{
-			BasicFile ContainerFile;
-			ContainerFile.Open(ContainerPath, BasicFile::Mode::kRead);
-			ContainerPayload = ContainerFile.ReadAll();
+			return LoadContainerResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent)}};
 		}
-		Result.ContainerObject = LoadCompactBinaryObject(ContainerPayload);
-		if (!Result.ContainerObject)
-		{
-			Result.ErrorCode	  = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
-			Result.Reason		  = fmt::format("The file {} is not formatted as a compact binary object", ContainerPath.string());
-			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
-			return Result;
-		}
-		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
-		return Result;
+		return LoadContainer(m_OptionalBaseName);
 	}
 	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
 	{
@@ -205,6 +187,37 @@ public:
 	}
 
 private:
+	LoadContainerResult LoadContainer(const std::string& Name)
+	{
+		Stopwatch			  Timer;
+		LoadContainerResult	  Result;
+		std::filesystem::path SourcePath = m_OutputPath;
+		SourcePath.append(Name);
+		if (!std::filesystem::is_regular_file(SourcePath))
+		{
+			Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound);
+			Result.Reason = fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", SourcePath.string());
+			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+			return Result;
+		}
+		IoBuffer ContainerPayload;
+		{
+			BasicFile ContainerFile;
+			ContainerFile.Open(SourcePath, BasicFile::Mode::kRead);
+			ContainerPayload = ContainerFile.ReadAll();
+		}
+		Result.ContainerObject = LoadCompactBinaryObject(ContainerPayload);
+		if (!Result.ContainerObject)
+		{
+			Result.ErrorCode	  = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+			Result.Reason		  = fmt::format("The file {} is not formatted as a compact binary object", SourcePath.string());
+			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+			return Result;
+		}
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+		return Result;
+	}
+
 	std::filesystem::path GetAttachmentPath(const IoHash& RawHash) const
 	{
 		ExtendablePathBuilder<128> ShardedPath;
@@ -225,6 +238,7 @@ private:
 	}
 
 	const std::string			m_Name;
+	const std::string			m_OptionalBaseName;
 	const std::filesystem::path m_OutputPath;
 	bool						m_EnableBlocks	= true;
 	bool						m_UseTempBlocks = false;
@@ -234,6 +248,7 @@ std::unique_ptr<RemoteProjectStore>
 CreateFileRemoteStore(const FileRemoteStoreOptions& Options)
 {
 	std::unique_ptr<RemoteProjectStore> RemoteStore = std::make_unique<LocalExportProjectStore>(Options.Name,
+																								Options.OptionalBaseName,
 																								std::filesystem::path(Options.FolderPath),
 																								Options.ForceDisableBlocks,
 																								Options.ForceEnableTempBlocks);