incremental oplog upload for block-based targets (#392)

* add option for base container for oplog export
read base oplog and fetch known blocks
* reuse blocks if a known block has 80+ % usage
* changelog
* better logging and added base to remotestore descriptions

1 files changed, 20 insertions, 2 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 7754f61cd..f993a7ec7 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -94,12 +94,14 @@ namespace {
 			{
 				return {nullptr, "Missing file name"};
 			}
-			bool ForceDisableBlocks	   = File["disableblocks"sv].AsBool(false);
-			bool ForceEnableTempBlocks = File["enabletempblocks"sv].AsBool(false);
+			std::string_view OptionalBaseName(File["basename"sv].AsString());
+			bool			 ForceDisableBlocks	   = File["disableblocks"sv].AsBool(false);
+			bool			 ForceEnableTempBlocks = File["enabletempblocks"sv].AsBool(false);
 
 			FileRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunkEmbedSize = MaxChunkEmbedSize},
 											  FolderPath,
 											  std::string(Name),
+											  std::string(OptionalBaseName),
 											  ForceDisableBlocks,
 											  ForceEnableTempBlocks};
 			RemoteStore					   = CreateFileRemoteStore(Options);
@@ -148,6 +150,21 @@ namespace {
 			{
 				return {nullptr, "Invalid key string"};
 			}
+			IoHash			 BaseKey	  = IoHash::Zero;
+			std::string_view BaseKeyParam = Cloud["basekey"sv].AsString();
+			if (!BaseKeyParam.empty())
+			{
+				if (BaseKeyParam.length() != IoHash::StringLength)
+				{
+					return {nullptr, "Invalid base key"};
+				}
+				BaseKey = IoHash::FromHexString(BaseKeyParam);
+				if (BaseKey == IoHash::Zero)
+				{
+					return {nullptr, "Invalid base key string"};
+				}
+			}
+
 			bool ForceDisableBlocks		= Cloud["disableblocks"sv].AsBool(false);
 			bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false);
 			bool AssumeHttp2			= Cloud["assumehttp2"sv].AsBool(false);
@@ -157,6 +174,7 @@ namespace {
 												 std::string(Namespace),
 												 std::string(Bucket),
 												 Key,
+												 BaseKey,
 												 std::string(OpenIdProvider),
 												 AccessToken,
 												 AuthManager,