incremental oplog upload for block-based targets (#392)

* add option for base container for oplog export
read base oplog and fetch known blocks
* reuse blocks if a known block has 80+ % usage
* changelog
* better logging and added base to remotestore descriptions

10 files changed, 258 insertions, 74 deletions

diff --git a/src/zen/cmds/projectstore.cpp b/src/zen/cmds/projectstore.cpp
index fb3db1fdd..c3aefa76d 100644
--- a/src/zen/cmds/projectstore.cpp
+++ b/src/zen/cmds/projectstore.cpp
@@ -486,6 +486,12 @@ ExportOplogCommand::ExportOplogCommand()
 	m_Options.add_option("cloud", "", "namespace", "Cloud Storage namespace", cxxopts::value(m_CloudNamespace), "<namespace>");
 	m_Options.add_option("cloud", "", "bucket", "Cloud Storage bucket", cxxopts::value(m_CloudBucket), "<bucket>");
 	m_Options.add_option("cloud", "", "key", "Cloud Storage key", cxxopts::value(m_CloudKey), "<key>");
+	m_Options.add_option("cloud",
+						 "",
+						 "basekey",
+						 "Optional Base Cloud Storage key for incremental export",
+						 cxxopts::value(m_BaseCloudKey),
+						 "<key>");
 	m_Options
 		.add_option("cloud", "", "openid-provider", "Cloud Storage openid provider", cxxopts::value(m_CloudOpenIdProvider), "<provider>");
 	m_Options.add_option("cloud", "", "access-token", "Cloud Storage access token", cxxopts::value(m_CloudAccessToken), "<accesstoken>");
@@ -517,6 +523,12 @@ ExportOplogCommand::ExportOplogCommand()
 	m_Options.add_option("file", "", "name", "Local file name", cxxopts::value(m_FileName), "<filename>");
 	m_Options.add_option("file",
 						 "",
+						 "basename",
+						 "Local base file name for incremental oplog export",
+						 cxxopts::value(m_BaseFileName),
+						 "<filename>");
+	m_Options.add_option("file",
+						 "",
 						 "forcetempblocks",
 						 "Force creation of temp attachment blocks",
 						 cxxopts::value(m_FileForceEnableTempBlocks),
@@ -687,6 +699,10 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 				{
 					Writer.AddString("path"sv, m_FileDirectoryPath);
 					Writer.AddString("name"sv, m_FileName);
+					if (!m_BaseFileName.empty())
+					{
+						Writer.AddString("basename"sv, m_BaseFileName);
+					}
 					if (m_DisableBlocks)
 					{
 						Writer.AddBool("disableblocks"sv, true);
@@ -697,7 +713,11 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 					}
 				}
 				Writer.EndObject();	 // "file"
-				TargetDescription = fmt::format("[file] {}/{}", m_FileDirectoryPath, m_FileName);
+				TargetDescription = fmt::format("[file] {}/{}{}{}",
+												m_FileDirectoryPath,
+												m_FileName,
+												m_BaseFileName.empty() ? "" : " Base: ",
+												m_BaseFileName);
 			}
 			if (!m_CloudUrl.empty())
 			{
@@ -707,6 +727,10 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 					Writer.AddString("namespace"sv, m_CloudNamespace);
 					Writer.AddString("bucket"sv, m_CloudBucket);
 					Writer.AddString("key"sv, m_CloudKey);
+					if (!m_BaseCloudKey.empty())
+					{
+						Writer.AddString("basekey"sv, m_BaseCloudKey);
+					}
 					if (!m_CloudOpenIdProvider.empty())
 					{
 						Writer.AddString("openid-provider"sv, m_CloudOpenIdProvider);
@@ -742,7 +766,13 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
 					}
 				}
 				Writer.EndObject();	 // "cloud"
-				TargetDescription = fmt::format("[cloud] {}/{}/{}/{}", m_CloudUrl, m_CloudNamespace, m_CloudBucket, m_CloudKey);
+				TargetDescription = fmt::format("[cloud] {}/{}/{}/{}{}{}",
+												m_CloudUrl,
+												m_CloudNamespace,
+												m_CloudBucket,
+												m_CloudKey,
+												m_BaseCloudKey.empty() ? "" : " Base: ",
+												m_BaseCloudKey);
 			}
 			if (!m_ZenUrl.empty())
 			{
diff --git a/src/zen/cmds/projectstore.h b/src/zen/cmds/projectstore.h
index 64779f76c..e22357d09 100644
--- a/src/zen/cmds/projectstore.h
+++ b/src/zen/cmds/projectstore.h
@@ -131,6 +131,7 @@ private:
 	std::string m_CloudNamespace;
 	std::string m_CloudBucket;
 	std::string m_CloudKey;
+	std::string m_BaseCloudKey;
 	std::string m_CloudOpenIdProvider;
 	std::string m_CloudAccessToken;
 	std::string m_CloudAccessTokenEnv;
@@ -144,6 +145,7 @@ private:
 
 	std::string m_FileDirectoryPath;
 	std::string m_FileName;
+	std::string m_BaseFileName;
 	bool		m_FileForceEnableTempBlocks = false;
 };
 
diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp
index 1279a294b..faa748c5d 100644
--- a/src/zenserver/projectstore/fileremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp
@@ -16,10 +16,12 @@ class LocalExportProjectStore : public RemoteProjectStore
 {
 public:
 	LocalExportProjectStore(std::string_view			 Name,
+							std::string_view			 OptionalBaseName,
 							const std::filesystem::path& FolderPath,
 							bool						 ForceDisableBlocks,
 							bool						 ForceEnableTempBlocks)
 	: m_Name(Name)
+	, m_OptionalBaseName(OptionalBaseName)
 	, m_OutputPath(FolderPath)
 	{
 		if (ForceDisableBlocks)
@@ -34,9 +36,11 @@ public:
 
 	virtual RemoteStoreInfo GetInfo() const override
 	{
-		return {.CreateBlocks	   = m_EnableBlocks,
-				.UseTempBlockFiles = m_UseTempBlocks,
-				.Description	   = fmt::format("[file] {}"sv, m_OutputPath)};
+		return {
+			.CreateBlocks	   = m_EnableBlocks,
+			.UseTempBlockFiles = m_UseTempBlocks,
+			.Description =
+				fmt::format("[file] {}/{}{}{}"sv, m_OutputPath, m_Name, m_OptionalBaseName.empty() ? "" : " Base: ", m_OptionalBaseName)};
 	}
 
 	virtual SaveResult SaveContainer(const IoBuffer& Payload) override
@@ -133,36 +137,14 @@ public:
 
 	virtual Result FinalizeContainer(const IoHash&) override { return {}; }
 
-	virtual LoadContainerResult LoadContainer() override
+	virtual LoadContainerResult LoadContainer() override { return LoadContainer(m_Name); }
+	virtual LoadContainerResult LoadBaseContainer() override
 	{
-		Stopwatch			  Timer;
-		LoadContainerResult	  Result;
-		std::filesystem::path ContainerPath = m_OutputPath;
-		ContainerPath.append(m_Name);
-		if (!std::filesystem::is_regular_file(ContainerPath))
-		{
-			Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound);
-			Result.Reason =
-				fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", ContainerPath.string());
-			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
-			return Result;
-		}
-		IoBuffer ContainerPayload;
+		if (m_OptionalBaseName.empty())
 		{
-			BasicFile ContainerFile;
-			ContainerFile.Open(ContainerPath, BasicFile::Mode::kRead);
-			ContainerPayload = ContainerFile.ReadAll();
+			return LoadContainerResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent)}};
 		}
-		Result.ContainerObject = LoadCompactBinaryObject(ContainerPayload);
-		if (!Result.ContainerObject)
-		{
-			Result.ErrorCode	  = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
-			Result.Reason		  = fmt::format("The file {} is not formatted as a compact binary object", ContainerPath.string());
-			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
-			return Result;
-		}
-		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
-		return Result;
+		return LoadContainer(m_OptionalBaseName);
 	}
 	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
 	{
@@ -205,6 +187,37 @@ public:
 	}
 
 private:
+	LoadContainerResult LoadContainer(const std::string& Name)
+	{
+		Stopwatch			  Timer;
+		LoadContainerResult	  Result;
+		std::filesystem::path SourcePath = m_OutputPath;
+		SourcePath.append(Name);
+		if (!std::filesystem::is_regular_file(SourcePath))
+		{
+			Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound);
+			Result.Reason = fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", SourcePath.string());
+			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+			return Result;
+		}
+		IoBuffer ContainerPayload;
+		{
+			BasicFile ContainerFile;
+			ContainerFile.Open(SourcePath, BasicFile::Mode::kRead);
+			ContainerPayload = ContainerFile.ReadAll();
+		}
+		Result.ContainerObject = LoadCompactBinaryObject(ContainerPayload);
+		if (!Result.ContainerObject)
+		{
+			Result.ErrorCode	  = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+			Result.Reason		  = fmt::format("The file {} is not formatted as a compact binary object", SourcePath.string());
+			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+			return Result;
+		}
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+		return Result;
+	}
+
 	std::filesystem::path GetAttachmentPath(const IoHash& RawHash) const
 	{
 		ExtendablePathBuilder<128> ShardedPath;
@@ -225,6 +238,7 @@ private:
 	}
 
 	const std::string			m_Name;
+	const std::string			m_OptionalBaseName;
 	const std::filesystem::path m_OutputPath;
 	bool						m_EnableBlocks	= true;
 	bool						m_UseTempBlocks = false;
@@ -234,6 +248,7 @@ std::unique_ptr<RemoteProjectStore>
 CreateFileRemoteStore(const FileRemoteStoreOptions& Options)
 {
 	std::unique_ptr<RemoteProjectStore> RemoteStore = std::make_unique<LocalExportProjectStore>(Options.Name,
+																								Options.OptionalBaseName,
 																								std::filesystem::path(Options.FolderPath),
 																								Options.ForceDisableBlocks,
 																								Options.ForceEnableTempBlocks);
diff --git a/src/zenserver/projectstore/fileremoteprojectstore.h b/src/zenserver/projectstore/fileremoteprojectstore.h
index b1a06ef15..f398bbfbc 100644
--- a/src/zenserver/projectstore/fileremoteprojectstore.h
+++ b/src/zenserver/projectstore/fileremoteprojectstore.h
@@ -10,6 +10,7 @@ struct FileRemoteStoreOptions : RemoteStoreOptions
 {
 	std::filesystem::path FolderPath;
 	std::string			  Name;
+	std::string			  OptionalBaseName;
 	bool				  ForceDisableBlocks	= false;
 	bool				  ForceEnableTempBlocks = false;
 };
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index 2bfa6851b..e1a4a9dd4 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -19,6 +19,7 @@ public:
 					   std::string_view				Namespace,
 					   std::string_view				Bucket,
 					   const IoHash&				Key,
+					   const IoHash&				OptionalBaseKey,
 					   bool							ForceDisableBlocks,
 					   bool							ForceDisableTempBlocks,
 					   const std::filesystem::path& TempFilePath)
@@ -26,6 +27,7 @@ public:
 	, m_Namespace(Namespace)
 	, m_Bucket(Bucket)
 	, m_Key(Key)
+	, m_OptionalBaseKey(OptionalBaseKey)
 	, m_TempFilePath(TempFilePath)
 	{
 		if (ForceDisableBlocks)
@@ -42,7 +44,13 @@ public:
 	{
 		return {.CreateBlocks	   = m_EnableBlocks,
 				.UseTempBlockFiles = m_UseTempBlocks,
-				.Description	   = fmt::format("[cloud] {} as {}/{}/{}"sv, m_CloudClient->ServiceUrl(), m_Namespace, m_Bucket, m_Key)};
+				.Description	   = fmt::format("[cloud] {} as {}/{}/{}{}{}"sv,
+											 m_CloudClient->ServiceUrl(),
+											 m_Namespace,
+											 m_Bucket,
+											 m_Key,
+											 m_OptionalBaseKey == IoHash::Zero ? "" : " Base: ",
+											 m_OptionalBaseKey)};
 	}
 
 	virtual SaveResult SaveContainer(const IoBuffer& Payload) override
@@ -145,48 +153,15 @@ public:
 		return Result;
 	}
 
-	virtual LoadContainerResult LoadContainer() override
-	{
-		const int32_t	 MaxAttempts = 3;
-		CloudCacheResult GetResult;
-		{
-			CloudCacheSession Session(m_CloudClient.Get());
-			for (int32_t Attempt = 0; Attempt < MaxAttempts && !GetResult.Success; Attempt++)
-			{
-				GetResult = Session.GetRef(m_Namespace, m_Bucket, m_Key, ZenContentType::kCbObject);
-				if (!GetResult.Success)
-				{
-					Sleep(100 * (Attempt + 1));
-				}
-			}
-		}
+	virtual LoadContainerResult LoadContainer() override { return LoadContainer(m_Key); }
 
-		if (GetResult.ErrorCode || !GetResult.Success)
-		{
-			LoadContainerResult Result{ConvertResult(GetResult)};
-			Result.Reason = fmt::format("Failed fetching oplog container from {}/{}/{}/{}. Reason: '{}'",
-										m_CloudClient->ServiceUrl(),
-										m_Namespace,
-										m_Bucket,
-										m_Key,
-										Result.Reason);
-			return Result;
-		}
-
-		CbObject ContainerObject = LoadCompactBinaryObject(GetResult.Response);
-		if (!ContainerObject)
+	virtual LoadContainerResult LoadBaseContainer() override
+	{
+		if (m_OptionalBaseKey == IoHash::Zero)
 		{
-			return LoadContainerResult{
-				RemoteProjectStore::Result{.ErrorCode	   = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError),
-										   .ElapsedSeconds = GetResult.ElapsedSeconds,
-										   .Reason = fmt::format("The ref {}/{}/{}/{} is not formatted as a compact binary object"sv,
-																 m_CloudClient->ServiceUrl(),
-																 m_Namespace,
-																 m_Bucket,
-																 m_Key)},
-				{}};
+			return LoadContainerResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent)}};
 		}
-		return LoadContainerResult{ConvertResult(GetResult), std::move(ContainerObject)};
+		return LoadContainer(m_OptionalBaseKey);
 	}
 
 	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
@@ -234,6 +209,50 @@ public:
 	}
 
 private:
+	LoadContainerResult LoadContainer(const IoHash& Key)
+	{
+		const int32_t	 MaxAttempts = 3;
+		CloudCacheResult GetResult;
+		{
+			CloudCacheSession Session(m_CloudClient.Get());
+			for (int32_t Attempt = 0; Attempt < MaxAttempts && !GetResult.Success; Attempt++)
+			{
+				GetResult = Session.GetRef(m_Namespace, m_Bucket, Key, ZenContentType::kCbObject);
+				if (!GetResult.Success)
+				{
+					Sleep(100 * (Attempt + 1));
+				}
+			}
+		}
+
+		if (GetResult.ErrorCode || !GetResult.Success)
+		{
+			LoadContainerResult Result{ConvertResult(GetResult)};
+			Result.Reason = fmt::format("Failed fetching oplog container from {}/{}/{}/{}. Reason: '{}'",
+										m_CloudClient->ServiceUrl(),
+										m_Namespace,
+										m_Bucket,
+										Key,
+										Result.Reason);
+			return Result;
+		}
+
+		CbObject ContainerObject = LoadCompactBinaryObject(GetResult.Response);
+		if (!ContainerObject)
+		{
+			return LoadContainerResult{
+				RemoteProjectStore::Result{.ErrorCode	   = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError),
+										   .ElapsedSeconds = GetResult.ElapsedSeconds,
+										   .Reason = fmt::format("The ref {}/{}/{}/{} is not formatted as a compact binary object"sv,
+																 m_CloudClient->ServiceUrl(),
+																 m_Namespace,
+																 m_Bucket,
+																 Key)},
+				{}};
+		}
+		return LoadContainerResult{ConvertResult(GetResult), std::move(ContainerObject)};
+	}
+
 	static Result ConvertResult(const CloudCacheResult& Response)
 	{
 		std::string Text;
@@ -258,6 +277,7 @@ private:
 	const std::string	  m_Namespace;
 	const std::string	  m_Bucket;
 	const IoHash		  m_Key;
+	const IoHash		  m_OptionalBaseKey;
 	std::filesystem::path m_TempFilePath;
 	bool				  m_EnableBlocks  = true;
 	bool				  m_UseTempBlocks = true;
@@ -303,6 +323,7 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi
 																						   Options.Namespace,
 																						   Options.Bucket,
 																						   Options.Key,
+																						   Options.OptionalBaseKey,
 																						   Options.ForceDisableBlocks,
 																						   Options.ForceDisableTempBlocks,
 																						   TempFilePath);
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.h b/src/zenserver/projectstore/jupiterremoteprojectstore.h
index d6fced91b..4ae6c88cb 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.h
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.h
@@ -14,6 +14,7 @@ struct JupiterRemoteStoreOptions : RemoteStoreOptions
 	std::string Namespace;
 	std::string Bucket;
 	IoHash		Key;
+	IoHash		OptionalBaseKey;
 	std::string OpenIdProvider;
 	std::string AccessToken;
 	AuthMgr&	AuthManager;
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 7754f61cd..f993a7ec7 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -94,12 +94,14 @@ namespace {
 			{
 				return {nullptr, "Missing file name"};
 			}
-			bool ForceDisableBlocks	   = File["disableblocks"sv].AsBool(false);
-			bool ForceEnableTempBlocks = File["enabletempblocks"sv].AsBool(false);
+			std::string_view OptionalBaseName(File["basename"sv].AsString());
+			bool			 ForceDisableBlocks	   = File["disableblocks"sv].AsBool(false);
+			bool			 ForceEnableTempBlocks = File["enabletempblocks"sv].AsBool(false);
 
 			FileRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunkEmbedSize = MaxChunkEmbedSize},
 											  FolderPath,
 											  std::string(Name),
+											  std::string(OptionalBaseName),
 											  ForceDisableBlocks,
 											  ForceEnableTempBlocks};
 			RemoteStore					   = CreateFileRemoteStore(Options);
@@ -148,6 +150,21 @@ namespace {
 			{
 				return {nullptr, "Invalid key string"};
 			}
+			IoHash			 BaseKey	  = IoHash::Zero;
+			std::string_view BaseKeyParam = Cloud["basekey"sv].AsString();
+			if (!BaseKeyParam.empty())
+			{
+				if (BaseKeyParam.length() != IoHash::StringLength)
+				{
+					return {nullptr, "Invalid base key"};
+				}
+				BaseKey = IoHash::FromHexString(BaseKeyParam);
+				if (BaseKey == IoHash::Zero)
+				{
+					return {nullptr, "Invalid base key string"};
+				}
+			}
+
 			bool ForceDisableBlocks		= Cloud["disableblocks"sv].AsBool(false);
 			bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false);
 			bool AssumeHttp2			= Cloud["assumehttp2"sv].AsBool(false);
@@ -157,6 +174,7 @@ namespace {
 												 std::string(Namespace),
 												 std::string(Bucket),
 												 Key,
+												 BaseKey,
 												 std::string(OpenIdProvider),
 												 AccessToken,
 												 AuthManager,
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index 080517a8d..235166659 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -195,6 +195,7 @@ BuildContainer(CidStore&																	ChunkStore,
 			   size_t																		MaxBlockSize,
 			   size_t																		MaxChunkEmbedSize,
 			   bool																			BuildBlocks,
+			   const std::vector<Block>&													KnownBlocks,
 			   WorkerThreadPool&															WorkerPool,
 			   const std::function<void(CompressedBuffer&&, const IoHash&)>&				AsyncOnBlock,
 			   const std::function<void(const IoHash&)>&									OnLargeAttachment,
@@ -383,6 +384,53 @@ BuildContainer(CidStore&																	ChunkStore,
 			OpCount++;
 		});
 
+		if (!Attachments.empty() && !KnownBlocks.empty())
+		{
+			size_t ReusedBlockCount = 0;
+			ZEN_INFO("Checking {} known blocks for reuse", KnownBlocks.size());
+			for (const Block& KnownBlock : KnownBlocks)
+			{
+				size_t BlockAttachmentCount = KnownBlock.ChunksInBlock.size();
+				if (BlockAttachmentCount == 0)
+				{
+					continue;
+				}
+				size_t FoundAttachmentCount = 0;
+				for (const IoHash& KnownHash : KnownBlock.ChunksInBlock)
+				{
+					if (Attachments.contains(KnownHash))
+					{
+						FoundAttachmentCount++;
+					}
+				}
+
+				size_t ReusePercent = (FoundAttachmentCount * 100) / BlockAttachmentCount;
+				// TODO: Configure reuse-level
+				if (ReusePercent > 80)
+				{
+					ZEN_DEBUG("Reusing block {}. {} attachments found, usage level: {}%",
+							  KnownBlock.BlockHash,
+							  FoundAttachmentCount,
+							  ReusePercent);
+					for (const IoHash& KnownHash : KnownBlock.ChunksInBlock)
+					{
+						Attachments.erase(KnownHash);
+					}
+
+					BlocksLock.WithExclusiveLock([&]() { Blocks.push_back(KnownBlock); });
+					ReusedBlockCount++;
+				}
+				else if (FoundAttachmentCount > 0)
+				{
+					ZEN_DEBUG("Skipping block {}. {} attachments found, usage level: {}%",
+							  KnownBlock.BlockHash,
+							  FoundAttachmentCount,
+							  ReusePercent);
+				}
+			}
+			ZEN_INFO("Reusing {} out of {} known blocks", ReusedBlockCount, KnownBlocks.size());
+		}
+
 		ZEN_INFO("Sorting {} attachments from {} ops", Attachments.size(), OpLSNToKey.size());
 
 		// Sort attachments so we get predictable blocks for the same oplog upload
@@ -649,6 +697,7 @@ BuildContainer(CidStore&																	ChunkStore,
 											   MaxBlockSize,
 											   MaxChunkEmbedSize,
 											   BuildBlocks,
+											   {},
 											   WorkerPool,
 											   AsyncOnBlock,
 											   OnLargeAttachment,
@@ -778,6 +827,46 @@ SaveOplog(CidStore&				 ChunkStore,
 		OnBlock = UploadBlock;
 	}
 
+	std::vector<Block> KnownBlocks;
+
+	if (BuildBlocks)
+	{
+		ZEN_INFO("Loading oplog base container");
+		RemoteProjectStore::LoadContainerResult BaseContainerResult = RemoteStore.LoadBaseContainer();
+		if (BaseContainerResult.ErrorCode != static_cast<int>(HttpResponseCode::NoContent))
+		{
+			if (BaseContainerResult.ErrorCode)
+			{
+				ZEN_WARN("Failed to load oplog base container, reason: '{}', error code: {}",
+						 BaseContainerResult.Reason,
+						 BaseContainerResult.ErrorCode);
+			}
+			else
+			{
+				CbArrayView BlocksArray = BaseContainerResult.ContainerObject["blocks"sv].AsArrayView();
+				for (CbFieldView BlockField : BlocksArray)
+				{
+					CbObjectView BlockView = BlockField.AsObjectView();
+					IoHash		 BlockHash = BlockView["rawhash"sv].AsBinaryAttachment();
+
+					std::vector<IoHash> ChunksInBlock;
+					CbArrayView			ChunksArray = BlockView["chunks"sv].AsArrayView();
+					if (BlockHash == IoHash::Zero)
+					{
+						continue;
+					}
+
+					ChunksInBlock.reserve(ChunksArray.Num());
+					for (CbFieldView ChunkField : ChunksArray)
+					{
+						ChunksInBlock.push_back(ChunkField.AsHash());
+					}
+					KnownBlocks.push_back({.BlockHash = BlockHash, .ChunksInBlock = std::move(ChunksInBlock)});
+				};
+			}
+		}
+	}
+
 	tsl::robin_map<IoHash, IoBuffer, IoHash::Hasher> TempAttachments;
 	CbObject										 OplogContainerObject = BuildContainer(ChunkStore,
 												   Project,
@@ -785,6 +874,7 @@ SaveOplog(CidStore&				 ChunkStore,
 												   MaxBlockSize,
 												   MaxChunkEmbedSize,
 												   BuildBlocks,
+												   KnownBlocks,
 												   WorkerPool,
 												   OnBlock,
 												   OnLargeAttachment,
diff --git a/src/zenserver/projectstore/remoteprojectstore.h b/src/zenserver/projectstore/remoteprojectstore.h
index 6fb6e739e..3134fdb4a 100644
--- a/src/zenserver/projectstore/remoteprojectstore.h
+++ b/src/zenserver/projectstore/remoteprojectstore.h
@@ -69,6 +69,7 @@ public:
 	virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Payloads)			= 0;
 
 	virtual LoadContainerResult	  LoadContainer()										= 0;
+	virtual LoadContainerResult	  LoadBaseContainer()									= 0;
 	virtual LoadAttachmentResult  LoadAttachment(const IoHash& RawHash)					= 0;
 	virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) = 0;
 };
diff --git a/src/zenserver/projectstore/zenremoteprojectstore.cpp b/src/zenserver/projectstore/zenremoteprojectstore.cpp
index d2ebc1484..72a7f00f8 100644
--- a/src/zenserver/projectstore/zenremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/zenremoteprojectstore.cpp
@@ -281,6 +281,11 @@ public:
 		return Result;
 	}
 
+	virtual LoadContainerResult LoadBaseContainer() override
+	{
+		return LoadContainerResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent)}};
+	}
+
 	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
 	{
 		Stopwatch Timer;