fix missing chunk in oplog export (#925)

* add reused block to oplog during export

1 files changed, 160 insertions, 0 deletions

diff --git a/src/zenremotestore/projectstore/remoteprojectstore.cpp b/src/zenremotestore/projectstore/remoteprojectstore.cpp
index 2076adb70..07588e58e 100644
--- a/src/zenremotestore/projectstore/remoteprojectstore.cpp
+++ b/src/zenremotestore/projectstore/remoteprojectstore.cpp
@@ -3006,6 +3006,17 @@ BuildContainer(CidStore&																	ChunkStore,
 		return {};
 	}
 
+	// Reused blocks were not composed (their chunks were erased from UploadAttachments) but must
+	// still appear in the container so that a fresh receiver knows to download them.
+	if (BuildBlocks)
+	{
+		for (size_t KnownBlockIndex : ReusedBlockIndexes)
+		{
+			const ChunkBlockDescription& Reused = KnownBlocks[KnownBlockIndex];
+			Blocks.push_back(Reused);
+		}
+	}
+
 	CbObjectWriter			OplogContainerWriter;
 	RwLock::SharedLockScope _(BlocksLock);
 	OplogContainerWriter.AddBinary("ops"sv, CompressedOpsSection.GetCompressed().Flatten().AsIoBuffer());
@@ -7585,6 +7596,155 @@ TEST_CASE("loadoplog.clean_oplog_with_populated_cache")
 	}
 }
 
+TEST_CASE("project.store.export.block_reuse_fresh_receiver")
+{
+	// Regression test: after a second export that reuses existing blocks, a fresh import must still
+	// receive all chunks. The bug: FindReuseBlocks erases reused-block chunks from UploadAttachments,
+	// but never adds the reused blocks to the container's "blocks" section. A fresh receiver then
+	// silently misses those chunks because ParseOplogContainer never sees them.
+	using namespace projectstore_testutils;
+	using namespace std::literals;
+
+	ScopedTemporaryDirectory TempDir;
+	ScopedTemporaryDirectory ExportDir;
+
+	// -- Export side ----------------------------------------------------------
+	GcManager			  ExportGc;
+	CidStore			  ExportCidStore(ExportGc);
+	CidStoreConfiguration ExportCidConfig = {.RootDirectory		 = TempDir.Path() / "export_cas",
+											 .TinyValueThreshold = 1024,
+											 .HugeValueThreshold = 4096};
+	ExportCidStore.Initialize(ExportCidConfig);
+
+	std::filesystem::path	   ExportBasePath = TempDir.Path() / "export_projectstore";
+	ProjectStore			   ExportProjectStore(ExportCidStore, ExportBasePath, ExportGc, ProjectStore::Configuration{});
+	std::filesystem::path	   RootDir		   = TempDir.Path() / "root";
+	std::filesystem::path	   EngineRootDir   = TempDir.Path() / "engine";
+	std::filesystem::path	   ProjectRootDir  = TempDir.Path() / "game";
+	std::filesystem::path	   ProjectFilePath = TempDir.Path() / "game" / "game.uproject";
+	Ref<ProjectStore::Project> ExportProject(ExportProjectStore.NewProject(ExportBasePath / "proj1"sv,
+																		   "proj1"sv,
+																		   RootDir.string(),
+																		   EngineRootDir.string(),
+																		   ProjectRootDir.string(),
+																		   ProjectFilePath.string()));
+
+	// 20 KB with None encoding: compressed ~ 20 KB < MaxChunkEmbedSize (32 KB) -> packed into blocks.
+	Ref<ProjectStore::Oplog> Oplog = ExportProject->NewOplog("oplog_reuse_rt", {});
+	REQUIRE(Oplog);
+	Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(
+		Oid::NewOid(),
+		CreateAttachments(std::initializer_list<size_t>{20u * 1024u, 20u * 1024u}, OodleCompressionLevel::None)));
+
+	TestWorkerPools	  Pools;
+	WorkerThreadPool& NetworkPool = Pools.NetworkPool;
+	WorkerThreadPool& WorkerPool  = Pools.WorkerPool;
+
+	constexpr size_t MaxBlockSize		= 64u * 1024u;
+	constexpr size_t MaxChunksPerBlock	= 1000;
+	constexpr size_t MaxChunkEmbedSize	= 32u * 1024u;
+	constexpr size_t ChunkFileSizeLimit = 64u * 1024u * 1024u;
+
+	// First export: creates blocks on disk.
+	FileRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize		 = MaxBlockSize,
+														 .MaxChunksPerBlock	 = MaxChunksPerBlock,
+														 .MaxChunkEmbedSize	 = MaxChunkEmbedSize,
+														 .ChunkFileSizeLimit = ChunkFileSizeLimit},
+									  /*.FolderPath         =*/ExportDir.Path(),
+									  /*.Name               =*/std::string("oplog_reuse_rt"),
+									  /*.OptionalBaseName   =*/std::string(),
+									  /*.ForceDisableBlocks =*/false,
+									  /*.ForceEnableTempBlocks =*/false};
+
+	std::shared_ptr<RemoteProjectStore> RemoteStore = CreateFileRemoteStore(Log(), Options);
+	SaveOplog(ExportCidStore,
+			  *RemoteStore,
+			  *ExportProject,
+			  *Oplog,
+			  NetworkPool,
+			  WorkerPool,
+			  MaxBlockSize,
+			  MaxChunksPerBlock,
+			  MaxChunkEmbedSize,
+			  ChunkFileSizeLimit,
+			  /*EmbedLooseFiles*/ true,
+			  /*ForceUpload*/ false,
+			  /*IgnoreMissingAttachments*/ false,
+			  /*OptionalContext*/ nullptr);
+
+	// Verify first export produced blocks.
+	RemoteProjectStore::GetKnownBlocksResult KnownAfterFirst = RemoteStore->GetKnownBlocks();
+	REQUIRE(!KnownAfterFirst.Blocks.empty());
+
+	// Second export to the SAME store: triggers block reuse via GetKnownBlocks.
+	SaveOplog(ExportCidStore,
+			  *RemoteStore,
+			  *ExportProject,
+			  *Oplog,
+			  NetworkPool,
+			  WorkerPool,
+			  MaxBlockSize,
+			  MaxChunksPerBlock,
+			  MaxChunkEmbedSize,
+			  ChunkFileSizeLimit,
+			  /*EmbedLooseFiles*/ true,
+			  /*ForceUpload*/ false,
+			  /*IgnoreMissingAttachments*/ false,
+			  /*OptionalContext*/ nullptr);
+
+	// Verify the container has no duplicate block entries.
+	{
+		RemoteProjectStore::LoadContainerResult ContainerResult = RemoteStore->LoadContainer();
+		REQUIRE(ContainerResult.ErrorCode == 0);
+		std::vector<IoHash> BlockHashes = GetBlockHashesFromOplog(ContainerResult.ContainerObject);
+		REQUIRE(!BlockHashes.empty());
+		std::unordered_set<IoHash, IoHash::Hasher> UniqueBlockHashes(BlockHashes.begin(), BlockHashes.end());
+		CHECK(UniqueBlockHashes.size() == BlockHashes.size());
+	}
+
+	// Collect all attachment hashes referenced by the oplog ops.
+	std::unordered_set<IoHash, IoHash::Hasher> ExpectedHashes;
+	Oplog->IterateOplogWithKey([&](int, const Oid&, CbObjectView Op) {
+		Op.IterateAttachments([&](CbFieldView FieldView) { ExpectedHashes.insert(FieldView.AsAttachment()); });
+	});
+	REQUIRE(!ExpectedHashes.empty());
+
+	// -- Import side (fresh, empty CAS) --------------------------------------
+	GcManager			  ImportGc;
+	CidStore			  ImportCidStore(ImportGc);
+	CidStoreConfiguration ImportCidConfig = {.RootDirectory		 = TempDir.Path() / "import_cas",
+											 .TinyValueThreshold = 1024,
+											 .HugeValueThreshold = 4096};
+	ImportCidStore.Initialize(ImportCidConfig);
+
+	std::filesystem::path	   ImportBasePath = TempDir.Path() / "import_projectstore";
+	ProjectStore			   ImportProjectStore(ImportCidStore, ImportBasePath, ImportGc, ProjectStore::Configuration{});
+	Ref<ProjectStore::Project> ImportProject(ImportProjectStore.NewProject(ImportBasePath / "proj1"sv,
+																		   "proj1"sv,
+																		   RootDir.string(),
+																		   EngineRootDir.string(),
+																		   ProjectRootDir.string(),
+																		   ProjectFilePath.string()));
+
+	Ref<ProjectStore::Oplog> ImportOplog = ImportProject->NewOplog("oplog_reuse_rt_import", {});
+	REQUIRE(ImportOplog);
+
+	LoadOplog(LoadOplogContext{.ChunkStore				 = ImportCidStore,
+							   .RemoteStore				 = *RemoteStore,
+							   .Oplog					 = *ImportOplog,
+							   .NetworkWorkerPool		 = NetworkPool,
+							   .WorkerPool				 = WorkerPool,
+							   .ForceDownload			 = true,
+							   .IgnoreMissingAttachments = false,
+							   .PartialBlockRequestMode	 = EPartialBlockRequestMode::All});
+
+	// Every attachment hash from the original oplog must be present in the import CAS.
+	for (const IoHash& Hash : ExpectedHashes)
+	{
+		CHECK_MESSAGE(ImportCidStore.ContainsChunk(Hash), "Missing chunk after import: ", Hash);
+	}
+}
+
 TEST_SUITE_END();
 
 #endif	// ZEN_WITH_TESTS