Add ability to read the oplog's ReferencedSet, as written by the cook… (#190)v5.5.9-pre7

Add ability to read the oplog's ReferencedSet, as written by the cooker, from the ReferencedSet op. Filter oplog entries requests by the ReferencedSet, if trim_by_referencedset parameter is present.. Add -trim=true/false parameter to oplog-mirror command, default to true, to request the trimmed/not trimmed oplog.
Helper functions: Add paging to IterateOpLogWithKey. Add unit tests for IterateOpLog functions. Move OpKeyStringAsOid from httpprojectstore into projectstore.

1 files changed, 197 insertions, 8 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 6dbdb7029..a5bd1859b 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -2016,6 +2016,17 @@ ProjectStore::Oplog::IterateOplog(std::function<void(CbObjectView)>&& Handler, c
 	IterateOplogLocked(std::move(Handler), EntryPaging);
 }
 
+template<typename ContainerElement>
+std::span<ContainerElement>
+CreateSpanFromPaging(std::vector<ContainerElement>& Container, const ProjectStore::Oplog::Paging& Paging)
+{
+	std::span<ContainerElement> Span(Container);
+	int32_t						Size  = int32_t(Container.size());
+	int32_t						Start = std::clamp(Paging.Start, 0, Size);
+	int32_t						End	  = Paging.Count < 0 ? Size : (Start + std::min<int32_t>(Paging.Count, Size - Start));
+	return Span.subspan(Start, End - Start);
+}
+
 void
 ProjectStore::Oplog::IterateOplogLocked(std::function<void(CbObjectView)>&& Handler, const Paging& EntryPaging)
 {
@@ -2040,12 +2051,7 @@ ProjectStore::Oplog::IterateOplogLocked(std::function<void(CbObjectView)>&& Hand
 		return Lhs.Offset < Rhs.Offset;
 	});
 
-	std::span<OplogEntryAddress> EntrySpan = Entries;
-	int32_t						 Size	   = int32_t(Entries.size());
-	int32_t						 Start	   = std::clamp(EntryPaging.Start, 0, Size);
-	int32_t						 End	   = std::clamp<uint32_t>(EntryPaging.Start + EntryPaging.Count, 0, Size);
-	EntrySpan							   = EntrySpan.subspan(Start, End - Start);
-
+	std::span<OplogEntryAddress> EntrySpan = CreateSpanFromPaging(Entries, EntryPaging);
 	m_Storage->ReplayLogEntries(EntrySpan, [&](CbObjectView Op) { Handler(Op); });
 }
 
@@ -2145,6 +2151,12 @@ ProjectStore::Oplog::GetOplogEntryCount() const
 void
 ProjectStore::Oplog::IterateOplogWithKey(std::function<void(uint32_t, const Oid&, CbObjectView)>&& Handler)
 {
+	IterateOplogWithKey(std::move(Handler), Paging{});
+}
+
+void
+ProjectStore::Oplog::IterateOplogWithKey(std::function<void(uint32_t, const Oid&, CbObjectView)>&& Handler, const Paging& EntryPaging)
+{
 	RwLock::SharedLockScope _(m_OplogLock);
 	if (!m_Storage)
 	{
@@ -2197,8 +2209,9 @@ ProjectStore::Oplog::IterateOplogWithKey(std::function<void(uint32_t, const Oid&
 		}
 	}
 
-	size_t EntryIndex = 0;
-	m_Storage->ReplayLogEntries(SortedEntries, [&](CbObjectView Op) {
+	std::span<OplogEntryAddress> EntrySpan	= CreateSpanFromPaging(SortedEntries, EntryPaging);
+	size_t						 EntryIndex = EntrySpan.empty() ? 0 : static_cast<size_t>(&EntrySpan.front() - &SortedEntries.front());
+	m_Storage->ReplayLogEntries(EntrySpan, [&](CbObjectView Op) {
 		Handler(SortedLSNs[EntryIndex], SortedKeys[EntryIndex], Op);
 		EntryIndex++;
 	});
@@ -5517,6 +5530,26 @@ ProjectStore::LockState(GcCtx& Ctx)
 
 //////////////////////////////////////////////////////////////////////////
 
+Oid
+OpKeyStringAsOid(std::string_view OpKey)
+{
+	using namespace std::literals;
+
+	CbObjectWriter Writer;
+	Writer << "key"sv << OpKey;
+
+	XXH3_128Stream KeyHasher;
+	Writer.Save()["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
+	XXH3_128 KeyHash = KeyHasher.GetHash();
+
+	Oid OpId;
+	memcpy(OpId.OidBits, &KeyHash, sizeof(OpId.OidBits));
+
+	return OpId;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
 #if ZEN_WITH_TESTS
 
 namespace testutils {
@@ -6106,6 +6139,162 @@ TEST_CASE("project.store.block")
 	CHECK(IterateBlock(Block.Decompress(), [](CompressedBuffer&&, const IoHash&) {}));
 }
 
+TEST_CASE("project.store.iterateoplog")
+{
+	using namespace std::literals;
+	using namespace testutils;
+
+	ScopedTemporaryDirectory TempDir;
+
+	auto				  JobQueue = MakeJobQueue(1, ""sv);
+	GcManager			  Gc;
+	CidStore			  CidStore(Gc);
+	CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
+	CidStore.Initialize(CidConfig);
+
+	std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv;
+	ProjectStore		  ProjectStore(CidStore, BasePath, Gc, *JobQueue, ProjectStore::Configuration{});
+	std::filesystem::path RootDir		= TempDir.Path() / "root"sv;
+	std::filesystem::path EngineRootDir = TempDir.Path() / "enginesv";
+
+	std::filesystem::path ProjectRootDir  = TempDir.Path() / "game"sv;
+	std::filesystem::path ProjectFilePath = TempDir.Path() / "game"sv / "game.uproject"sv;
+	{
+		CreateDirectories(ProjectFilePath.parent_path());
+		BasicFile ProjectFile;
+		ProjectFile.Open(ProjectFilePath, BasicFile::Mode::kTruncate);
+	}
+	std::filesystem::path ProjectOplogPath = TempDir.Path() / "game"sv / "saves"sv / "cooked"sv / ".projectstore"sv;
+	{
+		CreateDirectories(ProjectOplogPath.parent_path());
+		BasicFile OplogFile;
+		OplogFile.Open(ProjectOplogPath, BasicFile::Mode::kTruncate);
+	}
+
+	Ref<ProjectStore::Project> TestProject(ProjectStore.NewProject(BasePath / "proj"sv,
+																   "proj"sv,
+																   RootDir.string(),
+																   EngineRootDir.string(),
+																   ProjectRootDir.string(),
+																   ProjectFilePath.string()));
+	ProjectStore::Oplog*	   Oplog = TestProject->NewOplog("oplog"sv, ProjectOplogPath);
+	CHECK(Oplog != nullptr);
+
+	struct TestOidData
+	{
+		Oid			KeyAsOidNotOplogId = Oid::NewOid();
+		std::string Key				   = KeyAsOidNotOplogId.ToString();
+		bool		bFound			   = false;
+	};
+	constexpr int NumTestOids = 4;
+	TestOidData	  TestOids[NumTestOids];
+	for (const TestOidData& TestOid : TestOids)
+	{
+		Oplog->AppendNewOplogEntry(CreateOplogPackage(TestOid.KeyAsOidNotOplogId, {}));
+	}
+	int Count = 0;
+
+	auto ResetTest = [&Count, &TestOids]() {
+		Count = 0;
+		for (TestOidData& TestOid : TestOids)
+		{
+			TestOid.bFound = false;
+		}
+	};
+	auto IncrementCount = [&Count](CbObjectView /* Op */) { ++Count; };
+	auto MarkFound		= [&TestOids, &Count](uint32_t /* LSN */, const Oid& /* InId */, CbObjectView Op) {
+		 for (TestOidData& TestOid : TestOids)
+		 {
+			 if (Op["key"sv].AsString() == TestOid.Key)
+			 {
+				 TestOid.bFound = true;
+				 ++Count;
+			 }
+		 }
+	};
+
+	// Tests of IterateOpLog and IterateOplogWithKey, with various Paging arguments
+	{
+		ResetTest();
+		Oplog->IterateOplog(IncrementCount, ProjectStore::Oplog::Paging{});
+		CHECK(Count == NumTestOids);
+		ResetTest();
+		Oplog->IterateOplogWithKey(MarkFound, ProjectStore::Oplog::Paging{});
+		CHECK(Count == NumTestOids);
+		ResetTest();
+		Oplog->IterateOplogWithKey(MarkFound);
+		CHECK(Count == NumTestOids);
+
+		Count = 0;
+		for (int Start = 0; Start < NumTestOids; ++Start)
+		{
+			for (int Size = 0; Size < NumTestOids - Start; ++Size)
+			{
+				ResetTest();
+				Oplog->IterateOplog(IncrementCount, ProjectStore::Oplog::Paging{Start, Size});
+				CHECK(Count == Size);
+				ResetTest();
+				Oplog->IterateOplogWithKey(MarkFound, ProjectStore::Oplog::Paging{Start, Size});
+				CHECK(Count == Size);
+			}
+
+			// Out of range Size arguments
+			ResetTest();
+			Oplog->IterateOplog(IncrementCount, ProjectStore::Oplog::Paging{Start, -1});
+			CHECK(Count == NumTestOids - Start);
+			ResetTest();
+			Oplog->IterateOplogWithKey(MarkFound, ProjectStore::Oplog::Paging{Start, NumTestOids * 2});
+			CHECK(Count == NumTestOids - Start);
+		}
+
+		// Out of range Start arguments
+		for (int Size = 0; Size < NumTestOids; ++Size)
+		{
+			ResetTest();
+			Oplog->IterateOplog(IncrementCount, ProjectStore::Oplog::Paging{-1, Size});
+			CHECK(Count == Size);
+			ResetTest();
+			Oplog->IterateOplogWithKey(MarkFound, ProjectStore::Oplog::Paging{-1, Size});
+			CHECK(Count == Size);
+
+			ResetTest();
+			Oplog->IterateOplog(IncrementCount, ProjectStore::Oplog::Paging{NumTestOids, Size});
+			CHECK(Count == 0);
+			ResetTest();
+			Oplog->IterateOplogWithKey(MarkFound, ProjectStore::Oplog::Paging{NumTestOids, Size});
+			CHECK(Count == 0);
+		}
+		// Out of range Start and Size arguments
+		ResetTest();
+		Oplog->IterateOplog(IncrementCount, ProjectStore::Oplog::Paging{-1, -1});
+		CHECK(Count == NumTestOids);
+		ResetTest();
+		Oplog->IterateOplogWithKey(MarkFound, ProjectStore::Oplog::Paging{-1, -1});
+		CHECK(Count == NumTestOids);
+
+		ResetTest();
+		Oplog->IterateOplog(IncrementCount, ProjectStore::Oplog::Paging{-1, 2 * NumTestOids});
+		CHECK(Count == NumTestOids);
+		ResetTest();
+		Oplog->IterateOplogWithKey(MarkFound, ProjectStore::Oplog::Paging{-1, 2 * NumTestOids});
+		CHECK(Count == NumTestOids);
+
+		ResetTest();
+		Oplog->IterateOplog(IncrementCount, ProjectStore::Oplog::Paging{NumTestOids, -1});
+		CHECK(Count == 0);
+		ResetTest();
+		Oplog->IterateOplogWithKey(MarkFound, ProjectStore::Oplog::Paging{NumTestOids, -1});
+		CHECK(Count == 0);
+
+		ResetTest();
+		Oplog->IterateOplog(IncrementCount, ProjectStore::Oplog::Paging{NumTestOids, 2 * NumTestOids});
+		CHECK(Count == 0);
+		ResetTest();
+		Oplog->IterateOplogWithKey(MarkFound, ProjectStore::Oplog::Paging{NumTestOids, 2 * NumTestOids});
+		CHECK(Count == 0);
+	}
+}
+
 #endif
 
 void