Keep track of added ops during GCV2 instead of rescanning full oplog when added ops are detected (#652)

2 files changed, 25 insertions, 8 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 72a8e1409..7d4ce3809 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1282,6 +1282,11 @@ ProjectStore::Oplog::RegisterOplogEntry(RwLock::ExclusiveLockScope& OplogLock,
 	m_OpAddressMap.emplace(OpEntry.OpLsn, OplogEntryAddress{.Offset = OpEntry.OpCoreOffset, .Size = OpEntry.OpCoreSize});
 	m_LatestOpMap[OpEntry.OpKeyHash] = OpEntry.OpLsn;
 
+	if (m_UpdatedLSNs)
+	{
+		m_UpdatedLSNs->push_back(OpEntry.OpLsn);
+	}
+
 	return OpEntry.OpLsn;
 }
 
@@ -3571,7 +3576,11 @@ class ProjectStoreReferenceChecker : public GcReferenceChecker
 public:
 	ProjectStoreReferenceChecker(ProjectStore::Oplog& Owner, bool PreCache) : m_Oplog(Owner), m_PreCache(PreCache) {}
 
-	virtual ~ProjectStoreReferenceChecker() {}
+	virtual ~ProjectStoreReferenceChecker()
+	{
+		m_OplogLock.reset();
+		m_Oplog.m_OplogLock.WithExclusiveLock([&]() { m_Oplog.m_UpdatedLSNs.reset(); });
+	}
 
 	virtual void PreCache(GcCtx& Ctx) override
 	{
@@ -3593,6 +3602,8 @@ public:
 						 m_Oplog.OplogId());
 			});
 
+			m_Oplog.m_OplogLock.WithExclusiveLock([&]() { m_Oplog.m_UpdatedLSNs = std::make_unique<std::vector<int>>(); });
+
 			RwLock::SharedLockScope __(m_Oplog.m_OplogLock);
 			if (Ctx.IsCancelledFlag)
 			{
@@ -3624,14 +3635,18 @@ public:
 		});
 
 		m_OplogLock = std::make_unique<RwLock::SharedLockScope>(m_Oplog.m_OplogLock);
-		if (m_Oplog.GetMaxOpIndex() != m_PreCachedLsn)
+
+		if (m_Oplog.m_UpdatedLSNs)
 		{
-			// TODO: Maybe we could just check the added oplog entries - we might get a few extra references from obsolete entries
-			// but I don't think that would be critical
-			m_References.resize(0);
-			m_Oplog.IterateOplog([&](CbObjectView Op) {
-				Op.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); });
-			});
+			for (int UpdatedLSN : *m_Oplog.m_UpdatedLSNs)
+			{
+				std::optional<CbObject> UpdatedOp = m_Oplog.GetOpByIndex(UpdatedLSN);
+				if (UpdatedOp)
+				{
+					CbObjectView Op = UpdatedOp.value();
+					Op.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); });
+				}
+			}
 		}
 	}
 
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 5b873b758..897231a2e 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -164,6 +164,8 @@ public:
 		tsl::robin_map<int, OplogEntryAddress> m_OpAddressMap;	   // Index LSN -> op data in ops blob file
 		OidMap<int>							   m_LatestOpMap;	   // op key -> latest op LSN for key
 
+		std::unique_ptr<std::vector<int>> m_UpdatedLSNs;
+
 		RefPtr<OplogStorage> m_Storage;
 		std::string			 m_OplogId;