fix oplog import during gcv2 (#62)v5.5.0-pre3v5.5.0-pre2

- Bugfix: Always pre-cache oplog when creating project store GCv2 referencer
- Bugfix: Correctly capture attachments imported with oplog to void them being GCd before oplog is written

1 files changed, 44 insertions, 39 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index d177b0b2b..b36c8caa0 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -854,6 +854,7 @@ ProjectStore::Oplog::Reset()
 		m_Storage = new OplogStorage(this, m_BasePath);
 		m_Storage->Open(true);
 		CleanDirectory(m_TempPath);
+		Write();
 	}
 	// Erase content on disk
 	if (!MovedDir.empty())
@@ -1244,15 +1245,15 @@ ProjectStore::Oplog::IterateUpdatedLSNs(RwLock::SharedLockScope&, std::function<
 {
 	if (m_UpdatedLSNs)
 	{
+		if (!m_Storage)
+		{
+			return;
+		}
 		for (int UpdatedLSN : *m_UpdatedLSNs)
 		{
-			std::optional<CbObject> UpdatedOp = GetOpByIndex(UpdatedLSN);
-			if (UpdatedOp)
+			if (const auto AddressEntryIt = m_OpAddressMap.find(UpdatedLSN); AddressEntryIt != m_OpAddressMap.end())
 			{
-				if (!Callback(UpdatedOp.value()))
-				{
-					break;
-				}
+				Callback(m_Storage->GetOp(AddressEntryIt->second));
 			}
 		}
 	}
@@ -3134,8 +3135,19 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie
 
 	auto OnChunkedAttachment = [](const ChunkedInfo&) {};
 
-	RemoteProjectStore::Result RemoteResult =
-		SaveOplogContainer(*Oplog, ContainerObject, HasAttachment, OnNeedBlock, OnNeedAttachment, OnChunkedAttachment, nullptr);
+	auto OnReferencedAttachments = [&Oplog](std::span<IoHash> RawHashes) { Oplog->CaptureAddedAttachments(RawHashes); };
+	// Make sure we retain any attachments we download before writing the oplog
+	Oplog->EnableUpdateCapture();
+	auto _ = MakeGuard([&Oplog]() { Oplog->DisableUpdateCapture(); });
+
+	RemoteProjectStore::Result RemoteResult = SaveOplogContainer(*Oplog,
+																 ContainerObject,
+																 OnReferencedAttachments,
+																 HasAttachment,
+																 OnNeedBlock,
+																 OnNeedAttachment,
+																 OnChunkedAttachment,
+																 nullptr);
 
 	if (RemoteResult.ErrorCode)
 	{
@@ -3899,7 +3911,7 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 class ProjectStoreReferenceChecker : public GcReferenceChecker
 {
 public:
-	ProjectStoreReferenceChecker(ProjectStore::Oplog& Owner, bool PreCache) : m_Oplog(Owner), m_PreCache(PreCache) {}
+	ProjectStoreReferenceChecker(ProjectStore::Oplog& Owner) : m_Oplog(Owner) {}
 
 	virtual ~ProjectStoreReferenceChecker()
 	{
@@ -3921,36 +3933,33 @@ public:
 
 	virtual void PreCache(GcCtx& Ctx) override
 	{
-		if (m_PreCache)
-		{
-			ZEN_TRACE_CPU("Store::PreCache");
-
-			Stopwatch  Timer;
-			const auto _ = MakeGuard([&] {
-				if (!Ctx.Settings.Verbose)
-				{
-					return;
-				}
-				ZEN_INFO("GCV2: projectstore [PRECACHE] '{}': precached {} references in {} from {}/{}",
-						 m_Oplog.m_BasePath,
-						 m_References.size(),
-						 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
-						 m_Oplog.m_OuterProject->Identifier,
-						 m_Oplog.OplogId());
-			});
+		ZEN_TRACE_CPU("Store::PreCache");
 
-			m_Oplog.EnableUpdateCapture();
-			m_OplogCaptureEnabled = true;
-
-			RwLock::SharedLockScope __(m_Oplog.m_OplogLock);
-			if (Ctx.IsCancelledFlag)
+		Stopwatch  Timer;
+		const auto _ = MakeGuard([&] {
+			if (!Ctx.Settings.Verbose)
 			{
 				return;
 			}
-			m_Oplog.IterateOplog([&](CbObjectView Op) {
-				Op.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); });
-			});
+			ZEN_INFO("GCV2: projectstore [PRECACHE] '{}': precached {} references in {} from {}/{}",
+					 m_Oplog.m_BasePath,
+					 m_References.size(),
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+					 m_Oplog.m_OuterProject->Identifier,
+					 m_Oplog.OplogId());
+		});
+
+		m_Oplog.EnableUpdateCapture();
+		m_OplogCaptureEnabled = true;
+
+		RwLock::SharedLockScope __(m_Oplog.m_OplogLock);
+		if (Ctx.IsCancelledFlag)
+		{
+			return;
 		}
+		m_Oplog.IterateOplog([&](CbObjectView Op) {
+			Op.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); });
+		});
 	}
 
 	virtual void LockState(GcCtx& Ctx) override
@@ -3972,7 +3981,6 @@ public:
 		});
 
 		m_OplogLock = std::make_unique<RwLock::SharedLockScope>(m_Oplog.m_OplogLock);
-
 		m_Oplog.IterateUpdatedLSNs(*m_OplogLock, [&](const CbObjectView& UpdateOp) -> bool {
 			UpdateOp.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); });
 			return true;
@@ -4021,7 +4029,6 @@ public:
 		});
 	}
 	ProjectStore::Oplog&					 m_Oplog;
-	bool									 m_PreCache;
 	std::unique_ptr<RwLock::SharedLockScope> m_OplogLock;
 	std::vector<IoHash>						 m_References;
 	bool									 m_OplogCaptureEnabled = false;
@@ -4075,9 +4082,7 @@ ProjectStore::CreateReferenceCheckers(GcCtx& Ctx)
 				{
 					continue;
 				}
-				GcClock::TimePoint Now		   = GcClock::Now();
-				bool			   TryPreCache = Project->LastOplogAccessTime(OpLogId) < (Now - std::chrono::minutes(5));
-				Checkers.emplace_back(new ProjectStoreReferenceChecker(*Oplog, TryPreCache));
+				Checkers.emplace_back(new ProjectStoreReferenceChecker(*Oplog));
 				OplogCount++;
 			}
 		}