Add `gc-attachment-passes` option to zenserver (#167)

Added option `gc-attachment-passes` to zenserver
Cleaned up GCv2 start and stop logs and added identifier to easily find matching start and end of a GC pass in log file
Fixed project store not properly sorting references found during lock phase

1 files changed, 9 insertions, 6 deletions

diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index be50a03e2..903c31ecd 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -5410,7 +5410,7 @@ public:
 
 			Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		}
-		FilterReferences(Ctx, m_References);
+		FilterReferences(Ctx, fmt::format("projectstore [LOCKSTATE] '{}'", "projectstore"), m_References);
 	}
 
 	virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override
@@ -5544,7 +5544,7 @@ public:
 
 		Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		m_OplogAccessTime = m_Project->LastOplogAccessTime(m_OplogId);
-		FilterReferences(Ctx, m_References);
+		FilterReferences(Ctx, fmt::format("projectstore [PRECACHE] '{}'", m_OplogBasePath), m_References);
 	}
 
 	virtual void UpdateLockedState(GcCtx& Ctx) override
@@ -5561,7 +5561,7 @@ public:
 			}
 			ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': found {} references in {} from {}/{}",
 					 m_OplogBasePath,
-					 m_References.size(),
+					 m_AddedReferences.size(),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
 					 m_Project->Identifier,
 					 m_OplogId);
@@ -5571,11 +5571,11 @@ public:
 		{
 			ProjectStore::Oplog* Oplog = It->second.get();
 			Oplog->IterateCapturedLSNs([&](const CbObjectView& UpdateOp) -> bool {
-				UpdateOp.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); });
+				UpdateOp.IterateAttachments([&](CbFieldView Visitor) { m_AddedReferences.emplace_back(Visitor.AsAttachment()); });
 				return true;
 			});
 			std::vector<IoHash> AddedAttachments = Oplog->GetCapturedAttachments();
-			m_References.insert(m_References.end(), AddedAttachments.begin(), AddedAttachments.end());
+			m_AddedReferences.insert(m_AddedReferences.end(), AddedAttachments.begin(), AddedAttachments.end());
 		}
 		else if (m_Project->LastOplogAccessTime(m_OplogId) > m_OplogAccessTime && ProjectStore::Oplog::ExistsAt(m_OplogBasePath))
 		{
@@ -5588,8 +5588,9 @@ public:
 				}
 			});
 			Oplog->Read();
-			Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
+			Oplog->GetAttachmentsLocked(m_AddedReferences, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		}
+		FilterReferences(Ctx, fmt::format("projectstore [LOCKSTATE] '{}'", m_OplogBasePath), m_AddedReferences);
 	}
 
 	virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override
@@ -5617,6 +5618,7 @@ public:
 		});
 
 		std::span<IoHash> UnusedReferences = KeepUnusedReferences(m_References, IoCids);
+		UnusedReferences				   = KeepUnusedReferences(m_AddedReferences, UnusedReferences);
 		UsedCount						   = IoCids.size() - UnusedReferences.size();
 		return UnusedReferences;
 	}
@@ -5627,6 +5629,7 @@ public:
 	std::filesystem::path	   m_OplogBasePath;
 	ProjectStore::Oplog*	   m_OplogWithUpdateCapture = nullptr;
 	std::vector<IoHash>		   m_References;
+	std::vector<IoHash>		   m_AddedReferences;
 	GcClock::TimePoint		   m_OplogAccessTime;
 };