Add `gc-attachment-passes` option to zenserver (#167)

Added option `gc-attachment-passes` to zenserver
Cleaned up GCv2 start and stop logs and added identifier to easily find matching start and end of a GC pass in log file
Fixed project store not properly sorting references found during lock phase

5 files changed, 33 insertions, 7 deletions

diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index cd336c715..88290171d 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -291,6 +291,7 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 				Response << "CompactBlockUsageThresholdPercent" << State.Config.CompactBlockUsageThresholdPercent;
 				Response << "Verbose" << State.Config.Verbose;
 				Response << "SingleThreaded" << State.Config.SingleThreaded;
+				Response << "AttachmentPassCount" << State.Config.AttachmentPassCount;
 			}
 			Response.EndObject();
 			Response << "AreDiskWritesBlocked" << State.AreDiskWritesBlocked;
@@ -318,6 +319,11 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 					Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize);
 					Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize);
 				}
+				if (State.LastFullAttachmentRangeMin != IoHash::Zero || State.LastFullAttachmentRangeMax != IoHash::Max)
+				{
+					Response << "AttachmentRangeMin" << State.LastFullAttachmentRangeMin;
+					Response << "AttachmentRangeMax" << State.LastFullAttachmentRangeMax;
+				}
 			}
 			Response.EndObject();
 			Response.BeginObject("LightweightGC");
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 6c2bf40d8..2023a9d51 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -165,6 +165,11 @@ ValidateOptions(ZenServerOptions& ServerOptions)
 	{
 		throw zen::OptionParseException("Dedicated server can not be used with forced local server address");
 	}
+	if (ServerOptions.GcConfig.AttachmentPassCount > ZenGcConfig::GcMaxAttachmentPassCount)
+	{
+		throw zen::OptionParseException(
+			fmt::format("GC attachment pass count can not be larger than {}", ZenGcConfig::GcMaxAttachmentPassCount));
+	}
 }
 
 UpstreamCachePolicy
@@ -524,6 +529,7 @@ ParseConfigFile(const std::filesystem::path& Path,
 	LuaOptions.AddOption("gc.projectstore.attachment.store"sv,
 						 ServerOptions.ProjectStoreConfig.StoreAttachmentMetaData,
 						 "gc-projectstore-attachment-store");
+	LuaOptions.AddOption("gc.attachment.passes"sv, ServerOptions.GcConfig.AttachmentPassCount, "gc-attachment-passes"sv);
 
 	////// gc
 	LuaOptions.AddOption("gc.cache.maxdurationseconds"sv, ServerOptions.GcConfig.Cache.MaxDurationSeconds, "gc-cache-duration-seconds"sv);
@@ -1015,6 +1021,14 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 					   cxxopts::value<bool>(ServerOptions.GcConfig.SingleThreaded)->default_value("false"),
 					   "");
 
+	options.add_option("gc",
+					   "",
+					   "gc-attachment-passes",
+					   "Limit the range of unreferenced attachments included in GC check by breaking it into passes. Default is one pass "
+					   "which includes all the attachments.",
+					   cxxopts::value<uint16_t>(ServerOptions.GcConfig.AttachmentPassCount)->default_value("1"),
+					   "");
+
 	options.add_option("objectstore",
 					   "",
 					   "objectstore-enabled",
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index 6e45e7230..e7b734001 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -76,6 +76,8 @@ struct ZenGcConfig
 	uint32_t					  CompactBlockUsageThresholdPercent = 90;
 	bool						  Verbose							= false;
 	bool						  SingleThreaded					= false;
+	static constexpr uint16_t	  GcMaxAttachmentPassCount			= 256;
+	uint16_t					  AttachmentPassCount				= 1;
 };
 
 struct ZenOpenIdProviderConfig
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index be50a03e2..903c31ecd 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -5410,7 +5410,7 @@ public:
 
 			Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		}
-		FilterReferences(Ctx, m_References);
+		FilterReferences(Ctx, fmt::format("projectstore [LOCKSTATE] '{}'", "projectstore"), m_References);
 	}
 
 	virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override
@@ -5544,7 +5544,7 @@ public:
 
 		Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		m_OplogAccessTime = m_Project->LastOplogAccessTime(m_OplogId);
-		FilterReferences(Ctx, m_References);
+		FilterReferences(Ctx, fmt::format("projectstore [PRECACHE] '{}'", m_OplogBasePath), m_References);
 	}
 
 	virtual void UpdateLockedState(GcCtx& Ctx) override
@@ -5561,7 +5561,7 @@ public:
 			}
 			ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': found {} references in {} from {}/{}",
 					 m_OplogBasePath,
-					 m_References.size(),
+					 m_AddedReferences.size(),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
 					 m_Project->Identifier,
 					 m_OplogId);
@@ -5571,11 +5571,11 @@ public:
 		{
 			ProjectStore::Oplog* Oplog = It->second.get();
 			Oplog->IterateCapturedLSNs([&](const CbObjectView& UpdateOp) -> bool {
-				UpdateOp.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); });
+				UpdateOp.IterateAttachments([&](CbFieldView Visitor) { m_AddedReferences.emplace_back(Visitor.AsAttachment()); });
 				return true;
 			});
 			std::vector<IoHash> AddedAttachments = Oplog->GetCapturedAttachments();
-			m_References.insert(m_References.end(), AddedAttachments.begin(), AddedAttachments.end());
+			m_AddedReferences.insert(m_AddedReferences.end(), AddedAttachments.begin(), AddedAttachments.end());
 		}
 		else if (m_Project->LastOplogAccessTime(m_OplogId) > m_OplogAccessTime && ProjectStore::Oplog::ExistsAt(m_OplogBasePath))
 		{
@@ -5588,8 +5588,9 @@ public:
 				}
 			});
 			Oplog->Read();
-			Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
+			Oplog->GetAttachmentsLocked(m_AddedReferences, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		}
+		FilterReferences(Ctx, fmt::format("projectstore [LOCKSTATE] '{}'", m_OplogBasePath), m_AddedReferences);
 	}
 
 	virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override
@@ -5617,6 +5618,7 @@ public:
 		});
 
 		std::span<IoHash> UnusedReferences = KeepUnusedReferences(m_References, IoCids);
+		UnusedReferences				   = KeepUnusedReferences(m_AddedReferences, UnusedReferences);
 		UsedCount						   = IoCids.size() - UnusedReferences.size();
 		return UnusedReferences;
 	}
@@ -5627,6 +5629,7 @@ public:
 	std::filesystem::path	   m_OplogBasePath;
 	ProjectStore::Oplog*	   m_OplogWithUpdateCapture = nullptr;
 	std::vector<IoHash>		   m_References;
+	std::vector<IoHash>		   m_AddedReferences;
 	GcClock::TimePoint		   m_OplogAccessTime;
 };
 
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index ebc56d7d9..3c5d46a33 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -321,7 +321,8 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 							   .UseGCVersion					  = ServerOptions.GcConfig.UseGCV2 ? GcVersion::kV2 : GcVersion::kV1,
 							   .CompactBlockUsageThresholdPercent = ServerOptions.GcConfig.CompactBlockUsageThresholdPercent,
 							   .Verbose							  = ServerOptions.GcConfig.Verbose,
-							   .SingleThreaded					  = ServerOptions.GcConfig.SingleThreaded};
+							   .SingleThreaded					  = ServerOptions.GcConfig.SingleThreaded,
+							   .AttachmentPassCount				  = ServerOptions.GcConfig.AttachmentPassCount};
 	m_GcScheduler.Initialize(GcConfig);
 
 	// Create and register admin interface last to make sure all is properly initialized