Add `gc-attachment-passes` option to zenserver (#167)

Added option `gc-attachment-passes` to zenserver
Cleaned up GCv2 start and stop logs and added identifier to easily find matching start and end of a GC pass in log file
Fixed project store not properly sorting references found during lock phase

12 files changed, 201 insertions, 56 deletions

diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp
index dd0bf83de..fbac0bbf3 100644
--- a/src/zen/cmds/admin_cmd.cpp
+++ b/src/zen/cmds/admin_cmd.cpp
@@ -205,12 +205,10 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	{
 		throw OptionParseException(fmt::format("invalid reference range, reference-high must be higher value than reference-low"));
 	}
-	if (LowRef != IoHash::Zero)
+
+	if (!m_ReferenceHashLow.empty() || !m_ReferenceHashHigh.empty())
 	{
 		Params.Add({"referencehashlow", LowRef.ToHexString()});
-	}
-	if (HighRef != IoHash::Max)
-	{
 		Params.Add({"referencehashhigh", HighRef.ToHexString()});
 	}
 
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index cd336c715..88290171d 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -291,6 +291,7 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 				Response << "CompactBlockUsageThresholdPercent" << State.Config.CompactBlockUsageThresholdPercent;
 				Response << "Verbose" << State.Config.Verbose;
 				Response << "SingleThreaded" << State.Config.SingleThreaded;
+				Response << "AttachmentPassCount" << State.Config.AttachmentPassCount;
 			}
 			Response.EndObject();
 			Response << "AreDiskWritesBlocked" << State.AreDiskWritesBlocked;
@@ -318,6 +319,11 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 					Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize);
 					Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize);
 				}
+				if (State.LastFullAttachmentRangeMin != IoHash::Zero || State.LastFullAttachmentRangeMax != IoHash::Max)
+				{
+					Response << "AttachmentRangeMin" << State.LastFullAttachmentRangeMin;
+					Response << "AttachmentRangeMax" << State.LastFullAttachmentRangeMax;
+				}
 			}
 			Response.EndObject();
 			Response.BeginObject("LightweightGC");
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 6c2bf40d8..2023a9d51 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -165,6 +165,11 @@ ValidateOptions(ZenServerOptions& ServerOptions)
 	{
 		throw zen::OptionParseException("Dedicated server can not be used with forced local server address");
 	}
+	if (ServerOptions.GcConfig.AttachmentPassCount > ZenGcConfig::GcMaxAttachmentPassCount)
+	{
+		throw zen::OptionParseException(
+			fmt::format("GC attachment pass count can not be larger than {}", ZenGcConfig::GcMaxAttachmentPassCount));
+	}
 }
 
 UpstreamCachePolicy
@@ -524,6 +529,7 @@ ParseConfigFile(const std::filesystem::path& Path,
 	LuaOptions.AddOption("gc.projectstore.attachment.store"sv,
 						 ServerOptions.ProjectStoreConfig.StoreAttachmentMetaData,
 						 "gc-projectstore-attachment-store");
+	LuaOptions.AddOption("gc.attachment.passes"sv, ServerOptions.GcConfig.AttachmentPassCount, "gc-attachment-passes"sv);
 
 	////// gc
 	LuaOptions.AddOption("gc.cache.maxdurationseconds"sv, ServerOptions.GcConfig.Cache.MaxDurationSeconds, "gc-cache-duration-seconds"sv);
@@ -1015,6 +1021,14 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 					   cxxopts::value<bool>(ServerOptions.GcConfig.SingleThreaded)->default_value("false"),
 					   "");
 
+	options.add_option("gc",
+					   "",
+					   "gc-attachment-passes",
+					   "Limit the range of unreferenced attachments included in GC check by breaking it into passes. Default is one pass "
+					   "which includes all the attachments.",
+					   cxxopts::value<uint16_t>(ServerOptions.GcConfig.AttachmentPassCount)->default_value("1"),
+					   "");
+
 	options.add_option("objectstore",
 					   "",
 					   "objectstore-enabled",
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index 6e45e7230..e7b734001 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -76,6 +76,8 @@ struct ZenGcConfig
 	uint32_t					  CompactBlockUsageThresholdPercent = 90;
 	bool						  Verbose							= false;
 	bool						  SingleThreaded					= false;
+	static constexpr uint16_t	  GcMaxAttachmentPassCount			= 256;
+	uint16_t					  AttachmentPassCount				= 1;
 };
 
 struct ZenOpenIdProviderConfig
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index be50a03e2..903c31ecd 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -5410,7 +5410,7 @@ public:
 
 			Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		}
-		FilterReferences(Ctx, m_References);
+		FilterReferences(Ctx, fmt::format("projectstore [LOCKSTATE] '{}'", "projectstore"), m_References);
 	}
 
 	virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override
@@ -5544,7 +5544,7 @@ public:
 
 		Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		m_OplogAccessTime = m_Project->LastOplogAccessTime(m_OplogId);
-		FilterReferences(Ctx, m_References);
+		FilterReferences(Ctx, fmt::format("projectstore [PRECACHE] '{}'", m_OplogBasePath), m_References);
 	}
 
 	virtual void UpdateLockedState(GcCtx& Ctx) override
@@ -5561,7 +5561,7 @@ public:
 			}
 			ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': found {} references in {} from {}/{}",
 					 m_OplogBasePath,
-					 m_References.size(),
+					 m_AddedReferences.size(),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
 					 m_Project->Identifier,
 					 m_OplogId);
@@ -5571,11 +5571,11 @@ public:
 		{
 			ProjectStore::Oplog* Oplog = It->second.get();
 			Oplog->IterateCapturedLSNs([&](const CbObjectView& UpdateOp) -> bool {
-				UpdateOp.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); });
+				UpdateOp.IterateAttachments([&](CbFieldView Visitor) { m_AddedReferences.emplace_back(Visitor.AsAttachment()); });
 				return true;
 			});
 			std::vector<IoHash> AddedAttachments = Oplog->GetCapturedAttachments();
-			m_References.insert(m_References.end(), AddedAttachments.begin(), AddedAttachments.end());
+			m_AddedReferences.insert(m_AddedReferences.end(), AddedAttachments.begin(), AddedAttachments.end());
 		}
 		else if (m_Project->LastOplogAccessTime(m_OplogId) > m_OplogAccessTime && ProjectStore::Oplog::ExistsAt(m_OplogBasePath))
 		{
@@ -5588,8 +5588,9 @@ public:
 				}
 			});
 			Oplog->Read();
-			Oplog->GetAttachmentsLocked(m_References, m_ProjectStore.m_Config.StoreAttachmentMetaData);
+			Oplog->GetAttachmentsLocked(m_AddedReferences, m_ProjectStore.m_Config.StoreAttachmentMetaData);
 		}
+		FilterReferences(Ctx, fmt::format("projectstore [LOCKSTATE] '{}'", m_OplogBasePath), m_AddedReferences);
 	}
 
 	virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override
@@ -5617,6 +5618,7 @@ public:
 		});
 
 		std::span<IoHash> UnusedReferences = KeepUnusedReferences(m_References, IoCids);
+		UnusedReferences				   = KeepUnusedReferences(m_AddedReferences, UnusedReferences);
 		UsedCount						   = IoCids.size() - UnusedReferences.size();
 		return UnusedReferences;
 	}
@@ -5627,6 +5629,7 @@ public:
 	std::filesystem::path	   m_OplogBasePath;
 	ProjectStore::Oplog*	   m_OplogWithUpdateCapture = nullptr;
 	std::vector<IoHash>		   m_References;
+	std::vector<IoHash>		   m_AddedReferences;
 	GcClock::TimePoint		   m_OplogAccessTime;
 };
 
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index ebc56d7d9..3c5d46a33 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -321,7 +321,8 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 							   .UseGCVersion					  = ServerOptions.GcConfig.UseGCV2 ? GcVersion::kV2 : GcVersion::kV1,
 							   .CompactBlockUsageThresholdPercent = ServerOptions.GcConfig.CompactBlockUsageThresholdPercent,
 							   .Verbose							  = ServerOptions.GcConfig.Verbose,
-							   .SingleThreaded					  = ServerOptions.GcConfig.SingleThreaded};
+							   .SingleThreaded					  = ServerOptions.GcConfig.SingleThreaded,
+							   .AttachmentPassCount				  = ServerOptions.GcConfig.AttachmentPassCount};
 	m_GcScheduler.Initialize(GcConfig);
 
 	// Create and register admin interface last to make sure all is properly initialized
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 417b63fb4..08cb346b2 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -3796,7 +3796,7 @@ public:
 			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences.reset(); });
 			return;
 		}
-		FilterReferences(Ctx, m_PrecachedReferences);
+		FilterReferences(Ctx, fmt::format("cachebucket [PRECACHE] '{}'", m_CacheBucket.m_BucketDir), m_PrecachedReferences);
 	}
 
 	virtual void UpdateLockedState(GcCtx& Ctx) override
@@ -3827,7 +3827,7 @@ public:
 		ZEN_ASSERT(m_CacheBucket.m_TrackedReferences);
 
 		m_AddedReferences = std::move(*m_CacheBucket.m_TrackedReferences);
-		FilterReferences(Ctx, m_AddedReferences);
+		FilterReferences(Ctx, fmt::format("cachebucket [LOCKSTATE] '{}'", m_CacheBucket.m_BucketDir), m_AddedReferences);
 	}
 
 	virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override
diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index 9f1bcb41a..ac8b70c1c 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -1189,7 +1189,7 @@ public:
 				break;
 			}
 		}
-		FilterReferences(Ctx, m_References);
+		FilterReferences(Ctx, fmt::format("cachestore [LOCKSTATE] '{}'", "cachestore"), m_References);
 	}
 
 	virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index e0a7900f1..6af07a606 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -967,7 +967,7 @@ CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats&)
 			CidsToCheck.push_back(It.first);
 		}
 	}
-	if (FilterReferences(Ctx, CidsToCheck))
+	if (FilterReferences(Ctx, fmt::format("compactcas [CREATE PRUNER] '{}'", m_RootDirectory / m_ContainerBaseName), CidsToCheck))
 	{
 		return new CasContainerReferencePruner(*this, std::move(CidsToCheck));
 	}
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index 7bd17ee88..71eead9b2 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -1745,7 +1745,7 @@ FileCasStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats&)
 			CidsToCheck.push_back(It.first);
 		}
 	}
-	if (FilterReferences(Ctx, CidsToCheck))
+	if (FilterReferences(Ctx, fmt::format("filecas [CREATE PRUNER] '{}'", m_RootDirectory), CidsToCheck))
 	{
 		return new FileCasReferencePruner(*this, std::move(CidsToCheck));
 	}
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 904619222..cde89421e 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -145,6 +145,28 @@ namespace {
 		return std::error_code{};
 	}
 
+	uint8_t ComputeAttachmentRange(uint16_t AttachmentPassIndex, uint16_t AttachmentPassCount, IoHash& OutMin, IoHash& OutMax)
+	{
+		if (AttachmentPassCount <= 1)
+		{
+			OutMin = IoHash::Zero;
+			OutMax = IoHash::Max;
+			return 0;
+		}
+		if (AttachmentPassIndex >= AttachmentPassCount)
+		{
+			AttachmentPassIndex = 0;
+		}
+
+		uint32_t RangeBegin = (256 * AttachmentPassIndex) / AttachmentPassCount;
+		AttachmentPassIndex++;
+		uint32_t RangeEnd = ((256 * AttachmentPassIndex) / AttachmentPassCount) - 1;
+		OutMin			  = IoHash::Zero;
+		OutMin.Hash[0]	  = gsl::narrow<uint8_t>(RangeBegin);
+		OutMax			  = IoHash::Max;
+		OutMax.Hash[0]	  = gsl::narrow<uint8_t>(RangeEnd);
+		return gsl::narrow<uint8_t>((AttachmentPassIndex == AttachmentPassCount) ? 0 : AttachmentPassIndex);
+	};
 }  // namespace
 
 //////////////////////////////////////////////////////////////////////////
@@ -572,25 +594,38 @@ Sum(GcResult& Stat, bool Cancelled = false)
 }
 
 bool
-FilterReferences(GcCtx& Ctx, std::vector<IoHash>& InOutReferences)
+FilterReferences(GcCtx& Ctx, std::string_view Context, std::vector<IoHash>& InOutReferences)
 {
 	if (InOutReferences.empty())
 	{
 		return false;
 	}
-	if (Ctx.Settings.AttachmentRangeMax != IoHash::Max || Ctx.Settings.AttachmentRangeMin != IoHash::Zero)
+
+	const bool Filter = Ctx.Settings.AttachmentRangeMax != IoHash::Max || Ctx.Settings.AttachmentRangeMin != IoHash::Zero;
+
+	size_t TotalCount	= InOutReferences.size();
+	size_t RemovedCount = 0;
+
+	Stopwatch  Timer;
+	const auto _ = MakeGuard([&] {
+		if (!Ctx.Settings.Verbose)
+		{
+			return;
+		}
+		ZEN_INFO(
+			"GCV2: {}: {}sorted {} entries in {}",
+			Context,
+			Filter ? fmt::format("skipped {}% ({} out of {}) and ", (100 * RemovedCount) / TotalCount, RemovedCount, TotalCount) : ""sv,
+			TotalCount - RemovedCount,
+			NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+	});
+
+	if (Filter)
 	{
-		size_t TotalCount = InOutReferences.size();
 		std::erase_if(InOutReferences, [&Ctx](const IoHash& Key) {
 			return ((Ctx.Settings.AttachmentRangeMax < Key) || (Key < Ctx.Settings.AttachmentRangeMin));
 		});
-		size_t RemovedCount = TotalCount - InOutReferences.size();
-		ZEN_INFO("Skipped GC for {}% of references ({} out of {}) due to attachment filtering with range {} to {}",
-				 (100 * RemovedCount) / TotalCount,
-				 RemovedCount,
-				 TotalCount,
-				 Ctx.Settings.AttachmentRangeMin,
-				 Ctx.Settings.AttachmentRangeMax);
+		RemovedCount = TotalCount - InOutReferences.size();
 	}
 	if (InOutReferences.empty())
 	{
@@ -1502,6 +1537,7 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
 	m_LastGcTime			= GcClock::Now();
 	m_LastLightweightGcTime = m_LastGcTime;
 	m_LastGcExpireTime		= GcClock::TimePoint::min();
+	m_AttachmentPassIndex	= 0;
 
 	if (CbObject SchedulerState = LoadCompactBinaryObject(Config.RootDirectory / "gc_state"))
 	{
@@ -1514,6 +1550,7 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
 			m_LastGcTime			= GcClock::Now();
 			m_LastLightweightGcTime = m_LastGcTime;
 		}
+		m_AttachmentPassIndex = SchedulerState["AttachmentPassIndex"sv].AsUInt8();
 	}
 
 	m_DiskUsageLog.Open(m_Config.RootDirectory / "gc.dlog", CasLogFile::Mode::kWrite);
@@ -1646,14 +1683,13 @@ GcScheduler::CheckDiskSpace()
 }
 
 void
-GcScheduler::AppendGCLog(GcClock::TimePoint StartTime, const GcSettings& Settings, const GcResult& Result)
+GcScheduler::AppendGCLog(std::string_view Id, GcClock::TimePoint StartTime, const GcSettings& Settings, const GcResult& Result)
 {
 	try
 	{
 		std::vector<uint8_t> Blob;
 		{
 			CbObjectWriter Writer;
-			std::string	   Id = fmt::format("{}", gsl::narrow<int64_t>(StartTime.time_since_epoch().count()));
 			Writer.BeginObject(Id);
 			{
 				Writer << "StartTime"sv << ToDateTime(StartTime);
@@ -1667,6 +1703,8 @@ GcScheduler::AppendGCLog(GcClock::TimePoint StartTime, const GcSettings& Setting
 					Writer << "Verbose"sv << Settings.Verbose;
 					Writer << "SingleThread"sv << Settings.SingleThread;
 					Writer << "CompactBlockUsageThresholdPercent"sv << Settings.CompactBlockUsageThresholdPercent;
+					Writer << "AttachmentRangeMin"sv << Settings.AttachmentRangeMin;
+					Writer << "AttachmentRangeMax"sv << Settings.AttachmentRangeMin;
 				}
 				Writer.EndObject();
 
@@ -1781,6 +1819,9 @@ GcScheduler::GetState() const
 
 			Result.LastLightweightGCV2Result = m_LastLightweightGCV2Result;
 			Result.LastFullGCV2Result		 = m_LastFullGCV2Result;
+
+			Result.LastFullAttachmentRangeMin = m_LastFullAttachmentRangeMin;
+			Result.LastFullAttachmentRangeMax = m_LastFullAttachmentRangeMax;
 		}
 
 		Result.RemainingTimeUntilFullGc =
@@ -1860,6 +1901,8 @@ GcScheduler::SchedulerThread()
 			bool				 SingleThreaded					   = m_Config.SingleThreaded;
 			IoHash				 AttachmentRangeMin				   = IoHash::Zero;
 			IoHash				 AttachmentRangeMax				   = IoHash::Max;
+			uint8_t				 NextAttachmentPassIndex =
+				ComputeAttachmentRange(m_AttachmentPassIndex, m_Config.AttachmentPassCount, AttachmentRangeMin, AttachmentRangeMax);
 
 			bool DiskSpaceGCTriggered = false;
 			bool TimeBasedGCTriggered = false;
@@ -1898,9 +1941,13 @@ GcScheduler::SchedulerThread()
 					TriggerParams.CompactBlockUsageThresholdPercent.value_or(CompactBlockUsageThresholdPercent);
 				Verbose			   = TriggerParams.Verbose.value_or(Verbose);
 				SingleThreaded	   = TriggerParams.SingleThreaded.value_or(SingleThreaded);
-				AttachmentRangeMin = TriggerParams.AttachmentRangeMin;
-				AttachmentRangeMax = TriggerParams.AttachmentRangeMax;
-				DoGc			   = true;
+				AttachmentRangeMin = TriggerParams.AttachmentRangeMin.value_or(AttachmentRangeMin);
+				AttachmentRangeMax = TriggerParams.AttachmentRangeMax.value_or(AttachmentRangeMax);
+				if (TriggerParams.AttachmentRangeMin.has_value() || TriggerParams.AttachmentRangeMax.has_value())
+				{
+					NextAttachmentPassIndex = m_AttachmentPassIndex;
+				}
+				DoGc = true;
 			}
 
 			if (m_TriggerScrubParams)
@@ -2110,6 +2157,11 @@ GcScheduler::SchedulerThread()
 				}
 			}
 
+			if (!SkipCid)
+			{
+				m_AttachmentPassIndex = NextAttachmentPassIndex;
+			}
+
 			bool GcSuccess = CollectGarbage(CacheExpireTime,
 											ProjectStoreExpireTime,
 											DoDelete,
@@ -2266,11 +2318,6 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 			}
 		}
 
-		ZEN_INFO("garbage collection STARTING, small objects gc {}, {} CAS. Cache cutoff time {}, project store cutoff time {}",
-				 GcCtx.CollectSmallObjects() ? "ENABLED"sv : "DISABLED"sv,
-				 SkipCid ? "skip"sv : "include"sv,
-				 CacheExpireTime,
-				 ProjectStoreExpireTime);
 		{
 			Stopwatch  Timer;
 			const auto __ = MakeGuard([&] { ZEN_INFO("garbage collection DONE in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
@@ -2279,6 +2326,13 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 			switch (UseGCVersion)
 			{
 				case GcVersion::kV1:
+					ZEN_INFO(
+						"GCV1: Garbage collection STARTING, small objects gc {}, {} CAS. Cache cutoff time {}, project store cutoff time "
+						"{}",
+						GcCtx.CollectSmallObjects() ? "ENABLED"sv : "DISABLED"sv,
+						SkipCid ? "skip"sv : "include"sv,
+						CacheExpireTime,
+						ProjectStoreExpireTime);
 					Diff = m_GcManager.CollectGarbage(GcCtx);
 					if (SkipCid)
 					{
@@ -2291,6 +2345,8 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 					break;
 				case GcVersion::kV2:
 					{
+						std::string GcId = Oid::NewOid().ToString();
+
 						const GcSettings Settings = {.CacheExpireTime					= CacheExpireTime,
 													 .ProjectStoreExpireTime			= ProjectStoreExpireTime,
 													 .CollectSmallObjects				= CollectSmallObjects,
@@ -2303,44 +2359,60 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 													 .AttachmentRangeMin				= AttachmentRangeMin,
 													 .AttachmentRangeMax				= AttachmentRangeMax};
 
+						auto AppendSettings = [](StringBuilderBase& SB, const GcSettings& Settings) {
+							SB.Append(
+								fmt::format("      GC small objects:          {}\n", Settings.CollectSmallObjects ? "yes"sv : "no"sv));
+							SB.Append(fmt::format("      GC Cid store:              {}\n", Settings.SkipCidDelete ? "no"sv : "yes"sv));
+							if (!Settings.SkipCidDelete &&
+								(Settings.AttachmentRangeMin != IoHash::Zero || Settings.AttachmentRangeMax != IoHash::Max))
+							{
+								SB.Append(fmt::format("      Attachment range:          {}-{}\n",
+													  Settings.AttachmentRangeMin,
+													  Settings.AttachmentRangeMax));
+							}
+							SB.Append(fmt::format("      Cache cutoff time:         {}\n", Settings.CacheExpireTime));
+							SB.Append(fmt::format("      Project store cutoff time: {}", Settings.ProjectStoreExpireTime));
+						};
+
+						{
+							ExtendableStringBuilder<256> SB;
+							SB.Append(fmt::format("STARTING '{}'\n", GcId));
+							AppendSettings(SB, Settings);
+							ZEN_INFO("GCV2: {}", SB.ToView());
+						}
+
 						GcClock::TimePoint GcStartTime = GcClock::Now();
 						GcResult		   Result	   = m_GcManager.CollectGarbage(Settings);
 
 						ExtendableStringBuilder<256> SB;
 						if (Result.WasCancelled)
 						{
-							SB.Append(fmt::format("Cancelled after {}", NiceTimeSpanMs(Result.ElapsedMS.count())));
+							SB.Append(fmt::format("CANCELLED '{}' after {}", GcId, NiceTimeSpanMs(Result.ElapsedMS.count())));
 						}
 						else
 						{
-							SB.Append(
-								fmt::format("CacheExpireTime: {}, ProjectStoreExpireTime: {}, CollectSmallObjects: {}, IsDeleteMode: {}, "
-											"SkipCidDelete: {}\n",
-											Settings.CacheExpireTime,
-											Settings.ProjectStoreExpireTime,
-											Settings.CollectSmallObjects,
-											Settings.IsDeleteMode,
-											Settings.SkipCidDelete));
-							SB.Append(fmt::format("      Found {} expired items out of {}, deleted {}.\n",
+							SB.Append(fmt::format("COMPLETED '{}' in {}\n", GcId, NiceTimeSpanMs(Result.ElapsedMS.count())));
+							AppendSettings(SB, Settings);
+							SB.Append("\n\n");
+							SB.Append(fmt::format("      Found {} expired items out of {}, deleted {}\n",
 												  Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount,
 												  Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount,
 												  Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount));
 							if (!Settings.SkipCidDelete)
 							{
-								SB.Append(fmt::format("      Found {} unreferenced Cid entries out of {}, deleted {}.\n",
+								SB.Append(fmt::format("      Found {} unreferenced Cid entries out of {}, deleted {}\n",
 													  Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount,
 													  Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount,
 													  Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount));
 							}
-							SB.Append(fmt::format("      Freed {} on disk and {} of memory in {}.\n",
+							SB.Append(fmt::format("      Freed {} on disk and {} of memory\n",
 												  NiceBytes(Result.CompactStoresStatSum.RemovedDisk),
-												  NiceBytes(Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory),
-												  NiceTimeSpanMs(Result.ElapsedMS.count())));
+												  NiceBytes(Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory)));
 						}
 
 						ZEN_INFO("GCV2: {}", SB.ToView());
 
-						AppendGCLog(GcStartTime, Settings, Result);
+						AppendGCLog(GcId, GcStartTime, Settings, Result);
 
 						if (SkipCid)
 						{
@@ -2348,7 +2420,9 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 						}
 						else
 						{
-							m_LastFullGCV2Result = Result;
+							m_LastFullGCV2Result		 = Result;
+							m_LastFullAttachmentRangeMin = AttachmentRangeMin;
+							m_LastFullAttachmentRangeMin = AttachmentRangeMax;
 						}
 						Diff.DiskSize	= Result.CompactStoresStatSum.RemovedDisk;
 						Diff.MemorySize = Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory;
@@ -2393,6 +2467,8 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 					CbObjectWriter SchedulerState;
 					SchedulerState << "LastGcTime"sv << static_cast<int64_t>(m_LastGcTime.time_since_epoch().count());
 					SchedulerState << "LastGcExpireTime"sv << static_cast<int64_t>(m_LastGcExpireTime.time_since_epoch().count());
+					SchedulerState << "AttachmentPassIndex"sv << m_AttachmentPassIndex;
+
 					SaveCompactBinaryObject(Path, SchedulerState.Save());
 					if (RetryCount > 0)
 					{
@@ -3041,6 +3117,44 @@ TEST_CASE("gc.keepunusedreferences")
 	}
 }
 
+TEST_CASE("gc.attachmentrange")
+{
+	IoHash AttachmentRangeMin = IoHash::Zero;
+	IoHash AttachmentRangeMax = IoHash::Zero;
+
+	CHECK(ComputeAttachmentRange(0, 0, AttachmentRangeMin, AttachmentRangeMax) == 0);
+	CHECK(AttachmentRangeMin == IoHash::Zero);
+	CHECK(AttachmentRangeMax == IoHash::Max);
+
+	CHECK(ComputeAttachmentRange(1, 0, AttachmentRangeMin, AttachmentRangeMax) == 0);
+	CHECK(AttachmentRangeMin == IoHash::Zero);
+	CHECK(AttachmentRangeMax == IoHash::Max);
+
+	CHECK(ComputeAttachmentRange(1, 1, AttachmentRangeMin, AttachmentRangeMax) == 0);
+	CHECK(AttachmentRangeMin == IoHash::Zero);
+	CHECK(AttachmentRangeMax == IoHash::Max);
+
+	CHECK(ComputeAttachmentRange(0, 1, AttachmentRangeMin, AttachmentRangeMax) == 0);
+	CHECK(AttachmentRangeMin == IoHash::Zero);
+	CHECK(AttachmentRangeMax == IoHash::Max);
+
+	CHECK(ComputeAttachmentRange(0, 2, AttachmentRangeMin, AttachmentRangeMax) == 1);
+	CHECK(AttachmentRangeMin == IoHash::Zero);
+	CHECK(AttachmentRangeMax == IoHash::FromHexString("7fffffffffffffffffffffffffffffffffffffff"));
+
+	CHECK(ComputeAttachmentRange(1, 2, AttachmentRangeMin, AttachmentRangeMax) == 0);
+	CHECK(AttachmentRangeMin == IoHash::FromHexString("8000000000000000000000000000000000000000"));
+	CHECK(AttachmentRangeMax == IoHash::Max);
+
+	CHECK(ComputeAttachmentRange(1, 256, AttachmentRangeMin, AttachmentRangeMax) == 2);
+	CHECK(AttachmentRangeMin == IoHash::FromHexString("0100000000000000000000000000000000000000"));
+	CHECK(AttachmentRangeMax == IoHash::FromHexString("01ffffffffffffffffffffffffffffffffffffff"));
+
+	CHECK(ComputeAttachmentRange(255, 256, AttachmentRangeMin, AttachmentRangeMax) == 0);
+	CHECK(AttachmentRangeMin == IoHash::FromHexString("ff00000000000000000000000000000000000000"));
+	CHECK(AttachmentRangeMax == IoHash::Max);
+}
+
 #endif
 
 void
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index 3f2f5448d..56965e3e3 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -201,7 +201,7 @@ public:
 };
 
 std::span<IoHash> KeepUnusedReferences(std::span<const IoHash> SortedUsedReferences, std::span<IoHash> SortedReferences);
-bool			  FilterReferences(GcCtx& Ctx, std::vector<IoHash>& InOutReferences);
+bool			  FilterReferences(GcCtx& Ctx, std::string_view Context, std::vector<IoHash>& InOutReferences);
 
 /**
  * @brief An interface to implement a lock for Stop The World (from writing new data)
@@ -451,6 +451,7 @@ struct GcSchedulerConfig
 	uint32_t			  CompactBlockUsageThresholdPercent = 90;
 	bool				  Verbose							= false;
 	bool				  SingleThreaded					= false;
+	uint16_t			  AttachmentPassCount				= 1;
 };
 
 struct GcSchedulerState
@@ -477,6 +478,9 @@ struct GcSchedulerState
 
 	std::optional<GcResult> LastLightweightGCV2Result;
 	std::optional<GcResult> LastFullGCV2Result;
+
+	IoHash LastFullAttachmentRangeMin = IoHash::Zero;
+	IoHash LastFullAttachmentRangeMax = IoHash::Max;
 };
 
 class DiskUsageWindow
@@ -525,8 +529,8 @@ public:
 		std::optional<uint32_t>	 CompactBlockUsageThresholdPercent;
 		std::optional<bool>		 Verbose;
 		std::optional<bool>		 SingleThreaded;
-		IoHash					 AttachmentRangeMin = IoHash::Zero;
-		IoHash					 AttachmentRangeMax = IoHash::Max;
+		std::optional<IoHash>	 AttachmentRangeMin;
+		std::optional<IoHash>	 AttachmentRangeMax;
 	};
 
 	bool TriggerGc(const TriggerGcParams& Params);
@@ -561,7 +565,7 @@ private:
 	LoggerRef	 Log() { return m_Log; }
 	virtual bool AreDiskWritesAllowed() const override { return !m_AreDiskWritesBlocked.load(); }
 	DiskSpace	 CheckDiskSpace();
-	void		 AppendGCLog(GcClock::TimePoint GcStartTime, const GcSettings& Settings, const GcResult& Result);
+	void		 AppendGCLog(std::string_view Id, GcClock::TimePoint GcStartTime, const GcSettings& Settings, const GcResult& Result);
 
 	LoggerRef		   m_Log;
 	GcManager&		   m_GcManager;
@@ -569,6 +573,9 @@ private:
 	GcClock::TimePoint m_LastGcTime{};
 	GcClock::TimePoint m_LastLightweightGcTime{};
 	GcClock::TimePoint m_LastGcExpireTime{};
+	IoHash			   m_LastFullAttachmentRangeMin = IoHash::Zero;
+	IoHash			   m_LastFullAttachmentRangeMax = IoHash::Max;
+	uint8_t			   m_AttachmentPassIndex;
 
 	std::chrono::milliseconds m_LastFullGcDuration{};
 	GcStorageSize			  m_LastFullGCDiff;