add separate PreCache step for GcReferenceChecker (#578)

- Improvement: GCv2: Use separate PreCache step to improve concurrency when checking references
- Improvement: GCv2: Improved verbose logging
- Improvement: GCv2: Sort chunks to read by block/offset when finding references
- Improvement: GCv2: Exit as soon as no more unreferenced items are left

1 files changed, 54 insertions, 6 deletions

diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index e2ab34d1e..2660c2643 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -381,6 +381,7 @@ WriteReferencerStats(CbObjectWriter& Writer, const GcReferencerStats& Stats, boo
 	Writer.EndObject();
 
 	Writer << "CreateReferenceCheckers" << ToTimeSpan(Stats.CreateReferenceCheckersMS);
+	Writer << "PreCacheState" << ToTimeSpan(Stats.PreCacheStateMS);
 	Writer << "LockState" << ToTimeSpan(Stats.LockStateMS);
 	Writer << "Elapsed" << ToTimeSpan(Stats.ElapsedMS);
 };
@@ -449,6 +450,7 @@ WriteGCResult(CbObjectWriter& Writer, const GcResult& Result, bool HumanReadable
 
 	Writer << "RemoveExpiredData" << ToTimeSpan(Result.RemoveExpiredDataMS);
 	Writer << "CreateReferenceCheckers" << ToTimeSpan(Result.CreateReferenceCheckersMS);
+	Writer << "PreCacheState" << ToTimeSpan(Result.PreCacheStateMS);
 	Writer << "LockState" << ToTimeSpan(Result.LockStateMS);
 
 	Writer << "CreateReferencePruners" << ToTimeSpan(Result.CreateReferencePrunersMS);
@@ -507,8 +509,8 @@ Add(GcStats& Sum, const GcStats& Sub)
 void
 Sum(GcReferencerStats& Stat)
 {
-	Stat.ElapsedMS =
-		Stat.RemoveExpiredDataStats.ElapsedMS + Stat.CompactStoreStats.ElapsedMS + Stat.CreateReferenceCheckersMS + Stat.LockStateMS;
+	Stat.ElapsedMS = Stat.RemoveExpiredDataStats.ElapsedMS + Stat.CompactStoreStats.ElapsedMS + Stat.CreateReferenceCheckersMS +
+					 Stat.PreCacheStateMS + Stat.LockStateMS;
 }
 
 void
@@ -518,6 +520,7 @@ Add(GcReferencerStats& Sum, const GcReferencerStats& Sub)
 	Add(Sum.CompactStoreStats, Sub.CompactStoreStats);
 
 	Sum.CreateReferenceCheckersMS += Sub.CreateReferenceCheckersMS;
+	Sum.PreCacheStateMS += Sub.PreCacheStateMS;
 	Sum.LockStateMS += Sub.LockStateMS;
 
 	Sum.ElapsedMS += Sub.ElapsedMS;
@@ -802,10 +805,54 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 					}
 				}
 
-				ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size());
 				{
-					SCOPED_TIMER(uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); Result.WriteBlockMS = std::chrono::milliseconds(ElapsedMS);
-								 ZEN_INFO("GCV2: Writes blocked for {}", NiceTimeSpanMs(ElapsedMS)));
+					ZEN_INFO("GCV2: Precaching state for {} reference checkers", ReferenceCheckers.size());
+					if (!ReferenceCheckers.empty())
+					{
+						if (CheckGCCancel())
+						{
+							return Sum(Result, true);
+						}
+						ZEN_TRACE_CPU("GcV2::PreCache");
+
+						Latch WorkLeft(1);
+
+						{
+							SCOPED_TIMER(Result.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());
+										 if (Ctx.Settings.Verbose) {
+											 ZEN_INFO("GCV2: Precached state using {} reference checkers in {}",
+													  ReferenceCheckers.size(),
+													  NiceTimeSpanMs(Result.PreCacheStateMS.count()));
+										 });
+							for (auto& It : ReferenceCheckers)
+							{
+								if (CheckGCCancel())
+								{
+									WorkLeft.CountDown();
+									WorkLeft.Wait();
+									return Sum(Result, true);
+								}
+
+								GcReferenceChecker*						   Checker = It.first.get();
+								size_t									   Index   = It.second;
+								std::pair<std::string, GcReferencerStats>& Stats   = Result.ReferencerStats[Index];
+								WorkLeft.AddCount(1);
+								ThreadPool.ScheduleWork([&Ctx, Checker, Index, &Stats, &WorkLeft]() {
+									auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+									SCOPED_TIMER(Stats.second.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+									Checker->PreCache(Ctx);
+								});
+							}
+							WorkLeft.CountDown();
+							WorkLeft.Wait();
+						}
+					}
+				}
+
+				SCOPED_TIMER(uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); Result.WriteBlockMS = std::chrono::milliseconds(ElapsedMS);
+							 ZEN_INFO("GCV2: Writes blocked for {}", NiceTimeSpanMs(ElapsedMS)));
+				{
+					ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size());
 					if (!ReferenceCheckers.empty())
 					{
 						if (CheckGCCancel())
@@ -849,7 +896,8 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 							WorkLeft.Wait();
 						}
 					}
-
+				}
+				{
 					ZEN_INFO("GCV2: Removing unreferenced data for {} reference pruners", ReferencePruners.size());
 					{
 						const auto GetUnusedReferences = [&ReferenceCheckers, &Ctx](std::span<IoHash> References) -> std::vector<IoHash> {