Merge branch 'main' of https://github.com/EpicGames/zen

1 files changed, 691 insertions, 433 deletions

diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 778a47626..de653b0e3 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -18,6 +18,7 @@
 #include <zencore/workthreadpool.h>
 #include <zenstore/cidstore.h>
 #include <zenstore/scrubcontext.h>
+#include <zenutil/workerpools.h>
 
 #include "cas.h"
 
@@ -173,166 +174,6 @@ SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object)
 
 //////////////////////////////////////////////////////////////////////////
 
-void
-WriteReferencerStats(CbObjectWriter& Writer, const GcReferencerStats& Stats, bool HumanReadable)
-{
-	if (Stats.Count == 0)
-	{
-		return;
-	}
-	Writer << "Count" << Stats.Count;
-	Writer << "Expired" << Stats.Expired;
-	Writer << "Deleted" << Stats.Deleted;
-
-	if (HumanReadable)
-	{
-		Writer << "RemovedDisk" << NiceBytes(Stats.RemovedDisk);
-		Writer << "RemovedMemory" << NiceBytes(Stats.RemovedMemory);
-	}
-	else
-	{
-		Writer << "RemovedDiskBytes" << Stats.RemovedDisk;
-		Writer << "RemovedMemoryBytes" << Stats.RemovedMemory;
-	}
-
-	if (HumanReadable)
-	{
-		Writer << "RemoveExpiredData" << NiceTimeSpanMs(Stats.RemoveExpiredDataMS.count());
-		Writer << "CreateReferenceCheckers" << NiceTimeSpanMs(Stats.CreateReferenceCheckersMS.count());
-		Writer << "LockState" << NiceTimeSpanMs(Stats.LockStateMS.count());
-		Writer << "Elapsed" << NiceTimeSpanMs(Stats.ElapsedMS.count());
-	}
-	else
-	{
-		Writer << "RemoveExpiredDataMS" << gsl::narrow<int64_t>(Stats.RemoveExpiredDataMS.count());
-		Writer << "CreateReferenceCheckersMS" << gsl::narrow<int64_t>(Stats.CreateReferenceCheckersMS.count());
-		Writer << "LockStateMS" << gsl::narrow<int64_t>(Stats.LockStateMS.count());
-		Writer << "ElapsedMS" << gsl::narrow<int64_t>(Stats.ElapsedMS.count());
-	}
-};
-
-void
-WriteReferenceStoreStats(CbObjectWriter& Writer, const GcReferenceStoreStats& Stats, bool HumanReadable)
-{
-	if (Stats.Count == 0)
-	{
-		return;
-	}
-	Writer << "Count" << Stats.Count;
-	Writer << "Pruned" << Stats.Pruned;
-	Writer << "Compacted" << Stats.Compacted;
-
-	if (HumanReadable)
-	{
-		Writer << "RemovedDisk" << NiceBytes(Stats.RemovedDisk);
-		Writer << "RemovedMemory" << NiceBytes(Stats.RemovedMemory);
-	}
-	else
-	{
-		Writer << "RemovedDiskBytes" << Stats.RemovedDisk;
-		Writer << "RemovedMemoryBytes" << Stats.RemovedMemory;
-	}
-
-	if (HumanReadable)
-	{
-		Writer << "CreateReferencePruner" << NiceTimeSpanMs(Stats.CreateReferencePrunerMS.count());
-		Writer << "RemoveUnreferencedData" << NiceTimeSpanMs(Stats.RemoveUnreferencedDataMS.count());
-		Writer << "CompactReferenceStore" << NiceTimeSpanMs(Stats.CompactReferenceStoreMS.count());
-		Writer << "Elapsed" << NiceTimeSpanMs(Stats.ElapsedMS.count());
-	}
-	else
-	{
-		Writer << "CreateReferencePrunerMS" << gsl::narrow<int64_t>(Stats.CreateReferencePrunerMS.count());
-		Writer << "RemoveUnreferencedDataMS" << gsl::narrow<int64_t>(Stats.RemoveUnreferencedDataMS.count());
-		Writer << "CompactReferenceStoreMS" << gsl::narrow<int64_t>(Stats.CompactReferenceStoreMS.count());
-		Writer << "ElapsedMS" << gsl::narrow<int64_t>(Stats.ElapsedMS.count());
-	}
-};
-
-void
-WriteGCResult(CbObjectWriter& Writer, const GcResult& Result, bool HumanReadable, bool IncludeDetails)
-{
-	if (HumanReadable)
-	{
-		Writer << "RemovedDisk" << NiceBytes(Result.RemovedDisk);
-		Writer << "RemovedMemory" << NiceBytes(Result.RemovedMemory);
-		Writer << "WriteBlock" << NiceTimeSpanMs(Result.WriteBlockMS.count());
-		Writer << "Elapsed" << NiceTimeSpanMs(Result.ElapsedMS.count());
-	}
-	else
-	{
-		Writer << "RemovedDiskBytes" << gsl::narrow<int64_t>(Result.RemovedDisk);
-		Writer << "RemovedMemoryBytes" << gsl::narrow<int64_t>(Result.RemovedMemory);
-		Writer << "WriteBlockMS" << gsl::narrow<int64_t>(Result.WriteBlockMS.count());
-		Writer << "ElapsedMS" << gsl::narrow<int64_t>(Result.ElapsedMS.count());
-	}
-
-	if (!IncludeDetails)
-	{
-		return;
-	}
-
-	if (HumanReadable)
-	{
-		Writer << "RemoveExpiredData" << NiceTimeSpanMs(Result.RemoveExpiredDataMS.count());
-		Writer << "CreateReferenceCheckers" << NiceTimeSpanMs(Result.CreateReferenceCheckersMS.count());
-		Writer << "LockState" << NiceTimeSpanMs(Result.LockStateMS.count());
-
-		Writer << "CreateReferencePruner" << NiceTimeSpanMs(Result.CreateReferencePrunerMS.count());
-		Writer << "RemoveUnreferencedData" << NiceTimeSpanMs(Result.RemoveUnreferencedDataMS.count());
-		Writer << "CompactReferenceStore" << NiceTimeSpanMs(Result.CompactReferenceStoreMS.count());
-	}
-	else
-	{
-		Writer << "RemoveExpiredDataMS" << gsl::narrow<int64_t>(Result.RemoveExpiredDataMS.count());
-		Writer << "CreateReferenceCheckersMS" << gsl::narrow<int64_t>(Result.CreateReferenceCheckersMS.count());
-		Writer << "LockStateMS" << gsl::narrow<int64_t>(Result.LockStateMS.count());
-
-		Writer << "CreateReferencePrunerMS" << gsl::narrow<int64_t>(Result.CreateReferencePrunerMS.count());
-		Writer << "RemoveUnreferencedDataMS" << gsl::narrow<int64_t>(Result.RemoveUnreferencedDataMS.count());
-		Writer << "CompactReferenceStoreMS" << gsl::narrow<int64_t>(Result.CompactReferenceStoreMS.count());
-	}
-
-	Writer.BeginObject("ReferencerStats");
-	{
-		WriteReferencerStats(Writer, Result.ReferencerStat, HumanReadable);
-	}
-	Writer.EndObject();
-
-	Writer.BeginObject("ReferenceStoreStats");
-	{
-		WriteReferenceStoreStats(Writer, Result.ReferenceStoreStat, HumanReadable);
-	}
-	Writer.EndObject();
-
-	if (!Result.ReferencerStats.empty())
-	{
-		Writer.BeginArray("Referencers");
-		{
-			for (const std::pair<std::string, GcReferencerStats>& It : Result.ReferencerStats)
-			{
-				Writer.BeginObject();
-				Writer << "Name" << It.first;
-				WriteReferencerStats(Writer, It.second, HumanReadable);
-				Writer.EndObject();
-			}
-		}
-		Writer.EndArray();
-	}
-	if (!Result.ReferenceStoreStats.empty())
-	{
-		Writer.BeginArray("ReferenceStores");
-		for (const std::pair<std::string, GcReferenceStoreStats>& It : Result.ReferenceStoreStats)
-		{
-			Writer.BeginObject();
-			Writer << "Name" << It.first;
-			WriteReferenceStoreStats(Writer, It.second, HumanReadable);
-			Writer.EndObject();
-		}
-		Writer.EndArray();
-	}
-};
-
 struct GcContext::GcState
 {
 	using CacheKeyContexts = std::unordered_map<std::string, std::vector<IoHash>>;
@@ -490,44 +331,243 @@ GcManager::~GcManager()
 //////// Begin GC V2
 
 void
-GcResult::Sum()
+WriteGcStats(CbObjectWriter& Writer, const GcStats& Stats, bool HumanReadable)
 {
-	for (std::pair<std::string, GcReferencerStats>& Referencer : ReferencerStats)
+	Writer << "Checked" << Stats.CheckedCount;
+	Writer << "Found" << Stats.FoundCount;
+	Writer << "Deleted" << Stats.DeletedCount;
+	if (HumanReadable)
 	{
-		GcReferencerStats& SubStat = Referencer.second;
-		ReferencerStat.Count += SubStat.Count;
-		ReferencerStat.Expired += SubStat.Expired;
-		ReferencerStat.Deleted += SubStat.Deleted;
-		ReferencerStat.RemovedDisk += SubStat.RemovedDisk;
-		ReferencerStat.RemovedMemory += SubStat.RemovedMemory;
-		SubStat.ElapsedMS = SubStat.RemoveExpiredDataMS + SubStat.CreateReferenceCheckersMS + SubStat.LockStateMS;
+		Writer << "FreedMemory" << NiceBytes(Stats.FreedMemory);
+	}
+	else
+	{
+		Writer << "FreedMemoryBytes" << Stats.FreedMemory;
+	}
+	Writer << "Elapsed" << ToTimeSpan(Stats.ElapsedMS);
+}
 
-		ReferencerStat.RemoveExpiredDataMS += SubStat.RemoveExpiredDataMS;
-		ReferencerStat.CreateReferenceCheckersMS += SubStat.CreateReferenceCheckersMS;
-		ReferencerStat.LockStateMS += SubStat.LockStateMS;
-		ReferencerStat.ElapsedMS += SubStat.ElapsedMS;
+void
+WriteCompactStoreStats(CbObjectWriter& Writer, const GcCompactStoreStats& Stats, bool HumanReadable)
+{
+	if (HumanReadable)
+	{
+		Writer << "RemovedDisk" << NiceBytes(Stats.RemovedDisk);
+	}
+	else
+	{
+		Writer << "RemovedDiskBytes" << Stats.RemovedDisk;
+	}
+	Writer << "Elapsed" << ToTimeSpan(Stats.ElapsedMS);
+}
 
-		RemovedDisk += SubStat.RemovedDisk;
-		RemovedMemory += SubStat.RemovedMemory;
+void
+WriteReferencerStats(CbObjectWriter& Writer, const GcReferencerStats& Stats, bool HumanReadable)
+{
+	if (Stats.RemoveExpiredDataStats.CheckedCount == 0)
+	{
+		return;
 	}
-	for (std::pair<std::string, GcReferenceStoreStats>& ReferenceStore : ReferenceStoreStats)
+	Writer.BeginObject("RemoveExpired");
 	{
-		GcReferenceStoreStats& SubStat = ReferenceStore.second;
-		ReferenceStoreStat.Count += SubStat.Count;
-		ReferenceStoreStat.Pruned += SubStat.Pruned;
-		ReferenceStoreStat.Compacted += SubStat.Compacted;
-		ReferenceStoreStat.RemovedDisk += SubStat.RemovedDisk;
-		ReferenceStoreStat.RemovedMemory += SubStat.RemovedMemory;
-		SubStat.ElapsedMS = SubStat.CreateReferencePrunerMS + SubStat.RemoveUnreferencedDataMS + SubStat.CompactReferenceStoreMS;
+		WriteGcStats(Writer, Stats.RemoveExpiredDataStats, HumanReadable);
+	}
+	Writer.EndObject();
+
+	Writer.BeginObject("Compact");
+	{
+		WriteCompactStoreStats(Writer, Stats.CompactStoreStats, HumanReadable);
+	}
+	Writer.EndObject();
+
+	Writer << "CreateReferenceCheckers" << ToTimeSpan(Stats.CreateReferenceCheckersMS);
+	Writer << "PreCacheState" << ToTimeSpan(Stats.PreCacheStateMS);
+	Writer << "LockState" << ToTimeSpan(Stats.LockStateMS);
+	Writer << "Elapsed" << ToTimeSpan(Stats.ElapsedMS);
+};
+
+void
+WriteReferenceStoreStats(CbObjectWriter& Writer, const GcReferenceStoreStats& Stats, bool HumanReadable)
+{
+	if (Stats.RemoveUnreferencedDataStats.CheckedCount == 0)
+	{
+		return;
+	}
+	Writer.BeginObject("RemoveUnreferenced");
+	{
+		WriteGcStats(Writer, Stats.RemoveUnreferencedDataStats, HumanReadable);
+	}
+	Writer.EndObject();
+
+	Writer.BeginObject("Compact");
+	{
+		WriteCompactStoreStats(Writer, Stats.CompactStoreStats, HumanReadable);
+	}
+	Writer.EndObject();
+
+	Writer << "CreateReferencePruners" << ToTimeSpan(Stats.CreateReferencePrunersMS);
+	Writer << "Elapsed" << ToTimeSpan(Stats.ElapsedMS);
+};
+
+void
+WriteGCResult(CbObjectWriter& Writer, const GcResult& Result, bool HumanReadable, bool IncludeDetails)
+{
+	if (!IncludeDetails)
+	{
+		if (HumanReadable)
+		{
+			Writer << "RemovedDisk" << NiceBytes(Result.CompactStoresStatSum.RemovedDisk);
+			Writer << "FreedMemory" << NiceBytes(Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
+		}
+		else
+		{
+			Writer << "RemovedDiskBytes" << gsl::narrow<int64_t>(Result.CompactStoresStatSum.RemovedDisk);
+			Writer << "RemovedMemoryBytes" << gsl::narrow<int64_t>(Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
+		}
+		Writer << "WriteBlock" << ToTimeSpan(Result.WriteBlockMS);
+		Writer << "Elapsed" << ToTimeSpan(Result.ElapsedMS);
+		Writer << "Cancelled" << Result.WasCancelled;
+		return;
+	}
+
+	Writer.BeginObject("Referencer");
+	{
+		WriteReferencerStats(Writer, Result.ReferencerStatSum, HumanReadable);
+	}
+	Writer.EndObject();
+
+	Writer.BeginObject("ReferenceStore");
+	{
+		WriteReferenceStoreStats(Writer, Result.ReferenceStoreStatSum, HumanReadable);
+	}
+	Writer.EndObject();
+
+	Writer.BeginObject("Compact");
+	{
+		WriteCompactStoreStats(Writer, Result.CompactStoresStatSum, HumanReadable);
+	}
+	Writer.EndObject();
+
+	Writer << "RemoveExpiredData" << ToTimeSpan(Result.RemoveExpiredDataMS);
+	Writer << "CreateReferenceCheckers" << ToTimeSpan(Result.CreateReferenceCheckersMS);
+	Writer << "PreCacheState" << ToTimeSpan(Result.PreCacheStateMS);
+	Writer << "LockState" << ToTimeSpan(Result.LockStateMS);
+
+	Writer << "CreateReferencePruners" << ToTimeSpan(Result.CreateReferencePrunersMS);
+	Writer << "RemoveUnreferencedData" << ToTimeSpan(Result.RemoveUnreferencedDataMS);
+	Writer << "CompactStores" << ToTimeSpan(Result.CompactStoresMS);
+	Writer << "WriteBlock" << ToTimeSpan(Result.WriteBlockMS);
+	Writer << "Elapsed" << ToTimeSpan(Result.ElapsedMS);
+
+	if (!Result.ReferencerStats.empty())
+	{
+		Writer.BeginArray("Referencers");
+		{
+			for (const std::pair<std::string, GcReferencerStats>& It : Result.ReferencerStats)
+			{
+				Writer.BeginObject();
+				Writer << "Name" << It.first;
+				WriteReferencerStats(Writer, It.second, HumanReadable);
+				Writer.EndObject();
+			}
+		}
+		Writer.EndArray();
+	}
+	if (!Result.ReferenceStoreStats.empty())
+	{
+		Writer.BeginArray("ReferenceStores");
+		for (const std::pair<std::string, GcReferenceStoreStats>& It : Result.ReferenceStoreStats)
+		{
+			Writer.BeginObject();
+			Writer << "Name" << It.first;
+			WriteReferenceStoreStats(Writer, It.second, HumanReadable);
+			Writer.EndObject();
+		}
+		Writer.EndArray();
+	}
+};
+
+void
+Add(GcCompactStoreStats& Sum, const GcCompactStoreStats& Sub)
+{
+	Sum.RemovedDisk += Sub.RemovedDisk;
+
+	Sum.ElapsedMS += Sub.ElapsedMS;
+}
 
-		ReferenceStoreStat.CreateReferencePrunerMS += SubStat.CreateReferencePrunerMS;
-		ReferenceStoreStat.RemoveUnreferencedDataMS += SubStat.RemoveUnreferencedDataMS;
-		ReferenceStoreStat.CompactReferenceStoreMS += SubStat.CompactReferenceStoreMS;
-		ReferenceStoreStat.ElapsedMS += SubStat.ElapsedMS;
+void
+Add(GcStats& Sum, const GcStats& Sub)
+{
+	Sum.CheckedCount += Sub.CheckedCount;
+	Sum.FoundCount += Sub.FoundCount;
+	Sum.DeletedCount += Sub.DeletedCount;
+	Sum.FreedMemory += Sub.FreedMemory;
 
-		RemovedDisk += SubStat.RemovedDisk;
-		RemovedMemory += SubStat.RemovedMemory;
+	Sum.ElapsedMS += Sub.ElapsedMS;
+}
+
+void
+Sum(GcReferencerStats& Stat)
+{
+	Stat.ElapsedMS = Stat.RemoveExpiredDataStats.ElapsedMS + Stat.CompactStoreStats.ElapsedMS + Stat.CreateReferenceCheckersMS +
+					 Stat.PreCacheStateMS + Stat.LockStateMS;
+}
+
+void
+Add(GcReferencerStats& Sum, const GcReferencerStats& Sub)
+{
+	Add(Sum.RemoveExpiredDataStats, Sub.RemoveExpiredDataStats);
+	Add(Sum.CompactStoreStats, Sub.CompactStoreStats);
+
+	Sum.CreateReferenceCheckersMS += Sub.CreateReferenceCheckersMS;
+	Sum.PreCacheStateMS += Sub.PreCacheStateMS;
+	Sum.LockStateMS += Sub.LockStateMS;
+
+	Sum.ElapsedMS += Sub.ElapsedMS;
+}
+
+void
+Sum(GcReferenceStoreStats& Stat)
+{
+	Stat.ElapsedMS = Stat.RemoveUnreferencedDataStats.ElapsedMS + Stat.CompactStoreStats.ElapsedMS + Stat.CreateReferencePrunersMS;
+}
+
+void
+Add(GcReferenceStoreStats& Sum, const GcReferenceStoreStats& Sub)
+{
+	Add(Sum.RemoveUnreferencedDataStats, Sub.RemoveUnreferencedDataStats);
+	Add(Sum.CompactStoreStats, Sub.CompactStoreStats);
+
+	Sum.CreateReferencePrunersMS += Sub.CreateReferencePrunersMS;
+
+	Sum.ElapsedMS += Sub.ElapsedMS;
+}
+
+GcResult&
+Sum(GcResult& Stat, bool Cancelled = false)
+{
+	for (std::pair<std::string, GcReferencerStats>& Referencer : Stat.ReferencerStats)
+	{
+		GcReferencerStats& SubStat = Referencer.second;
+		Sum(SubStat);
+		Add(Stat.ReferencerStatSum, SubStat);
 	}
+	for (std::pair<std::string, GcReferenceStoreStats>& ReferenceStore : Stat.ReferenceStoreStats)
+	{
+		GcReferenceStoreStats& SubStat = ReferenceStore.second;
+		Sum(SubStat);
+		Add(Stat.ReferenceStoreStatSum, SubStat);
+	}
+
+	Sum(Stat.ReferencerStatSum);
+	Sum(Stat.ReferenceStoreStatSum);
+
+	Add(Stat.CompactStoresStatSum, Stat.ReferencerStatSum.CompactStoreStats);
+	Add(Stat.CompactStoresStatSum, Stat.ReferenceStoreStatSum.CompactStoreStats);
+
+	Stat.WasCancelled = Cancelled;
+
+	return Stat;
 }
 
 void
@@ -563,7 +603,9 @@ GcManager::RemoveGcReferenceStore(GcReferenceStore& ReferenceStore)
 GcResult
 GcManager::CollectGarbage(const GcSettings& Settings)
 {
-	GcCtx	 Ctx{.Settings = Settings};
+	ZEN_TRACE_CPU("GcV2::CollectGarbage");
+
+	GcCtx	 Ctx{.Settings = Settings, .IsCancelledFlag = m_CancelGC};
 	GcResult Result;
 
 	{
@@ -572,256 +614,417 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 
 		RwLock::SharedLockScope GcLock(m_Lock);
 
-		int WorkerThreadPoolCount = 0;
-		if (!Settings.SingleThread)
-		{
-			const size_t MaxHwTreadUse = Max((std::thread::hardware_concurrency() / 4u), 1u);
-			WorkerThreadPoolCount	   = gsl::narrow<int>(Min(MaxHwTreadUse, m_GcReferencers.size()));
-		}
-
 		Result.ReferencerStats.resize(m_GcReferencers.size());
 
-		WorkerThreadPool ThreadPool(WorkerThreadPoolCount);
+		std::unordered_map<std::unique_ptr<GcStoreCompactor>, GcCompactStoreStats*> StoreCompactors;
+		RwLock																		StoreCompactorsLock;
+		WorkerThreadPool& ThreadPool = Settings.SingleThread ? GetSyncWorkerPool() : GetSmallWorkerPool();
 
 		ZEN_INFO("GCV2: Removing expired data from {} referencers", m_GcReferencers.size());
 		if (!m_GcReferencers.empty())
 		{
+			if (CheckGCCancel())
+			{
+				return Sum(Result, true);
+			}
+			ZEN_TRACE_CPU("GcV2::RemoveExpiredData");
+
 			Latch WorkLeft(1);
-			// First remove any cache keys that may own references
-			SCOPED_TIMER(Result.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-			for (size_t Index = 0; Index < m_GcReferencers.size(); Index++)
 			{
-				GcReferencer*							   Owner = m_GcReferencers[Index];
-				std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index];
-				WorkLeft.AddCount(1);
-				ThreadPool.ScheduleWork([&Ctx, Owner, &Stats, &WorkLeft]() {
-					auto _		= MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-					Stats.first = Owner->GetGcName(Ctx);
-					SCOPED_TIMER(Stats.second.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-					Owner->RemoveExpiredData(Ctx, Stats.second);
+				// First remove any cache keys that may own references
+				SCOPED_TIMER(Result.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()); if (Ctx.Settings.Verbose) {
+					ZEN_INFO("GCV2: Removed epxired data for {} referenceners in {}",
+							 m_GcReferencers.size(),
+							 NiceTimeSpanMs(Result.RemoveExpiredDataMS.count()));
 				});
+				for (size_t Index = 0; Index < m_GcReferencers.size(); Index++)
+				{
+					if (CheckGCCancel())
+					{
+						WorkLeft.CountDown();
+						WorkLeft.Wait();
+						return Sum(Result, true);
+					}
+					GcReferencer*							   Owner = m_GcReferencers[Index];
+					std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index];
+					WorkLeft.AddCount(1);
+					ThreadPool.ScheduleWork([&Ctx, &WorkLeft, Owner, &Stats, &StoreCompactorsLock, &StoreCompactors]() {
+						auto _		= MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+						Stats.first = Owner->GetGcName(Ctx);
+						SCOPED_TIMER(Stats.second.RemoveExpiredDataStats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+						std::unique_ptr<GcStoreCompactor> StoreCompactor(
+							Owner->RemoveExpiredData(Ctx, Stats.second.RemoveExpiredDataStats));
+						if (StoreCompactor)
+						{
+							RwLock::ExclusiveLockScope __(StoreCompactorsLock);
+							StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats.second.CompactStoreStats);
+						}
+					});
+				}
+				WorkLeft.CountDown();
+				WorkLeft.Wait();
 			}
-			WorkLeft.CountDown();
-			WorkLeft.Wait();
 		}
 
-		if (Ctx.Settings.SkipCidDelete)
+		if (!Ctx.Settings.SkipCidDelete)
 		{
-			Result.Sum();
-			return Result;
-		}
+			if (CheckGCCancel())
+			{
+				return Sum(Result, true);
+			}
 
-		Result.ReferenceStoreStats.resize(m_GcReferenceStores.size());
+			Result.ReferenceStoreStats.resize(m_GcReferenceStores.size());
 
-		ZEN_INFO("GCV2: Creating reference pruners from {} reference stores", m_GcReferenceStores.size());
-		std::unordered_map<size_t, std::unique_ptr<GcReferencePruner>> ReferencePruners;
-		if (!m_GcReferenceStores.empty())
-		{
-			ReferencePruners.reserve(m_GcReferenceStores.size());
-			Latch  WorkLeft(1);
-			RwLock ReferencePrunersLock;
-			// CreateReferencePruner is usually not very heavy but big data sets change that
-			SCOPED_TIMER(Result.CreateReferencePrunerMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-			for (size_t Index = 0; Index < m_GcReferenceStores.size(); Index++)
+			ZEN_INFO("GCV2: Creating reference pruners from {} reference stores", m_GcReferenceStores.size());
+			std::unordered_map<size_t, std::unique_ptr<GcReferencePruner>> ReferencePruners;
+			if (!m_GcReferenceStores.empty())
 			{
-				GcReferenceStore*							   ReferenceStore = m_GcReferenceStores[Index];
-				std::pair<std::string, GcReferenceStoreStats>& Stats		  = Result.ReferenceStoreStats[Index];
-				WorkLeft.AddCount(1);
-				ThreadPool.ScheduleWork([&Ctx, ReferenceStore, &Stats, Index, &WorkLeft, &ReferencePrunersLock, &ReferencePruners]() {
-					auto _		= MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-					Stats.first = ReferenceStore->GetGcName(Ctx);
-					std::unique_ptr<GcReferencePruner> ReferencePruner;
-					{
-						SCOPED_TIMER(Stats.second.CreateReferencePrunerMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-						// The ReferenceStore will pick a list of CId entries to check, returning a collector
-						ReferencePruner = std::unique_ptr<GcReferencePruner>(ReferenceStore->CreateReferencePruner(Ctx, Stats.second));
-					}
-					if (ReferencePruner)
+				ZEN_TRACE_CPU("GcV2::CreateReferencePruners");
+
+				ReferencePruners.reserve(m_GcReferenceStores.size());
+				Latch  WorkLeft(1);
+				RwLock ReferencePrunersLock;
+				{
+					// CreateReferencePruner is usually not very heavy but big data sets change that
+					SCOPED_TIMER(Result.CreateReferencePrunersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());
+								 if (Ctx.Settings.Verbose) {
+									 ZEN_INFO("GCV2: Created {} reference pruners using {} referencer stores in {}",
+											  ReferencePruners.size(),
+											  m_GcReferenceStores.size(),
+											  NiceTimeSpanMs(Result.CreateReferencePrunersMS.count()));
+								 });
+					for (size_t Index = 0; Index < m_GcReferenceStores.size(); Index++)
 					{
-						RwLock::ExclusiveLockScope __(ReferencePrunersLock);
-						ReferencePruners.insert_or_assign(Index, std::move(ReferencePruner));
+						if (CheckGCCancel())
+						{
+							WorkLeft.CountDown();
+							WorkLeft.Wait();
+							return Sum(Result, true);
+						}
+
+						GcReferenceStore*							   ReferenceStore = m_GcReferenceStores[Index];
+						std::pair<std::string, GcReferenceStoreStats>& Stats		  = Result.ReferenceStoreStats[Index];
+						WorkLeft.AddCount(1);
+						ThreadPool.ScheduleWork(
+							[&Ctx, ReferenceStore, &Stats, Index, &WorkLeft, &ReferencePrunersLock, &ReferencePruners]() {
+								auto _		= MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+								Stats.first = ReferenceStore->GetGcName(Ctx);
+								std::unique_ptr<GcReferencePruner> ReferencePruner;
+								{
+									SCOPED_TIMER(Stats.second.CreateReferencePrunersMS =
+													 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+									// The ReferenceStore will pick a list of CId entries to check, returning a collector
+									ReferencePruner =
+										std::unique_ptr<GcReferencePruner>(ReferenceStore->CreateReferencePruner(Ctx, Stats.second));
+								}
+								if (ReferencePruner)
+								{
+									RwLock::ExclusiveLockScope __(ReferencePrunersLock);
+									ReferencePruners.insert_or_assign(Index, std::move(ReferencePruner));
+								}
+							});
 					}
-				});
+					WorkLeft.CountDown();
+					WorkLeft.Wait();
+				}
 			}
-			WorkLeft.CountDown();
-			WorkLeft.Wait();
-		}
 
-		ZEN_INFO("GCV2: Creating reference checkers from {} referencers", m_GcReferencers.size());
-		std::unordered_map<std::unique_ptr<GcReferenceChecker>, size_t> ReferenceCheckers;
-		if (!m_GcReferencers.empty())
-		{
-			ReferenceCheckers.reserve(m_GcReferencers.size());
-			Latch  WorkLeft(1);
-			RwLock ReferenceCheckersLock;
-			SCOPED_TIMER(Result.CreateReferenceCheckersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-			// Lock all reference owners from changing the reference data and get access to check for referenced data
-			for (size_t Index = 0; Index < m_GcReferencers.size(); Index++)
+			if (!ReferencePruners.empty())
 			{
-				GcReferencer*							   Referencer = m_GcReferencers[Index];
-				std::pair<std::string, GcReferencerStats>& Stats	  = Result.ReferencerStats[Index];
-				WorkLeft.AddCount(1);
-				ThreadPool.ScheduleWork([&Ctx, &WorkLeft, Referencer, Index, &Stats, &ReferenceCheckersLock, &ReferenceCheckers]() {
-					auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-					// The Referencer will create a reference checker that guarrantees that the references do not change as long as it lives
-					std::vector<GcReferenceChecker*> Checkers;
-					{
-						SCOPED_TIMER(Stats.second.CreateReferenceCheckersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-						Checkers = Referencer->CreateReferenceCheckers(Ctx);
-					}
-					try
+				if (CheckGCCancel())
+				{
+					return Sum(Result, true);
+				}
+
+				ZEN_INFO("GCV2: Creating reference checkers from {} referencers", m_GcReferencers.size());
+				std::unordered_map<std::unique_ptr<GcReferenceChecker>, size_t> ReferenceCheckers;
+				if (!m_GcReferencers.empty())
+				{
+					ZEN_TRACE_CPU("GcV2::CreateReferenceCheckers");
+
+					ReferenceCheckers.reserve(m_GcReferencers.size());
+					Latch  WorkLeft(1);
+					RwLock ReferenceCheckersLock;
 					{
-						if (!Checkers.empty())
+						SCOPED_TIMER(Result.CreateReferenceCheckersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());
+									 if (Ctx.Settings.Verbose) {
+										 ZEN_INFO("GCV2: Created {} reference checkers using {} referencers in {}",
+												  ReferenceCheckers.size(),
+												  m_GcReferencers.size(),
+												  NiceTimeSpanMs(Result.CreateReferenceCheckersMS.count()));
+									 });
+						// Lock all reference owners from changing the reference data and get access to check for referenced data
+						for (size_t Index = 0; Index < m_GcReferencers.size(); Index++)
 						{
-							RwLock::ExclusiveLockScope __(ReferenceCheckersLock);
-							for (auto& Checker : Checkers)
+							if (CheckGCCancel())
 							{
-								ReferenceCheckers.insert_or_assign(std::unique_ptr<GcReferenceChecker>(Checker), Index);
-								Checker = nullptr;
+								WorkLeft.CountDown();
+								WorkLeft.Wait();
+								return Sum(Result, true);
 							}
+
+							GcReferencer*							   Referencer = m_GcReferencers[Index];
+							std::pair<std::string, GcReferencerStats>& Stats	  = Result.ReferencerStats[Index];
+							WorkLeft.AddCount(1);
+							ThreadPool.ScheduleWork(
+								[&Ctx, &WorkLeft, Referencer, Index, &Stats, &ReferenceCheckersLock, &ReferenceCheckers]() {
+									auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+									// The Referencer will create a reference checker that guarrantees that the references do not change as
+									// long as it lives
+									std::vector<GcReferenceChecker*> Checkers;
+									{
+										SCOPED_TIMER(Stats.second.CreateReferenceCheckersMS =
+														 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+										Checkers = Referencer->CreateReferenceCheckers(Ctx);
+									}
+									try
+									{
+										if (!Checkers.empty())
+										{
+											RwLock::ExclusiveLockScope __(ReferenceCheckersLock);
+											for (auto& Checker : Checkers)
+											{
+												ReferenceCheckers.insert_or_assign(std::unique_ptr<GcReferenceChecker>(Checker), Index);
+												Checker = nullptr;
+											}
+										}
+									}
+									catch (std::exception&)
+									{
+										while (!Checkers.empty())
+										{
+											delete Checkers.back();
+											Checkers.pop_back();
+										}
+										throw;
+									}
+								});
 						}
+						WorkLeft.CountDown();
+						WorkLeft.Wait();
 					}
-					catch (std::exception&)
+				}
+
+				{
+					ZEN_INFO("GCV2: Precaching state for {} reference checkers", ReferenceCheckers.size());
+					if (!ReferenceCheckers.empty())
 					{
-						while (!Checkers.empty())
+						if (CheckGCCancel())
 						{
-							delete Checkers.back();
-							Checkers.pop_back();
+							return Sum(Result, true);
 						}
-						throw;
-					}
-				});
-			}
-			WorkLeft.CountDown();
-			WorkLeft.Wait();
-		}
-
-		std::unordered_map<std::unique_ptr<GcReferenceStoreCompactor>, size_t> ReferenceStoreCompactors;
-		ReferenceStoreCompactors.reserve(ReferencePruners.size());
+						ZEN_TRACE_CPU("GcV2::PreCache");
 
-		ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size());
-		{
-			SCOPED_TIMER(uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); Result.WriteBlockMS = std::chrono::milliseconds(ElapsedMS);
-						 ZEN_INFO("GCV2: Writes blocked for {}", NiceTimeSpanMs(ElapsedMS)));
-			if (!ReferenceCheckers.empty())
-			{
-				// Locking all references checkers so we have a steady state of which references are used
-				// From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until
-				// we delete the ReferenceCheckers
-				Latch WorkLeft(1);
+						Latch WorkLeft(1);
 
-				SCOPED_TIMER(Result.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-				for (auto& It : ReferenceCheckers)
-				{
-					GcReferenceChecker*						   Checker = It.first.get();
-					size_t									   Index   = It.second;
-					std::pair<std::string, GcReferencerStats>& Stats   = Result.ReferencerStats[Index];
-					WorkLeft.AddCount(1);
-					ThreadPool.ScheduleWork([&Ctx, Checker, Index, &Stats, &WorkLeft]() {
-						auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-						SCOPED_TIMER(Stats.second.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-						Checker->LockState(Ctx);
-					});
+						{
+							SCOPED_TIMER(Result.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());
+										 if (Ctx.Settings.Verbose) {
+											 ZEN_INFO("GCV2: Precached state using {} reference checkers in {}",
+													  ReferenceCheckers.size(),
+													  NiceTimeSpanMs(Result.PreCacheStateMS.count()));
+										 });
+							for (auto& It : ReferenceCheckers)
+							{
+								if (CheckGCCancel())
+								{
+									WorkLeft.CountDown();
+									WorkLeft.Wait();
+									return Sum(Result, true);
+								}
+
+								GcReferenceChecker*						   Checker = It.first.get();
+								size_t									   Index   = It.second;
+								std::pair<std::string, GcReferencerStats>& Stats   = Result.ReferencerStats[Index];
+								WorkLeft.AddCount(1);
+								ThreadPool.ScheduleWork([&Ctx, Checker, Index, &Stats, &WorkLeft]() {
+									auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+									SCOPED_TIMER(Stats.second.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+									Checker->PreCache(Ctx);
+								});
+							}
+							WorkLeft.CountDown();
+							WorkLeft.Wait();
+						}
+					}
 				}
-				WorkLeft.CountDown();
-				WorkLeft.Wait();
-			}
 
-			ZEN_INFO("GCV2: Removing unreferenced data for {} reference pruners", ReferencePruners.size());
-			if (!ReferencePruners.empty())
-			{
-				const auto GetUnusedReferences = [&ReferenceCheckers, &Ctx](std::span<IoHash> References) -> std::vector<IoHash> {
-					HashSet UnusedCids(References.begin(), References.end());
-					for (const auto& It : ReferenceCheckers)
+				SCOPED_TIMER(uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); Result.WriteBlockMS = std::chrono::milliseconds(ElapsedMS);
+							 ZEN_INFO("GCV2: Writes blocked for {}", NiceTimeSpanMs(ElapsedMS)));
+				{
+					ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size());
+					if (!ReferenceCheckers.empty())
 					{
-						GcReferenceChecker* ReferenceChecker = It.first.get();
-						ReferenceChecker->RemoveUsedReferencesFromSet(Ctx, UnusedCids);
-						if (UnusedCids.empty())
+						if (CheckGCCancel())
 						{
-							return {};
+							return Sum(Result, true);
 						}
-					}
-					return std::vector<IoHash>(UnusedCids.begin(), UnusedCids.end());
-				};
-
-				// checking all Cids agains references in cache
-				// Ask stores to remove data that the ReferenceCheckers says are not referenced - this should be a lightweight operation
-				// that only updates in-memory index, actual disk changes should be done by the ReferenceStoreCompactors
+						ZEN_TRACE_CPU("GcV2::LockState");
 
-				Latch  WorkLeft(1);
-				RwLock ReferenceStoreCompactorsLock;
+						// Locking all references checkers so we have a steady state of which references are used
+						// From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until
+						// we delete the ReferenceCheckers
+						Latch WorkLeft(1);
 
-				SCOPED_TIMER(Result.RemoveUnreferencedDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-				for (auto& It : ReferencePruners)
-				{
-					GcReferencePruner*	   Pruner = It.second.get();
-					size_t				   Index  = It.first;
-					GcReferenceStoreStats& Stats  = Result.ReferenceStoreStats[Index].second;
-					WorkLeft.AddCount(1);
-					ThreadPool.ScheduleWork([&Ctx,
-											 Pruner,
-											 &Stats,
-											 &WorkLeft,
-											 Index,
-											 &GetUnusedReferences,
-											 &ReferenceStoreCompactorsLock,
-											 &ReferenceStoreCompactors]() {
-						auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-						// Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
-						std::unique_ptr<GcReferenceStoreCompactor> ReferenceCompactor;
 						{
-							SCOPED_TIMER(Stats.RemoveUnreferencedDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-							ReferenceCompactor =
-								std::unique_ptr<GcReferenceStoreCompactor>(Pruner->RemoveUnreferencedData(Ctx, Stats, GetUnusedReferences));
+							SCOPED_TIMER(Result.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());
+										 if (Ctx.Settings.Verbose) {
+											 ZEN_INFO("GCV2: Locked state using {} reference checkers in {}",
+													  ReferenceCheckers.size(),
+													  NiceTimeSpanMs(Result.LockStateMS.count()));
+										 });
+							for (auto& It : ReferenceCheckers)
+							{
+								if (CheckGCCancel())
+								{
+									WorkLeft.CountDown();
+									WorkLeft.Wait();
+									return Sum(Result, true);
+								}
+
+								GcReferenceChecker*						   Checker = It.first.get();
+								size_t									   Index   = It.second;
+								std::pair<std::string, GcReferencerStats>& Stats   = Result.ReferencerStats[Index];
+								WorkLeft.AddCount(1);
+								ThreadPool.ScheduleWork([&Ctx, Checker, Index, &Stats, &WorkLeft]() {
+									auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+									SCOPED_TIMER(Stats.second.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+									Checker->LockState(Ctx);
+								});
+							}
+							WorkLeft.CountDown();
+							WorkLeft.Wait();
 						}
-						if (ReferenceCompactor)
+					}
+				}
+				{
+					ZEN_INFO("GCV2: Removing unreferenced data for {} reference pruners", ReferencePruners.size());
+					{
+						const auto GetUnusedReferences = [&ReferenceCheckers, &Ctx](std::span<IoHash> References) -> std::vector<IoHash> {
+							HashSet UnusedCids(References.begin(), References.end());
+							for (const auto& It : ReferenceCheckers)
+							{
+								GcReferenceChecker* ReferenceChecker = It.first.get();
+								ReferenceChecker->RemoveUsedReferencesFromSet(Ctx, UnusedCids);
+								if (UnusedCids.empty())
+								{
+									return {};
+								}
+							}
+							return std::vector<IoHash>(UnusedCids.begin(), UnusedCids.end());
+						};
+
+						// checking all Cids agains references in cache
+						// Ask stores to remove data that the ReferenceCheckers says are not referenced - this should be a lightweight
+						// operation that only updates in-memory index, actual disk changes should be done by the ReferenceStoreCompactors
+
+						ZEN_TRACE_CPU("GcV2::RemoveUnreferencedData");
+
+						Latch WorkLeft(1);
+
 						{
-							RwLock::ExclusiveLockScope __(ReferenceStoreCompactorsLock);
-							ReferenceStoreCompactors.insert_or_assign(std::move(ReferenceCompactor), Index);
+							SCOPED_TIMER(Result.RemoveUnreferencedDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());
+										 if (Ctx.Settings.Verbose) {
+											 ZEN_INFO("GCV2: Removed unused data using {} pruners in {}",
+													  ReferencePruners.size(),
+													  NiceTimeSpanMs(Result.RemoveUnreferencedDataMS.count()));
+										 });
+							for (auto& It : ReferencePruners)
+							{
+								if (CheckGCCancel())
+								{
+									WorkLeft.CountDown();
+									WorkLeft.Wait();
+									return Sum(Result, true);
+								}
+
+								GcReferencePruner*	   Pruner = It.second.get();
+								size_t				   Index  = It.first;
+								GcReferenceStoreStats& Stats  = Result.ReferenceStoreStats[Index].second;
+								WorkLeft.AddCount(1);
+								ThreadPool.ScheduleWork(
+									[&Ctx, Pruner, &Stats, &WorkLeft, &GetUnusedReferences, &StoreCompactorsLock, &StoreCompactors]() {
+										auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+										// Go through all the ReferenceCheckers to see if the list of Cids the collector selected are
+										// referenced or not.
+										std::unique_ptr<GcStoreCompactor> StoreCompactor;
+										{
+											SCOPED_TIMER(Stats.RemoveUnreferencedDataStats.ElapsedMS =
+															 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+											StoreCompactor = std::unique_ptr<GcStoreCompactor>(
+												Pruner->RemoveUnreferencedData(Ctx,
+																			   Stats.RemoveUnreferencedDataStats,
+																			   GetUnusedReferences));
+										}
+										if (StoreCompactor)
+										{
+											RwLock::ExclusiveLockScope __(StoreCompactorsLock);
+											StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats.CompactStoreStats);
+										}
+									});
+							}
+							WorkLeft.CountDown();
+							WorkLeft.Wait();
 						}
-					});
+					}
+					// Let the GcReferencers add new data, we will only change on-disk data at this point, adding new data is allowed
+					ReferenceCheckers.clear();
+					ReferencePruners.clear();
 				}
-				WorkLeft.CountDown();
-				WorkLeft.Wait();
 			}
-			// Let the GcReferencers add new data, we will only change on-disk data at this point, adding new data is allowed
-			ReferenceCheckers.clear();
 		}
 
-		// Let go of the pruners
-		ReferencePruners.clear();
-
-		ZEN_INFO("GCV2: Compacting reference stores for {} reference store compactors", ReferenceStoreCompactors.size());
-		if (!ReferenceStoreCompactors.empty())
+		ZEN_INFO("GCV2: Compacting using {} store compactors", StoreCompactors.size());
+		if (!StoreCompactors.empty())
 		{
-			Latch WorkLeft(1);
+			if (CheckGCCancel())
+			{
+				return Sum(Result, true);
+			}
+
+			ZEN_TRACE_CPU("GcV2::CompactStores");
 
+			auto ClaimDiskReserve = [&]() -> uint64_t {
+				if (!std::filesystem::is_regular_file(Settings.DiskReservePath))
+				{
+					return 0;
+				}
+				uint64_t ReclaimedSize = std::filesystem::file_size(Settings.DiskReservePath);
+				if (std::filesystem::remove(Settings.DiskReservePath))
+				{
+					return ReclaimedSize;
+				}
+				return 0;
+			};
 			// Remove the stuff we deemed unreferenced from disk - may be heavy operation
-			SCOPED_TIMER(Result.CompactReferenceStoreMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-			for (auto& It : ReferenceStoreCompactors)
+			// Don't do in parallel, we don't want to steal CPU/Disk from regular operation
 			{
-				GcReferenceStoreCompactor* Compactor = It.first.get();
-				size_t					   Index	 = It.second;
-				GcReferenceStoreStats&	   Stats	 = Result.ReferenceStoreStats[Index].second;
-				WorkLeft.AddCount(1);
-				ThreadPool.ScheduleWork([&Ctx, Compactor, &Stats, &WorkLeft]() {
-					auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-					// Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
-					SCOPED_TIMER(Stats.CompactReferenceStoreMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-					Compactor->CompactReferenceStore(Ctx, Stats);
+				SCOPED_TIMER(Result.CompactStoresMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()); if (Ctx.Settings.Verbose) {
+					ZEN_INFO("GCV2: Compacted {} stores in {}", StoreCompactors.size(), NiceTimeSpanMs(Result.CompactStoresMS.count()));
 				});
+				for (auto& It : StoreCompactors)
+				{
+					if (CheckGCCancel())
+					{
+						return Sum(Result, true);
+					}
+
+					GcStoreCompactor*	 Compactor = It.first.get();
+					GcCompactStoreStats& Stats	   = *It.second;
+					{
+						// Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
+						SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+						Compactor->CompactStore(Ctx, Stats, ClaimDiskReserve);
+					}
+				}
 			}
-			WorkLeft.CountDown();
-			WorkLeft.Wait();
+			StoreCompactors.clear();
 		}
 
-		ReferenceStoreCompactors.clear();
-
 		ZEN_INFO("GCV2: Completed in {}", NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()));
 	}
 
-	Result.Sum();
-	return Result;
+	return Sum(Result);
 }
 
 #undef SCOPED_TIMER
@@ -829,6 +1032,12 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 //////// End GC V2
 
 void
+GcManager::SetCancelGC(bool CancelFlag)
+{
+	m_CancelGC.store(CancelFlag);
+}
+
+void
 GcManager::AddGcContributor(GcContributor* Contributor)
 {
 	RwLock::ExclusiveLockScope _(m_Lock);
@@ -884,6 +1093,10 @@ GcManager::CollectGarbage(GcContext& GcCtx)
 		const auto Guard = MakeGuard([&] { ZEN_INFO("gathered references in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
 		for (GcContributor* Contributor : m_GcContribs)
 		{
+			if (CheckGCCancel())
+			{
+				return GCTotalSizeDiff;
+			}
 			Contributor->GatherReferences(GcCtx);
 		}
 	}
@@ -901,6 +1114,11 @@ GcManager::CollectGarbage(GcContext& GcCtx)
 		});
 		for (GcStorage* Storage : m_GcStorage)
 		{
+			if (CheckGCCancel())
+			{
+				break;
+			}
+
 			const auto PreSize = Storage->StorageSize();
 			Storage->CollectGarbage(GcCtx);
 			const auto PostSize = Storage->StorageSize();
@@ -1127,7 +1345,12 @@ GcScheduler::Shutdown()
 	if (static_cast<uint32_t>(GcSchedulerStatus::kStopped) != m_Status)
 	{
 		bool GcIsRunning = m_Status == static_cast<uint32_t>(GcSchedulerStatus::kRunning);
-		m_Status		 = static_cast<uint32_t>(GcSchedulerStatus::kStopped);
+		if (GcIsRunning)
+		{
+			ZEN_INFO("Requesting cancel running garbage collection");
+		}
+		m_GcManager.SetCancelGC(true);
+		m_Status = static_cast<uint32_t>(GcSchedulerStatus::kStopped);
 		m_GcSignal.notify_one();
 
 		if (m_GcThread.joinable())
@@ -1183,6 +1406,20 @@ GcScheduler::TriggerScrub(const TriggerScrubParams& Params)
 	return false;
 }
 
+bool
+GcScheduler::CancelGC()
+{
+	std::unique_lock Lock(m_GcMutex);
+
+	if (static_cast<uint32_t>(GcSchedulerStatus::kRunning) == m_Status)
+	{
+		ZEN_INFO("Cancel requested for running garbage collection");
+		m_GcManager.SetCancelGC(true);
+		return true;
+	}
+	return false;
+}
+
 DiskSpace
 GcScheduler::CheckDiskSpace()
 {
@@ -1227,22 +1464,17 @@ GcScheduler::AppendGCLog(GcClock::TimePoint StartTime, const GcSettings& Setting
 			std::string	   Id = fmt::format("{}", gsl::narrow<int64_t>(StartTime.time_since_epoch().count()));
 			Writer.BeginObject(Id);
 			{
-				Writer << "StartTimeSec"sv
-					   << gsl::narrow<int64_t>(std::chrono::duration_cast<std::chrono::seconds>(StartTime.time_since_epoch()).count());
+				Writer << "StartTime"sv << ToDateTime(StartTime);
 				Writer.BeginObject("Settings"sv);
 				{
-					Writer << "CacheExpireTimeSec"sv
-						   << gsl::narrow<int64_t>(
-								  std::chrono::duration_cast<std::chrono::seconds>(Settings.CacheExpireTime.time_since_epoch()).count());
-					Writer << "ProjectStoreExpireTimeSec"sv
-						   << gsl::narrow<int64_t>(
-								  std::chrono::duration_cast<std::chrono::seconds>(Settings.ProjectStoreExpireTime.time_since_epoch())
-									  .count());
+					Writer << "CacheExpireTime"sv << ToDateTime(Settings.CacheExpireTime);
+					Writer << "ProjectStoreExpireTime"sv << ToDateTime(Settings.ProjectStoreExpireTime);
 					Writer << "CollectSmallObjects"sv << Settings.CollectSmallObjects;
 					Writer << "IsDeleteMode"sv << Settings.IsDeleteMode;
 					Writer << "SkipCidDelete"sv << Settings.SkipCidDelete;
 					Writer << "Verbose"sv << Settings.Verbose;
 					Writer << "SingleThread"sv << Settings.SingleThread;
+					Writer << "CompactBlockUsageThresholdPercent"sv << Settings.CompactBlockUsageThresholdPercent;
 				}
 				Writer.EndObject();
 
@@ -1417,18 +1649,20 @@ GcScheduler::SchedulerThread()
 
 		try
 		{
-			bool				 DoGc					 = m_Config.Enabled;
-			bool				 DoScrubbing			 = false;
-			std::chrono::seconds ScrubTimeslice			 = std::chrono::seconds::max();
-			bool				 DoDelete				 = true;
-			bool				 CollectSmallObjects	 = m_Config.CollectSmallObjects;
-			std::chrono::seconds GcInterval				 = m_Config.Interval;
-			std::chrono::seconds LightweightGcInterval	 = m_Config.LightweightInterval;
-			std::chrono::seconds MaxCacheDuration		 = m_Config.MaxCacheDuration;
-			std::chrono::seconds MaxProjectStoreDuration = m_Config.MaxProjectStoreDuration;
-			uint64_t			 DiskSizeSoftLimit		 = m_Config.DiskSizeSoftLimit;
-			bool				 SkipCid				 = false;
-			GcVersion			 UseGCVersion			 = m_Config.UseGCVersion;
+			bool				 DoGc							   = m_Config.Enabled;
+			bool				 DoScrubbing					   = false;
+			std::chrono::seconds ScrubTimeslice					   = std::chrono::seconds::max();
+			bool				 DoDelete						   = true;
+			bool				 CollectSmallObjects			   = m_Config.CollectSmallObjects;
+			std::chrono::seconds GcInterval						   = m_Config.Interval;
+			std::chrono::seconds LightweightGcInterval			   = m_Config.LightweightInterval;
+			std::chrono::seconds MaxCacheDuration				   = m_Config.MaxCacheDuration;
+			std::chrono::seconds MaxProjectStoreDuration		   = m_Config.MaxProjectStoreDuration;
+			uint64_t			 DiskSizeSoftLimit				   = m_Config.DiskSizeSoftLimit;
+			bool				 SkipCid						   = false;
+			GcVersion			 UseGCVersion					   = m_Config.UseGCVersion;
+			uint32_t			 CompactBlockUsageThresholdPercent = m_Config.CompactBlockUsageThresholdPercent;
+			bool				 Verbose						   = m_Config.Verbose;
 
 			bool DiskSpaceGCTriggered = false;
 			bool TimeBasedGCTriggered = false;
@@ -1463,7 +1697,10 @@ GcScheduler::SchedulerThread()
 					DoDelete = false;
 				}
 				UseGCVersion = TriggerParams.ForceGCVersion.value_or(UseGCVersion);
-				DoGc		 = true;
+				CompactBlockUsageThresholdPercent =
+					TriggerParams.CompactBlockUsageThresholdPercent.value_or(CompactBlockUsageThresholdPercent);
+				Verbose = TriggerParams.Verbose.value_or(Verbose);
+				DoGc	= true;
 			}
 
 			if (m_TriggerScrubParams)
@@ -1475,12 +1712,18 @@ GcScheduler::SchedulerThread()
 					DoGc = false;
 				}
 
+				if (m_TriggerScrubParams->SkipCas)
+				{
+					SkipCid = true;
+				}
+
+				DoDelete	   = !m_TriggerScrubParams->SkipDelete;
 				ScrubTimeslice = m_TriggerScrubParams->MaxTimeslice;
 			}
 
 			if (DoScrubbing)
 			{
-				ScrubStorage(DoDelete, ScrubTimeslice);
+				ScrubStorage(DoDelete, SkipCid, ScrubTimeslice);
 				m_TriggerScrubParams.reset();
 			}
 
@@ -1668,7 +1911,16 @@ GcScheduler::SchedulerThread()
 				}
 			}
 
-			CollectGarbage(CacheExpireTime, ProjectStoreExpireTime, DoDelete, CollectSmallObjects, SkipCid, UseGCVersion);
+			CollectGarbage(CacheExpireTime,
+						   ProjectStoreExpireTime,
+						   DoDelete,
+						   CollectSmallObjects,
+						   SkipCid,
+						   UseGCVersion,
+						   CompactBlockUsageThresholdPercent,
+						   Verbose);
+
+			m_GcManager.SetCancelGC(false);
 
 			uint32_t RunningState = static_cast<uint32_t>(GcSchedulerStatus::kRunning);
 			if (!m_Status.compare_exchange_strong(RunningState, static_cast<uint32_t>(GcSchedulerStatus::kIdle)))
@@ -1715,7 +1967,7 @@ GcScheduler::SchedulerThread()
 }
 
 void
-GcScheduler::ScrubStorage(bool DoDelete, std::chrono::seconds TimeSlice)
+GcScheduler::ScrubStorage(bool DoDelete, bool SkipCid, std::chrono::seconds TimeSlice)
 {
 	const std::chrono::steady_clock::time_point TimeNow	 = std::chrono::steady_clock::now();
 	std::chrono::steady_clock::time_point		Deadline = TimeNow + TimeSlice;
@@ -1726,13 +1978,14 @@ GcScheduler::ScrubStorage(bool DoDelete, std::chrono::seconds TimeSlice)
 	}
 
 	Stopwatch Timer;
-	ZEN_INFO("scrubbing STARTING (delete mode => {})", DoDelete);
+	ZEN_INFO("scrubbing STARTING (delete mode => {}, skip CID => {})", DoDelete, SkipCid);
 
-	WorkerThreadPool ThreadPool{4, "scrubber"};
-	ScrubContext	 Ctx{ThreadPool, Deadline};
+	WorkerThreadPool& ThreadPool = GetSmallWorkerPool();
+	ScrubContext	  Ctx{ThreadPool, Deadline};
 
 	try
 	{
+		Ctx.SetSkipCas(SkipCid);
 		Ctx.SetShouldDelete(DoDelete);
 		m_GcManager.ScrubStorage(Ctx);
 	}
@@ -1750,7 +2003,9 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 							bool					  Delete,
 							bool					  CollectSmallObjects,
 							bool					  SkipCid,
-							GcVersion				  UseGCVersion)
+							GcVersion				  UseGCVersion,
+							uint32_t				  CompactBlockUsageThresholdPercent,
+							bool					  Verbose)
 {
 	ZEN_TRACE_CPU("GcScheduler::CollectGarbage");
 
@@ -1813,30 +2068,33 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 				break;
 			case GcVersion::kV2:
 				{
-					const GcSettings   Settings	   = {.CacheExpireTime		  = CacheExpireTime,
-													  .ProjectStoreExpireTime = ProjectStoreExpireTime,
-													  .CollectSmallObjects	  = CollectSmallObjects,
-													  .IsDeleteMode			  = Delete,
-													  .SkipCidDelete		  = SkipCid};
+					const GcSettings   Settings	   = {.CacheExpireTime					 = CacheExpireTime,
+													  .ProjectStoreExpireTime			 = ProjectStoreExpireTime,
+													  .CollectSmallObjects				 = CollectSmallObjects,
+													  .IsDeleteMode						 = Delete,
+													  .SkipCidDelete					 = SkipCid,
+													  .Verbose							 = Verbose,
+													  .CompactBlockUsageThresholdPercent = CompactBlockUsageThresholdPercent,
+													  .DiskReservePath					 = m_Config.RootDirectory / "reserve.gc"};
 					GcClock::TimePoint GcStartTime = GcClock::Now();
 					GcResult		   Result	   = m_GcManager.CollectGarbage(Settings);
 
 					ZEN_INFO(
-						"GCV2: Removed {} items out of {}, deleted {} out of {}. Pruned {} Cid entries out of {}, compacted {} Cid entries "
-						"out of {}, "
-						"freed "
-						"{} on disk and {} of memory in {}. CacheExpireTime: {}, ProjectStoreExpireTime: {}, CollectSmallObjects: {}, "
+						"GCV2: Found {} expired items out of {}, deleted {}. "
+						"Found {} unreferenced Cid entries out of {}, deleted {}. "
+						"Freed {} on disk and {} of memory in {}. "
+						"CacheExpireTime: {}, ProjectStoreExpireTime: {}, CollectSmallObjects: {}, "
 						"IsDeleteMode: {}, SkipCidDelete: {}",
-						Result.ReferencerStat.Expired,
-						Result.ReferencerStat.Count,
-						Result.ReferencerStat.Deleted,
-						Result.ReferencerStat.Expired,
-						Result.ReferenceStoreStat.Pruned,
-						Result.ReferenceStoreStat.Count,
-						Result.ReferenceStoreStat.Compacted,
-						Result.ReferenceStoreStat.Pruned,
-						NiceBytes(Result.RemovedDisk),
-						NiceBytes(Result.RemovedMemory),
+						Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount,
+						Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount,
+						Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount,
+
+						Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount,
+						Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount,
+						Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount,
+
+						NiceBytes(Result.CompactStoresStatSum.RemovedDisk),
+						NiceBytes(Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory),
 						NiceTimeSpanMs(Result.ElapsedMS.count()),
 						Settings.CacheExpireTime,
 						Settings.ProjectStoreExpireTime,
@@ -1854,8 +2112,8 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 					{
 						m_LastFullGCV2Result = Result;
 					}
-					Diff.DiskSize	= Result.RemovedDisk;
-					Diff.MemorySize = Result.RemovedMemory;
+					Diff.DiskSize	= Result.CompactStoresStatSum.RemovedDisk;
+					Diff.MemorySize = Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory;
 				}
 				break;
 		}