Merge branch 'main' into zs/put-overwrite-policy

1 files changed, 416 insertions, 165 deletions

diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 7ac10d613..a15a2e084 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -62,11 +62,11 @@ namespace {
 	{
 		if (Size == 0)
 		{
-			std::filesystem::remove(Path);
+			RemoveFile(Path);
 			return std::error_code{};
 		}
 		CreateDirectories(Path.parent_path());
-		if (std::filesystem::is_regular_file(Path) && std::filesystem::file_size(Path) == Size)
+		if (IsFile(Path) && FileSizeFromPath(Path) == Size)
 		{
 			return std::error_code();
 		}
@@ -709,7 +709,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 		RwLock																		StoreCompactorsLock;
 		std::unordered_map<std::unique_ptr<GcReferenceValidator>, size_t>			ReferenceValidators;
 		RwLock																		ReferenceValidatorsLock;
-		WorkerThreadPool&															PreCachePhaseThreadPool =
+		WorkerThreadPool&															ParallelWorkThreadPool =
 			  Settings.SingleThread ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Background);
 
 		if (!m_GcReferencers.empty())
@@ -721,7 +721,6 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 			ZEN_INFO("GCV2: Removing expired data from {} referencers", m_GcReferencers.size());
 			ZEN_TRACE_CPU("GcV2::RemoveExpiredData");
 
-			Latch WorkLeft(1);
 			{
 				// First remove any cache keys that may own references
 				SCOPED_TIMER(Result.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()); if (Ctx.Settings.Verbose) {
@@ -733,39 +732,45 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 				{
 					if (CheckGCCancel())
 					{
-						WorkLeft.CountDown();
-						WorkLeft.Wait();
 						return Sum(Result, true);
 					}
 					GcReferencer*							   Owner = m_GcReferencers[Index];
 					std::pair<std::string, GcReferencerStats>* Stats = &Result.ReferencerStats[Index];
-					WorkLeft.AddCount(1);
-					PreCachePhaseThreadPool.ScheduleWork([this, &Ctx, &WorkLeft, Owner, Stats, &StoreCompactorsLock, &StoreCompactors]() {
-						ZEN_MEMSCOPE(GetGcTag());
-
-						auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-						try
+					try
+					{
+						Stats->first = Owner->GetGcName(Ctx);
+						SCOPED_TIMER(Stats->second.RemoveExpiredDataStats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+						std::unique_ptr<GcStoreCompactor> StoreCompactor(
+							Owner->RemoveExpiredData(Ctx, Stats->second.RemoveExpiredDataStats));
+						if (StoreCompactor)
 						{
-							Stats->first = Owner->GetGcName(Ctx);
-							SCOPED_TIMER(Stats->second.RemoveExpiredDataStats.ElapsedMS =
-											 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-							std::unique_ptr<GcStoreCompactor> StoreCompactor(
-								Owner->RemoveExpiredData(Ctx, Stats->second.RemoveExpiredDataStats));
-							if (StoreCompactor)
-							{
-								RwLock::ExclusiveLockScope __(StoreCompactorsLock);
-								StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats->second.CompactStoreStats);
-							}
+							RwLock::ExclusiveLockScope __(StoreCompactorsLock);
+							StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats->second.CompactStoreStats);
 						}
-						catch (const std::exception& Ex)
+					}
+					catch (const std::system_error& Ex)
+					{
+						if (IsOOD(Ex) || IsOOM(Ex))
+						{
+							ZEN_WARN("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
+						}
+						else
 						{
 							ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
-							SetCancelGC(true);
 						}
-					});
+						SetCancelGC(true);
+					}
+					catch (const std::bad_alloc& Ex)
+					{
+						ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
+						SetCancelGC(true);
+					}
+					catch (const std::exception& Ex)
+					{
+						ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what());
+						SetCancelGC(true);
+					}
 				}
-				WorkLeft.CountDown();
-				WorkLeft.Wait();
 			}
 		}
 
@@ -810,7 +815,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						GcReferenceStore*							   ReferenceStore = m_GcReferenceStores[Index];
 						std::pair<std::string, GcReferenceStoreStats>* Stats		  = &Result.ReferenceStoreStats[Index];
 						WorkLeft.AddCount(1);
-						PreCachePhaseThreadPool.ScheduleWork(
+						ParallelWorkThreadPool.ScheduleWork(
 							[this, &Ctx, ReferenceStore, Stats, Index, &WorkLeft, &ReferencePrunersLock, &ReferencePruners]() {
 								ZEN_MEMSCOPE(GetGcTag());
 
@@ -832,6 +837,29 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										ReferencePruners.insert_or_assign(Index, std::move(ReferencePruner));
 									}
 								}
+								catch (const std::system_error& Ex)
+								{
+									if (IsOOD(Ex) || IsOOM(Ex))
+									{
+										ZEN_WARN("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
+												 ReferenceStore->GetGcName(Ctx),
+												 Ex.what());
+									}
+									else
+									{
+										ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
+												  ReferenceStore->GetGcName(Ctx),
+												  Ex.what());
+									}
+									SetCancelGC(true);
+								}
+								catch (const std::bad_alloc& Ex)
+								{
+									ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
+											  ReferenceStore->GetGcName(Ctx),
+											  Ex.what());
+									SetCancelGC(true);
+								}
 								catch (const std::exception& Ex)
 								{
 									ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'",
@@ -885,41 +913,70 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 								GcReferencer*							   Referencer = m_GcReferencers[Index];
 								std::pair<std::string, GcReferencerStats>* Stats	  = &Result.ReferencerStats[Index];
 								WorkLeft.AddCount(1);
-								PreCachePhaseThreadPool.ScheduleWork(
+								ParallelWorkThreadPool.ScheduleWork(
 									[this, &Ctx, &WorkLeft, Referencer, Index, Stats, &ReferenceCheckersLock, &ReferenceCheckers]() {
 										ZEN_MEMSCOPE(GetGcTag());
 
 										auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-										// The Referencer will create a reference checker that guarantees that the references do not change
-										// as long as it lives
-										std::vector<GcReferenceChecker*> Checkers;
-										try
+										if (!CheckGCCancel())
 										{
+											// The Referencer will create a reference checker that guarantees that the references do not
+											// change as long as it lives
+											std::vector<GcReferenceChecker*> Checkers;
+											auto							 __ = MakeGuard([&Checkers]() {
+												while (!Checkers.empty())
+												{
+													delete Checkers.back();
+													Checkers.pop_back();
+												}
+											});
+											try
 											{
-												SCOPED_TIMER(Stats->second.CreateReferenceCheckersMS =
-																 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-												Checkers = Referencer->CreateReferenceCheckers(Ctx);
+												{
+													SCOPED_TIMER(Stats->second.CreateReferenceCheckersMS =
+																	 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+													Checkers = Referencer->CreateReferenceCheckers(Ctx);
+												}
+												if (!Checkers.empty())
+												{
+													RwLock::ExclusiveLockScope __(ReferenceCheckersLock);
+													for (auto& Checker : Checkers)
+													{
+														ReferenceCheckers.insert_or_assign(std::unique_ptr<GcReferenceChecker>(Checker),
+																						   Index);
+														Checker = nullptr;
+													}
+												}
 											}
-											if (!Checkers.empty())
+											catch (const std::system_error& Ex)
 											{
-												RwLock::ExclusiveLockScope __(ReferenceCheckersLock);
-												for (auto& Checker : Checkers)
+												if (IsOOD(Ex) || IsOOM(Ex))
+												{
+													ZEN_WARN("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
+															 Referencer->GetGcName(Ctx),
+															 Ex.what());
+												}
+												else
 												{
-													ReferenceCheckers.insert_or_assign(std::unique_ptr<GcReferenceChecker>(Checker), Index);
-													Checker = nullptr;
+													ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
+															  Referencer->GetGcName(Ctx),
+															  Ex.what());
 												}
+												SetCancelGC(true);
 											}
-										}
-										catch (const std::exception& Ex)
-										{
-											ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
-													  Referencer->GetGcName(Ctx),
-													  Ex.what());
-											SetCancelGC(true);
-											while (!Checkers.empty())
+											catch (const std::bad_alloc& Ex)
+											{
+												ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
+														  Referencer->GetGcName(Ctx),
+														  Ex.what());
+												SetCancelGC(true);
+											}
+											catch (const std::exception& Ex)
 											{
-												delete Checkers.back();
-												Checkers.pop_back();
+												ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'",
+														  Referencer->GetGcName(Ctx),
+														  Ex.what());
+												SetCancelGC(true);
 											}
 										}
 									});
@@ -962,19 +1019,26 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 							GcReferencer*							   Referencer	   = m_GcReferencers[Index];
 							std::pair<std::string, GcReferencerStats>* ReferemcerStats = &Result.ReferencerStats[Index];
 							WorkLeft.AddCount(1);
-							PreCachePhaseThreadPool.ScheduleWork([this,
-																  &Ctx,
-																  &WorkLeft,
-																  Referencer,
-																  Index,
-																  Result = &Result,
-																  ReferemcerStats,
-																  &ReferenceValidatorsLock,
-																  &ReferenceValidators]() {
+							ParallelWorkThreadPool.ScheduleWork([this,
+																 &Ctx,
+																 &WorkLeft,
+																 Referencer,
+																 Index,
+																 Result = &Result,
+																 ReferemcerStats,
+																 &ReferenceValidatorsLock,
+																 &ReferenceValidators]() {
 								ZEN_MEMSCOPE(GetGcTag());
 
 								auto							   _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
 								std::vector<GcReferenceValidator*> Validators;
+								auto							   __ = MakeGuard([&Validators]() {
+									  while (!Validators.empty())
+									  {
+										  delete Validators.back();
+										  Validators.pop_back();
+									  }
+								  });
 								try
 								{
 									{
@@ -995,17 +1059,35 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 										}
 									}
 								}
+								catch (const std::system_error& Ex)
+								{
+									if (IsOOD(Ex) || IsOOM(Ex))
+									{
+										ZEN_WARN("GCV2: Failed creating reference validators for {}. Reason: '{}'",
+												 Referencer->GetGcName(Ctx),
+												 Ex.what());
+									}
+									else
+									{
+										ZEN_ERROR("GCV2: Failed creating reference validators for {}. Reason: '{}'",
+												  Referencer->GetGcName(Ctx),
+												  Ex.what());
+									}
+									SetCancelGC(true);
+								}
+								catch (const std::bad_alloc& Ex)
+								{
+									ZEN_ERROR("GCV2: Failed creating reference validators for {}. Reason: '{}'",
+											  Referencer->GetGcName(Ctx),
+											  Ex.what());
+									SetCancelGC(true);
+								}
 								catch (const std::exception& Ex)
 								{
 									ZEN_ERROR("GCV2: Failed creating reference validators for {}. Reason: '{}'",
 											  Referencer->GetGcName(Ctx),
 											  Ex.what());
 									SetCancelGC(true);
-									while (!Validators.empty())
-									{
-										delete Validators.back();
-										Validators.pop_back();
-									}
 								}
 							});
 						}
@@ -1023,8 +1105,6 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 					ZEN_INFO("GCV2: Precaching state for {} reference checkers", ReferenceCheckers.size());
 					ZEN_TRACE_CPU("GcV2::PreCache");
 
-					Latch WorkLeft(1);
-
 					{
 						SCOPED_TIMER(Result.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());
 									 if (Ctx.Settings.Verbose) {
@@ -1036,33 +1116,40 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						{
 							if (CheckGCCancel())
 							{
-								WorkLeft.CountDown();
-								WorkLeft.Wait();
 								return Sum(Result, true);
 							}
 
 							GcReferenceChecker*						   Checker = It.first.get();
 							size_t									   Index   = It.second;
 							std::pair<std::string, GcReferencerStats>* Stats   = &Result.ReferencerStats[Index];
-							WorkLeft.AddCount(1);
-							PreCachePhaseThreadPool.ScheduleWork([this, &Ctx, Checker, Index, Stats, &WorkLeft]() {
-								ZEN_MEMSCOPE(GetGcTag());
-
-								auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-								try
+							try
+							{
+								SCOPED_TIMER(Stats->second.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+								Checker->PreCache(Ctx);
+							}
+							catch (const std::system_error& Ex)
+							{
+								if (IsOOD(Ex) || IsOOM(Ex))
 								{
-									SCOPED_TIMER(Stats->second.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-									Checker->PreCache(Ctx);
+									ZEN_WARN("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
 								}
-								catch (const std::exception& Ex)
+								else
 								{
 									ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
-									SetCancelGC(true);
 								}
-							});
+								SetCancelGC(true);
+							}
+							catch (const std::bad_alloc& Ex)
+							{
+								ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
+								SetCancelGC(true);
+							}
+							catch (const std::exception& Ex)
+							{
+								ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what());
+								SetCancelGC(true);
+							}
 						}
-						WorkLeft.CountDown();
-						WorkLeft.Wait();
 					}
 				}
 
@@ -1081,7 +1168,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size());
 						{
 							ZEN_TRACE_CPU("GcV2::LockReferencers");
-							// From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until
+							// From this point we have blocked all writes to all References (DiskBucket/ProjectStore/BuildStore) until
 							// we delete the ReferenceLockers
 							Latch WorkLeft(1);
 							{
@@ -1108,7 +1195,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 							ZEN_TRACE_CPU("GcV2::UpdateLockedState");
 
 							// Locking all references checkers so we have a steady state of which references are used
-							// From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until
+							// From this point we have blocked all writes to all References (DiskBucket/ProjectStore/BuildStore) until
 							// we delete the ReferenceCheckers
 							Latch WorkLeft(1);
 
@@ -1142,6 +1229,29 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 															 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
 											Checker->UpdateLockedState(Ctx);
 										}
+										catch (const std::system_error& Ex)
+										{
+											if (IsOOD(Ex) || IsOOM(Ex))
+											{
+												ZEN_WARN("GCV2: Failed Updating locked state for {}. Reason: '{}'",
+														 Checker->GetGcName(Ctx),
+														 Ex.what());
+											}
+											else
+											{
+												ZEN_ERROR("GCV2: Failed Updating locked state for {}. Reason: '{}'",
+														  Checker->GetGcName(Ctx),
+														  Ex.what());
+											}
+											SetCancelGC(true);
+										}
+										catch (const std::bad_alloc& Ex)
+										{
+											ZEN_WARN("GCV2: Failed Updating locked state for {}. Reason: '{}'",
+													 Checker->GetGcName(Ctx),
+													 Ex.what());
+											SetCancelGC(true);
+										}
 										catch (const std::exception& Ex)
 										{
 											ZEN_ERROR("GCV2: Failed Updating locked state for {}. Reason: '{}'",
@@ -1231,6 +1341,29 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 											StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats->CompactStoreStats);
 										}
 									}
+									catch (const std::system_error& Ex)
+									{
+										if (IsOOD(Ex) || IsOOM(Ex))
+										{
+											ZEN_WARN("GCV2: Failed removing unused data for {}. Reason: '{}'",
+													 Pruner->GetGcName(Ctx),
+													 Ex.what());
+										}
+										else
+										{
+											ZEN_ERROR("GCV2: Failed removing unused data for {}. Reason: '{}'",
+													  Pruner->GetGcName(Ctx),
+													  Ex.what());
+										}
+										SetCancelGC(true);
+									}
+									catch (const std::bad_alloc& Ex)
+									{
+										ZEN_ERROR("GCV2: Failed removing unused data for {}. Reason: '{}'",
+												  Pruner->GetGcName(Ctx),
+												  Ex.what());
+										SetCancelGC(true);
+									}
 									catch (const std::exception& Ex)
 									{
 										ZEN_ERROR("GCV2: Failed removing unused data for {}. Reason: '{}'",
@@ -1262,12 +1395,12 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 			ZEN_TRACE_CPU("GcV2::CompactStores");
 
 			auto ClaimDiskReserve = [&]() -> uint64_t {
-				if (!std::filesystem::is_regular_file(Settings.DiskReservePath))
+				if (!IsFile(Settings.DiskReservePath))
 				{
 					return 0;
 				}
-				uint64_t ReclaimedSize = std::filesystem::file_size(Settings.DiskReservePath);
-				if (std::filesystem::remove(Settings.DiskReservePath))
+				uint64_t ReclaimedSize = FileSizeFromPath(Settings.DiskReservePath);
+				if (RemoveFile(Settings.DiskReservePath))
 				{
 					return ReclaimedSize;
 				}
@@ -1294,6 +1427,23 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
 						Compactor->CompactStore(Ctx, Stats, ClaimDiskReserve);
 					}
+					catch (const std::system_error& Ex)
+					{
+						if (IsOOD(Ex) || IsOOM(Ex))
+						{
+							ZEN_WARN("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what());
+						}
+						else
+						{
+							ZEN_ERROR("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what());
+						}
+						SetCancelGC(true);
+					}
+					catch (const std::bad_alloc& Ex)
+					{
+						ZEN_ERROR("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what());
+						SetCancelGC(true);
+					}
 					catch (const std::exception& Ex)
 					{
 						ZEN_ERROR("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what());
@@ -1335,6 +1485,23 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
 						ReferenceValidator->Validate(Ctx, Stats);
 					}
+					catch (const std::system_error& Ex)
+					{
+						if (IsOOD(Ex) || IsOOM(Ex))
+						{
+							ZEN_WARN("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what());
+						}
+						else
+						{
+							ZEN_ERROR("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what());
+						}
+						SetCancelGC(true);
+					}
+					catch (const std::bad_alloc& Ex)
+					{
+						ZEN_ERROR("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what());
+						SetCancelGC(true);
+					}
 					catch (const std::exception& Ex)
 					{
 						ZEN_ERROR("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what());
@@ -1557,7 +1724,7 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
 		m_Config.LightweightInterval = m_Config.MonitorInterval;
 	}
 
-	std::filesystem::create_directories(Config.RootDirectory);
+	CreateDirectories(Config.RootDirectory);
 
 	std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
 	if (Ec)
@@ -1739,6 +1906,7 @@ GcScheduler::AppendGCLog(std::string_view Id, GcClock::TimePoint StartTime, cons
 				{
 					Writer << "CacheExpireTime"sv << ToDateTime(Settings.CacheExpireTime);
 					Writer << "ProjectStoreExpireTime"sv << ToDateTime(Settings.ProjectStoreExpireTime);
+					Writer << "BuildStoreExpireTime"sv << ToDateTime(Settings.BuildStoreExpireTime);
 					Writer << "CollectSmallObjects"sv << Settings.CollectSmallObjects;
 					Writer << "IsDeleteMode"sv << Settings.IsDeleteMode;
 					Writer << "SkipCidDelete"sv << Settings.SkipCidDelete;
@@ -1849,7 +2017,7 @@ GcScheduler::GetState() const
 	if (Result.Config.DiskReserveSize != 0)
 	{
 		Ec.clear();
-		Result.HasDiskReserve = std::filesystem::is_regular_file(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec;
+		Result.HasDiskReserve = IsFile(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec;
 	}
 
 	if (Result.Status != GcSchedulerStatus::kRunning)
@@ -1900,17 +2068,46 @@ GcScheduler::SchedulerThread()
 	ZEN_MEMSCOPE(GetGcTag());
 	SetCurrentThreadName("GcScheduler");
 
-	std::chrono::seconds WaitTime{0};
-
-	bool SilenceErrors = false;
+	std::chrono::seconds	   WaitTime{0};
+	const std::chrono::seconds ShortWaitTime{5};
+	bool					   SilenceErrors = false;
 
 	for (;;)
 	{
-		bool Timeout = false;
+		(void)CheckDiskSpace();
+
+		std::chrono::seconds WaitedTime{0};
+		bool				 Timeout = false;
 		{
 			ZEN_ASSERT(WaitTime.count() >= 0);
 			std::unique_lock Lock(m_GcMutex);
-			Timeout = std::cv_status::timeout == m_GcSignal.wait_for(Lock, WaitTime);
+			while (!Timeout)
+			{
+				std::chrono::seconds ShortWait	  = Min(WaitTime, ShortWaitTime);
+				bool				 ShortTimeout = std::cv_status::timeout == m_GcSignal.wait_for(Lock, ShortWait);
+				if (ShortTimeout)
+				{
+					if (WaitTime > ShortWaitTime)
+					{
+						DiskSpace Space = CheckDiskSpace();
+						if (!AreDiskWritesAllowed())
+						{
+							ZEN_INFO("Triggering GC due to low disk space ({}) on {}", NiceBytes(Space.Free), m_Config.RootDirectory);
+							Timeout = true;
+						}
+						WaitTime -= ShortWaitTime;
+					}
+					else
+					{
+						Timeout = true;
+					}
+				}
+				else
+				{
+					// We got a signal
+					break;
+				}
+			}
 		}
 
 		if (Status() == GcSchedulerStatus::kStopped)
@@ -1940,7 +2137,9 @@ GcScheduler::SchedulerThread()
 			std::chrono::seconds LightweightGcInterval			   = m_Config.LightweightInterval;
 			std::chrono::seconds MaxCacheDuration				   = m_Config.MaxCacheDuration;
 			std::chrono::seconds MaxProjectStoreDuration		   = m_Config.MaxProjectStoreDuration;
+			std::chrono::seconds MaxBuildStoreDuration			   = m_Config.MaxBuildStoreDuration;
 			uint64_t			 DiskSizeSoftLimit				   = m_Config.DiskSizeSoftLimit;
+			uint64_t			 MinimumFreeDiskSpaceToAllowWrites = m_Config.MinimumFreeDiskSpaceToAllowWrites;
 			bool				 SkipCid						   = false;
 			GcVersion			 UseGCVersion					   = m_Config.UseGCVersion;
 			uint32_t			 CompactBlockUsageThresholdPercent = m_Config.CompactBlockUsageThresholdPercent;
@@ -1955,8 +2154,9 @@ GcScheduler::SchedulerThread()
 			uint8_t NextAttachmentPassIndex =
 				ComputeAttachmentRange(m_AttachmentPassIndex, m_Config.AttachmentPassCount, AttachmentRangeMin, AttachmentRangeMax);
 
-			bool DiskSpaceGCTriggered = false;
-			bool TimeBasedGCTriggered = false;
+			bool LowDiskSpaceGCTriggered	   = false;
+			bool HighDiskSpaceUsageGCTriggered = false;
+			bool TimeBasedGCTriggered		   = false;
 
 			GcClock::TimePoint Now = GcClock::Now();
 
@@ -1975,6 +2175,10 @@ GcScheduler::SchedulerThread()
 				{
 					MaxProjectStoreDuration = TriggerParams.MaxProjectStoreDuration;
 				}
+				if (TriggerParams.MaxBuildStoreDuration != std::chrono::seconds::max())
+				{
+					MaxBuildStoreDuration = TriggerParams.MaxBuildStoreDuration;
+				}
 				if (TriggerParams.DiskSizeSoftLimit != 0)
 				{
 					DiskSizeSoftLimit = TriggerParams.DiskSizeSoftLimit;
@@ -2046,6 +2250,8 @@ GcScheduler::SchedulerThread()
 				MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxCacheDuration;
 			GcClock::TimePoint ProjectStoreExpireTime =
 				MaxProjectStoreDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxProjectStoreDuration;
+			GcClock::TimePoint BuildStoreExpireTime =
+				MaxBuildStoreDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxBuildStoreDuration;
 
 			const GcStorageSize TotalSize = m_GcManager.TotalStorageSize();
 
@@ -2087,12 +2293,32 @@ GcScheduler::SchedulerThread()
 					}
 				}
 
-				uint64_t GcDiskSpaceGoal = 0;
+				uint64_t MaximumDiskUseGcSpaceGoal	= 0;
+				uint64_t MinimumFreeDiskGcSpaceGoal = 0;
+
 				if (DiskSizeSoftLimit != 0 && TotalSize.DiskSize > DiskSizeSoftLimit)
 				{
-					GcDiskSpaceGoal = TotalSize.DiskSize - DiskSizeSoftLimit;
+					MaximumDiskUseGcSpaceGoal	  = TotalSize.DiskSize - DiskSizeSoftLimit;
+					HighDiskSpaceUsageGCTriggered = true;
+				}
+
+				if (MinimumFreeDiskSpaceToAllowWrites != 0 && Space.Free < MinimumFreeDiskSpaceToAllowWrites)
+				{
+					MinimumFreeDiskGcSpaceGoal = MinimumFreeDiskSpaceToAllowWrites - Space.Free;
+					if (MinimumFreeDiskGcSpaceGoal > MaximumDiskUseGcSpaceGoal)
+					{
+						LowDiskSpaceGCTriggered = true;
+						EnableValidation		= false;
+					}
+				}
+
+				if (MaximumDiskUseGcSpaceGoal > 0 || MinimumFreeDiskGcSpaceGoal > 0)
+				{
+					const uint64_t GcDiskSpaceRemoveGoal = Max(MaximumDiskUseGcSpaceGoal, MinimumFreeDiskGcSpaceGoal);
+
 					std::unique_lock Lock(m_GcMutex);
-					GcClock::Tick	 AgeTick = m_DiskUsageWindow.FindTimepointThatRemoves(GcDiskSpaceGoal, Now.time_since_epoch().count());
+					GcClock::Tick	 AgeTick =
+						m_DiskUsageWindow.FindTimepointThatRemoves(GcDiskSpaceRemoveGoal, Now.time_since_epoch().count());
 					GcClock::TimePoint SizeBasedExpireTime = GcClock::TimePointFromTick(AgeTick);
 					if (SizeBasedExpireTime > CacheExpireTime)
 					{
@@ -2102,6 +2328,10 @@ GcScheduler::SchedulerThread()
 					{
 						ProjectStoreExpireTime = SizeBasedExpireTime;
 					}
+					if (SizeBasedExpireTime > BuildStoreExpireTime)
+					{
+						BuildStoreExpireTime = SizeBasedExpireTime;
+					}
 				}
 
 				std::chrono::seconds RemainingTimeUntilGc =
@@ -2130,29 +2360,33 @@ GcScheduler::SchedulerThread()
 					RemainingTimeUntilLightweightGc = RemainingTimeUntilGc;
 				}
 
-				if (GcDiskSpaceGoal > 0)
-				{
-					DiskSpaceGCTriggered = true;
-				}
-				else if (RemainingTimeUntilGc.count() == 0)
-				{
-					TimeBasedGCTriggered = true;
-				}
-				else if (RemainingTimeUntilLightweightGc.count() == 0)
+				if (MaximumDiskUseGcSpaceGoal == 0 && MinimumFreeDiskGcSpaceGoal == 0)
 				{
-					TimeBasedGCTriggered = true;
-					SkipCid				 = true;
+					if (RemainingTimeUntilGc.count() == 0)
+					{
+						TimeBasedGCTriggered = true;
+					}
+					else if (RemainingTimeUntilLightweightGc.count() == 0)
+					{
+						TimeBasedGCTriggered = true;
+						SkipCid				 = true;
+					}
 				}
 
 				std::string NextTriggerStatus;
-				if (GcInterval.count() != 0 || LightweightGcInterval.count() != 0 || DiskSizeSoftLimit != 0)
 				{
 					ExtendableStringBuilder<256> Sb;
-					if (DiskSpaceGCTriggered)
+					if (LowDiskSpaceGCTriggered)
+					{
+						Sb.Append(fmt::format(" Free disk space is below {}, trying to reclaim {}.",
+											  NiceBytes(MinimumFreeDiskSpaceToAllowWrites),
+											  NiceBytes(MinimumFreeDiskGcSpaceGoal)));
+					}
+					else if (HighDiskSpaceUsageGCTriggered)
 					{
 						Sb.Append(fmt::format(" Disk space exceeds {}, trying to reclaim {}.",
 											  NiceBytes(DiskSizeSoftLimit),
-											  NiceBytes(GcDiskSpaceGoal)));
+											  NiceBytes(MaximumDiskUseGcSpaceGoal)));
 					}
 					else if (TimeBasedGCTriggered)
 					{
@@ -2182,6 +2416,10 @@ GcScheduler::SchedulerThread()
 						{
 							Sb.Append(fmt::format(" Disk usage GC in {}.", NiceBytes(DiskSizeSoftLimit - TotalSize.DiskSize)));
 						}
+						else if (MinimumFreeDiskSpaceToAllowWrites != 0 && Space.Free > MinimumFreeDiskSpaceToAllowWrites)
+						{
+							Sb.Append(fmt::format(" Disk usage GC in {}.", NiceBytes(Space.Free - MinimumFreeDiskSpaceToAllowWrites)));
+						}
 					}
 					NextTriggerStatus = Sb;
 				}
@@ -2198,7 +2436,7 @@ GcScheduler::SchedulerThread()
 						 NiceBytes(MaxLoad / uint64_t(std::chrono::seconds(m_Config.MonitorInterval).count())),
 						 NextTriggerStatus);
 
-				if (!DiskSpaceGCTriggered && !TimeBasedGCTriggered)
+				if (!HighDiskSpaceUsageGCTriggered && !LowDiskSpaceGCTriggered && !TimeBasedGCTriggered)
 				{
 					WaitTime = m_Config.MonitorInterval;
 					if (RemainingTimeUntilGc < WaitTime)
@@ -2227,6 +2465,7 @@ GcScheduler::SchedulerThread()
 
 			bool GcSuccess = CollectGarbage(CacheExpireTime,
 											ProjectStoreExpireTime,
+											BuildStoreExpireTime,
 											DoDelete,
 											CollectSmallObjects,
 											SkipCid,
@@ -2333,6 +2572,7 @@ GcScheduler::ScrubStorage(bool DoDelete, bool SkipCid, std::chrono::seconds Time
 bool
 GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 							const GcClock::TimePoint& ProjectStoreExpireTime,
+							const GcClock::TimePoint& BuildStoreExpireTime,
 							bool					  Delete,
 							bool					  CollectSmallObjects,
 							bool					  SkipCid,
@@ -2375,12 +2615,12 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 		{
 			// We are low on disk, check if we can release our extra storage reserve, if we can't bail from doing GC
 			auto ClaimDiskReserve = [&]() -> uint64_t {
-				if (!std::filesystem::is_regular_file(DiskReservePath))
+				if (!IsFile(DiskReservePath))
 				{
 					return 0;
 				}
-				uint64_t ReclaimedSize = std::filesystem::file_size(DiskReservePath);
-				if (std::filesystem::remove(DiskReservePath))
+				uint64_t ReclaimedSize = FileSizeFromPath(DiskReservePath);
+				if (RemoveFile(DiskReservePath))
 				{
 					return ReclaimedSize;
 				}
@@ -2416,6 +2656,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 
 						const GcSettings Settings = {.CacheExpireTime					= CacheExpireTime,
 													 .ProjectStoreExpireTime			= ProjectStoreExpireTime,
+													 .BuildStoreExpireTime				= BuildStoreExpireTime,
 													 .CollectSmallObjects				= CollectSmallObjects,
 													 .IsDeleteMode						= Delete,
 													 .SkipCidDelete						= SkipCid,
@@ -2447,6 +2688,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 							}
 							SB.Append(fmt::format("      Cache cutoff time:           {}\n", Settings.CacheExpireTime));
 							SB.Append(fmt::format("      Project store cutoff time:   {}\n", Settings.ProjectStoreExpireTime));
+							SB.Append(fmt::format("      Build store cutoff time:     {}\n", Settings.BuildStoreExpireTime));
 						};
 
 						{
@@ -2522,7 +2764,11 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 
 						ZEN_INFO("GCV2: {}", SB.ToView());
 
-						AppendGCLog(GcId, GcStartTime, Settings, Result);
+						CheckDiskSpace();
+						if (!m_AreDiskWritesBlocked.load())
+						{
+							AppendGCLog(GcId, GcStartTime, Settings, Result);
+						}
 
 						if (SkipCid)
 						{
@@ -2552,6 +2798,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 				if (Delete)
 				{
 					GcClock::TimePoint KeepRangeStart = Min(CacheExpireTime, ProjectStoreExpireTime);
+					KeepRangeStart					  = Min(KeepRangeStart, BuildStoreExpireTime);
 					m_LastGcExpireTime				  = KeepRangeStart;
 					std::unique_lock Lock(m_GcMutex);
 					m_DiskUsageWindow.KeepRange(KeepRangeStart.time_since_epoch().count(), GcClock::Duration::max().count());
@@ -2563,65 +2810,69 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 				m_LastFullGCDiff		= Diff;
 			}
 
-			for (uint32_t RetryCount = 0; RetryCount < 3; RetryCount++)
+			CheckDiskSpace();
+			if (!m_AreDiskWritesBlocked.load())
 			{
-				if (RetryCount > 0)
+				for (uint32_t RetryCount = 0; RetryCount < 3; RetryCount++)
 				{
-					ZEN_INFO("Writing GC state failed {} time(s), pausing and trying again", RetryCount);
-					Sleep(250);
-				}
-				try
-				{
-					const fs::path Path = m_Config.RootDirectory / "gc_state";
-					ZEN_DEBUG("saving scheduler state to '{}'", Path);
-					CbObjectWriter SchedulerState;
-					SchedulerState << "LastGcTime"sv << static_cast<int64_t>(m_LastGcTime.time_since_epoch().count());
-					SchedulerState << "LastGcExpireTime"sv << static_cast<int64_t>(m_LastGcExpireTime.time_since_epoch().count());
-					SchedulerState << "AttachmentPassIndex"sv << m_AttachmentPassIndex;
-
-					SaveCompactBinaryObject(Path, SchedulerState.Save());
 					if (RetryCount > 0)
 					{
-						ZEN_INFO("Writing GC state succeeded after {} attempts", RetryCount + 1);
-					}
-					break;
-				}
-				catch (const std::system_error& SystemError)
-				{
-					if (IsOOM(SystemError.code()))
-					{
-						ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", SystemError.what());
+						ZEN_INFO("Writing GC state failed {} time(s), pausing and trying again", RetryCount);
+						Sleep(250);
 					}
-					else if (IsOOD(SystemError.code()))
-					{
-						ZEN_WARN("writing gc scheduler state ran out of disk space: '{}'", SystemError.what());
-					}
-					if (RetryCount == 0)
+					try
 					{
-						ZEN_ERROR("writing gc scheduler state failed with system error exception: '{}' ({})",
-								  SystemError.what(),
-								  SystemError.code().value());
+						const fs::path Path = m_Config.RootDirectory / "gc_state";
+						ZEN_DEBUG("saving scheduler state to '{}'", Path);
+						CbObjectWriter SchedulerState;
+						SchedulerState << "LastGcTime"sv << static_cast<int64_t>(m_LastGcTime.time_since_epoch().count());
+						SchedulerState << "LastGcExpireTime"sv << static_cast<int64_t>(m_LastGcExpireTime.time_since_epoch().count());
+						SchedulerState << "AttachmentPassIndex"sv << m_AttachmentPassIndex;
+
+						SaveCompactBinaryObject(Path, SchedulerState.Save());
+						if (RetryCount > 0)
+						{
+							ZEN_INFO("Writing GC state succeeded after {} attempts", RetryCount + 1);
+						}
+						break;
 					}
-					else
+					catch (const std::system_error& SystemError)
 					{
-						ZEN_WARN("writing gc scheduler state failed with system error exception: '{}' ({})",
-								 SystemError.what(),
-								 SystemError.code().value());
+						if (IsOOM(SystemError.code()))
+						{
+							ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", SystemError.what());
+						}
+						else if (IsOOD(SystemError.code()))
+						{
+							ZEN_WARN("writing gc scheduler state ran out of disk space: '{}'", SystemError.what());
+						}
+						else if (RetryCount == 0)
+						{
+							ZEN_ERROR("writing gc scheduler state failed with system error exception: '{}' ({})",
+									  SystemError.what(),
+									  SystemError.code().value());
+						}
+						else
+						{
+							ZEN_WARN("writing gc scheduler state failed with system error exception: '{}' ({})",
+									 SystemError.what(),
+									 SystemError.code().value());
+						}
 					}
-				}
-				catch (const std::bad_alloc& BadAlloc)
-				{
-					ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", BadAlloc.what());
-				}
-				catch (const std::exception& Ex)
-				{
-					if (RetryCount == 0)
+					catch (const std::bad_alloc& BadAlloc)
 					{
-						ZEN_ERROR("writing gc scheduler state failed with: '{}'", Ex.what());
+						ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", BadAlloc.what());
 					}
-					else
+					catch (const std::exception& Ex)
 					{
-						ZEN_WARN("writing gc scheduler state failed with: '{}'", Ex.what());
+						if (RetryCount == 0)
+						{
+							ZEN_ERROR("writing gc scheduler state failed with: '{}'", Ex.what());
+						}
+						else
+						{
+							ZEN_WARN("writing gc scheduler state failed with: '{}'", Ex.what());
+						}
 					}
 				}
 			}