safer gc on low disk (#373)

* - Improvement: Make sure we have disk space available to do GC and use reserve up front if need be

2 files changed, 46 insertions, 37 deletions

diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 2a599629a..245f76c92 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -590,19 +590,7 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
 
 	std::filesystem::create_directories(Config.RootDirectory);
 
-	std::error_code Ec;
-	DiskSpace		Space = DiskSpaceInfo(m_Config.RootDirectory, Ec);
-	if (Ec)
-	{
-		m_AreDiskWritesBlocked.store(true);
-		ZEN_WARN("get disk space info FAILED, blocking disk writes, reason: '{}'", Ec.message());
-	}
-	else
-	{
-		CheckDiskSpace(Space);
-	}
-
-	Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
+	std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
 	if (Ec)
 	{
 		ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason '{}'",
@@ -611,6 +599,8 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
 				 Ec.message());
 	}
 
+	CheckDiskSpace();
+
 	m_LastGcTime	   = GcClock::Now();
 	m_LastGcExpireTime = GcClock::TimePoint::min();
 
@@ -708,9 +698,18 @@ GcScheduler::TriggerScrub(const TriggerScrubParams& Params)
 	return false;
 }
 
-void
-GcScheduler::CheckDiskSpace(const DiskSpace& Space)
+DiskSpace
+GcScheduler::CheckDiskSpace()
 {
+	std::error_code Ec;
+	DiskSpace		Space = DiskSpaceInfo(m_Config.RootDirectory, Ec);
+	if (Ec)
+	{
+		m_AreDiskWritesBlocked.store(true);
+		ZEN_WARN("get disk space info for path '{}' FAILED, reason: '{}'", m_Config.RootDirectory, Ec.message());
+		return {0, 0};
+	}
+
 	bool AreDiskWritesBlocked = m_AreDiskWritesBlocked;
 	bool IsLowOnDiskSpace = (m_Config.MinimumFreeDiskSpaceToAllowWrites) != 0 && (Space.Free < m_Config.MinimumFreeDiskSpaceToAllowWrites);
 	if (IsLowOnDiskSpace != AreDiskWritesBlocked)
@@ -729,6 +728,7 @@ GcScheduler::CheckDiskSpace(const DiskSpace& Space)
 					 NiceBytes(m_Config.MinimumFreeDiskSpaceToAllowWrites));
 		}
 	}
+	return Space;
 }
 
 void
@@ -828,17 +828,7 @@ GcScheduler::SchedulerThread()
 
 			if (Timeout && Status() == GcSchedulerStatus::kIdle)
 			{
-				std::error_code Ec;
-				DiskSpace		Space = DiskSpaceInfo(m_Config.RootDirectory, Ec);
-				if (Ec)
-				{
-					m_AreDiskWritesBlocked.store(true);
-					ZEN_WARN("get disk space info FAILED, reason: '{}'", Ec.message());
-				}
-				else
-				{
-					CheckDiskSpace(Space);
-				}
+				DiskSpace Space = CheckDiskSpace();
 
 				const int64_t				PressureGraphLength = 30;
 				const std::chrono::duration LoadGraphTime		= PressureGraphLength * m_Config.MonitorInterval;
@@ -1001,6 +991,34 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 	GcCtx.CollectSmallObjects(CollectSmallObjects);
 	GcCtx.DiskReservePath(m_Config.RootDirectory / "reserve.gc");
 
+	auto ReclaimDiskReserve = [&]() {
+		std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
+		if (Ec)
+		{
+			ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason: '{}'",
+					 m_Config.RootDirectory / "reserve.gc",
+					 NiceBytes(m_Config.DiskReserveSize),
+					 Ec.message());
+		}
+	};
+
+	ReclaimDiskReserve();
+	const auto _ = MakeGuard([&] { ReclaimDiskReserve(); });
+
+	CheckDiskSpace();
+
+	if (m_AreDiskWritesBlocked.load())
+	{
+		// We are low on disk, check if we can release our extra storage reserve, if we can't bail from doing GC
+		uint64_t ReleasedSpace = GcCtx.ClaimGCReserve();
+		if (ReleasedSpace == 0)
+		{
+			ZEN_WARN("Disk space is very low and we have no GC reserve, skipping GC as this requires at least some space to write to '{}'",
+					 m_Config.RootDirectory);
+			return;
+		}
+	}
+
 	ZEN_INFO("garbage collection STARTING, small objects gc {}, cache cutoff time {}, project store cutoff time {}",
 			 GcCtx.CollectSmallObjects() ? "ENABLED"sv : "DISABLED"sv,
 			 CacheExpireTime,
@@ -1033,16 +1051,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 		}
 		catch (std::exception& Ex)
 		{
-			ZEN_ERROR("writing gc scheduler state failed with: '{}'", Ex.what());
-		}
-
-		std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
-		if (Ec)
-		{
-			ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason: '{}'",
-					 m_Config.RootDirectory / "reserve.gc",
-					 NiceBytes(m_Config.DiskReserveSize),
-					 Ec.message());
+			ZEN_WARN("writing gc scheduler state failed with: '{}'", Ex.what());
 		}
 	}
 }
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index 22b9bc284..28e5c2ec2 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -246,7 +246,7 @@ private:
 	GcClock::TimePoint NextGcTime(GcClock::TimePoint CurrentTime);
 	spdlog::logger&	   Log() { return m_Log; }
 	virtual bool	   AreDiskWritesAllowed() const override { return !m_AreDiskWritesBlocked.load(); }
-	void			   CheckDiskSpace(const DiskSpace& Space);
+	DiskSpace		   CheckDiskSpace();
 
 	spdlog::logger&					  m_Log;
 	GcManager&						  m_GcManager;