Low disk space detector (#277)

* - Feature: Disk writes are now blocked early and return an insufficient storage error if free disk space falls below the `--low-diskspace-threshold` value
* Never keep an entry in m_ChunkBlocks that points to a nullptr

1 files changed, 45 insertions, 3 deletions

diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 370c3c965..f9888722b 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -559,10 +559,12 @@ DiskUsageWindow::FindTimepointThatRemoves(uint64_t Amount, GcClock::Tick EndTick
 
 GcScheduler::GcScheduler(GcManager& GcManager) : m_Log(logging::Get("gc")), m_GcManager(GcManager)
 {
+	m_GcManager.SetDiskWriteBlocker(this);
 }
 
 GcScheduler::~GcScheduler()
 {
+	m_GcManager.SetDiskWriteBlocker(nullptr);
 	Shutdown();
 }
 
@@ -573,6 +575,18 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
 
 	m_Config = Config;
 
+	std::error_code Ec;
+	DiskSpace		Space = DiskSpaceInfo(m_Config.RootDirectory, Ec);
+	if (Ec)
+	{
+		m_AreDiskWritesBlocked.store(true);
+		ZEN_WARN("get disk space info FAILED, blocking disk writes, reason: '{}'", Ec.message());
+	}
+	else
+	{
+		CheckDiskSpace(Space);
+	}
+
 	if (m_Config.Interval.count() && m_Config.Interval < m_Config.MonitorInterval)
 	{
 		m_Config.Interval = m_Config.MonitorInterval;
@@ -580,7 +594,7 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
 
 	std::filesystem::create_directories(Config.RootDirectory);
 
-	std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
+	Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
 	if (Ec)
 	{
 		ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason '{}'",
@@ -664,6 +678,29 @@ GcScheduler::Trigger(const GcScheduler::TriggerParams& Params)
 }
 
 void
+GcScheduler::CheckDiskSpace(const DiskSpace& Space)
+{
+	bool AreDiskWritesBlocked = m_AreDiskWritesBlocked;
+	bool IsLowOnDiskSpace = (m_Config.MinimumFreeDiskSpaceToAllowWrites) != 0 && (Space.Free < m_Config.MinimumFreeDiskSpaceToAllowWrites);
+	if (IsLowOnDiskSpace != AreDiskWritesBlocked)
+	{
+		m_AreDiskWritesBlocked.store(IsLowOnDiskSpace);
+		if (IsLowOnDiskSpace)
+		{
+			ZEN_WARN("Writing to disk is blocked, free disk space: {}, minimum required {}",
+					 NiceBytes(Space.Free),
+					 NiceBytes(m_Config.MinimumFreeDiskSpaceToAllowWrites));
+		}
+		else
+		{
+			ZEN_INFO("Writing to disk is unblocked, free disk space: {}, minimum required {}",
+					 NiceBytes(Space.Free),
+					 NiceBytes(m_Config.MinimumFreeDiskSpaceToAllowWrites));
+		}
+	}
+}
+
+void
 GcScheduler::SchedulerThread()
 {
 	std::chrono::seconds WaitTime{0};
@@ -716,16 +753,21 @@ GcScheduler::SchedulerThread()
 
 		GcClock::TimePoint ExpireTime = MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxCacheDuration;
 
-		std::error_code		Ec;
 		const GcStorageSize TotalSize = m_GcManager.TotalStorageSize();
 
 		if (Timeout && Status() == GcSchedulerStatus::kIdle)
 		{
-			DiskSpace Space = DiskSpaceInfo(m_Config.RootDirectory, Ec);
+			std::error_code Ec;
+			DiskSpace		Space = DiskSpaceInfo(m_Config.RootDirectory, Ec);
 			if (Ec)
 			{
+				m_AreDiskWritesBlocked.store(true);
 				ZEN_WARN("get disk space info FAILED, reason: '{}'", Ec.message());
 			}
+			else
+			{
+				CheckDiskSpace(Space);
+			}
 
 			const int64_t				PressureGraphLength = 30;
 			const std::chrono::duration LoadGraphTime		= PressureGraphLength * m_Config.MonitorInterval;