frequent disk space check (#407)

* check low disk space condition more frequently and trigger GC when low water mark is reached
* show waited time when waiting for zenserver instance to exit

2 files changed, 89 insertions, 26 deletions

diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index f6b3dca6f..a15a2e084 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -2068,17 +2068,46 @@ GcScheduler::SchedulerThread()
 	ZEN_MEMSCOPE(GetGcTag());
 	SetCurrentThreadName("GcScheduler");
 
-	std::chrono::seconds WaitTime{0};
-
-	bool SilenceErrors = false;
+	std::chrono::seconds	   WaitTime{0};
+	const std::chrono::seconds ShortWaitTime{5};
+	bool					   SilenceErrors = false;
 
 	for (;;)
 	{
-		bool Timeout = false;
+		(void)CheckDiskSpace();
+
+		std::chrono::seconds WaitedTime{0};
+		bool				 Timeout = false;
 		{
 			ZEN_ASSERT(WaitTime.count() >= 0);
 			std::unique_lock Lock(m_GcMutex);
-			Timeout = std::cv_status::timeout == m_GcSignal.wait_for(Lock, WaitTime);
+			while (!Timeout)
+			{
+				std::chrono::seconds ShortWait	  = Min(WaitTime, ShortWaitTime);
+				bool				 ShortTimeout = std::cv_status::timeout == m_GcSignal.wait_for(Lock, ShortWait);
+				if (ShortTimeout)
+				{
+					if (WaitTime > ShortWaitTime)
+					{
+						DiskSpace Space = CheckDiskSpace();
+						if (!AreDiskWritesAllowed())
+						{
+							ZEN_INFO("Triggering GC due to low disk space ({}) on {}", NiceBytes(Space.Free), m_Config.RootDirectory);
+							Timeout = true;
+						}
+						WaitTime -= ShortWaitTime;
+					}
+					else
+					{
+						Timeout = true;
+					}
+				}
+				else
+				{
+					// We got a signal
+					break;
+				}
+			}
 		}
 
 		if (Status() == GcSchedulerStatus::kStopped)
@@ -2110,6 +2139,7 @@ GcScheduler::SchedulerThread()
 			std::chrono::seconds MaxProjectStoreDuration		   = m_Config.MaxProjectStoreDuration;
 			std::chrono::seconds MaxBuildStoreDuration			   = m_Config.MaxBuildStoreDuration;
 			uint64_t			 DiskSizeSoftLimit				   = m_Config.DiskSizeSoftLimit;
+			uint64_t			 MinimumFreeDiskSpaceToAllowWrites = m_Config.MinimumFreeDiskSpaceToAllowWrites;
 			bool				 SkipCid						   = false;
 			GcVersion			 UseGCVersion					   = m_Config.UseGCVersion;
 			uint32_t			 CompactBlockUsageThresholdPercent = m_Config.CompactBlockUsageThresholdPercent;
@@ -2124,8 +2154,9 @@ GcScheduler::SchedulerThread()
 			uint8_t NextAttachmentPassIndex =
 				ComputeAttachmentRange(m_AttachmentPassIndex, m_Config.AttachmentPassCount, AttachmentRangeMin, AttachmentRangeMax);
 
-			bool DiskSpaceGCTriggered = false;
-			bool TimeBasedGCTriggered = false;
+			bool LowDiskSpaceGCTriggered	   = false;
+			bool HighDiskSpaceUsageGCTriggered = false;
+			bool TimeBasedGCTriggered		   = false;
 
 			GcClock::TimePoint Now = GcClock::Now();
 
@@ -2262,12 +2293,32 @@ GcScheduler::SchedulerThread()
 					}
 				}
 
-				uint64_t GcDiskSpaceGoal = 0;
+				uint64_t MaximumDiskUseGcSpaceGoal	= 0;
+				uint64_t MinimumFreeDiskGcSpaceGoal = 0;
+
 				if (DiskSizeSoftLimit != 0 && TotalSize.DiskSize > DiskSizeSoftLimit)
 				{
-					GcDiskSpaceGoal = TotalSize.DiskSize - DiskSizeSoftLimit;
+					MaximumDiskUseGcSpaceGoal	  = TotalSize.DiskSize - DiskSizeSoftLimit;
+					HighDiskSpaceUsageGCTriggered = true;
+				}
+
+				if (MinimumFreeDiskSpaceToAllowWrites != 0 && Space.Free < MinimumFreeDiskSpaceToAllowWrites)
+				{
+					MinimumFreeDiskGcSpaceGoal = MinimumFreeDiskSpaceToAllowWrites - Space.Free;
+					if (MinimumFreeDiskGcSpaceGoal > MaximumDiskUseGcSpaceGoal)
+					{
+						LowDiskSpaceGCTriggered = true;
+						EnableValidation		= false;
+					}
+				}
+
+				if (MaximumDiskUseGcSpaceGoal > 0 || MinimumFreeDiskGcSpaceGoal > 0)
+				{
+					const uint64_t GcDiskSpaceRemoveGoal = Max(MaximumDiskUseGcSpaceGoal, MinimumFreeDiskGcSpaceGoal);
+
 					std::unique_lock Lock(m_GcMutex);
-					GcClock::Tick	 AgeTick = m_DiskUsageWindow.FindTimepointThatRemoves(GcDiskSpaceGoal, Now.time_since_epoch().count());
+					GcClock::Tick	 AgeTick =
+						m_DiskUsageWindow.FindTimepointThatRemoves(GcDiskSpaceRemoveGoal, Now.time_since_epoch().count());
 					GcClock::TimePoint SizeBasedExpireTime = GcClock::TimePointFromTick(AgeTick);
 					if (SizeBasedExpireTime > CacheExpireTime)
 					{
@@ -2309,29 +2360,33 @@ GcScheduler::SchedulerThread()
 					RemainingTimeUntilLightweightGc = RemainingTimeUntilGc;
 				}
 
-				if (GcDiskSpaceGoal > 0)
+				if (MaximumDiskUseGcSpaceGoal == 0 && MinimumFreeDiskGcSpaceGoal == 0)
 				{
-					DiskSpaceGCTriggered = true;
-				}
-				else if (RemainingTimeUntilGc.count() == 0)
-				{
-					TimeBasedGCTriggered = true;
-				}
-				else if (RemainingTimeUntilLightweightGc.count() == 0)
-				{
-					TimeBasedGCTriggered = true;
-					SkipCid				 = true;
+					if (RemainingTimeUntilGc.count() == 0)
+					{
+						TimeBasedGCTriggered = true;
+					}
+					else if (RemainingTimeUntilLightweightGc.count() == 0)
+					{
+						TimeBasedGCTriggered = true;
+						SkipCid				 = true;
+					}
 				}
 
 				std::string NextTriggerStatus;
-				if (GcInterval.count() != 0 || LightweightGcInterval.count() != 0 || DiskSizeSoftLimit != 0)
 				{
 					ExtendableStringBuilder<256> Sb;
-					if (DiskSpaceGCTriggered)
+					if (LowDiskSpaceGCTriggered)
+					{
+						Sb.Append(fmt::format(" Free disk space is below {}, trying to reclaim {}.",
+											  NiceBytes(MinimumFreeDiskSpaceToAllowWrites),
+											  NiceBytes(MinimumFreeDiskGcSpaceGoal)));
+					}
+					else if (HighDiskSpaceUsageGCTriggered)
 					{
 						Sb.Append(fmt::format(" Disk space exceeds {}, trying to reclaim {}.",
 											  NiceBytes(DiskSizeSoftLimit),
-											  NiceBytes(GcDiskSpaceGoal)));
+											  NiceBytes(MaximumDiskUseGcSpaceGoal)));
 					}
 					else if (TimeBasedGCTriggered)
 					{
@@ -2361,6 +2416,10 @@ GcScheduler::SchedulerThread()
 						{
 							Sb.Append(fmt::format(" Disk usage GC in {}.", NiceBytes(DiskSizeSoftLimit - TotalSize.DiskSize)));
 						}
+						else if (MinimumFreeDiskSpaceToAllowWrites != 0 && Space.Free > MinimumFreeDiskSpaceToAllowWrites)
+						{
+							Sb.Append(fmt::format(" Disk usage GC in {}.", NiceBytes(Space.Free - MinimumFreeDiskSpaceToAllowWrites)));
+						}
 					}
 					NextTriggerStatus = Sb;
 				}
@@ -2377,7 +2436,7 @@ GcScheduler::SchedulerThread()
 						 NiceBytes(MaxLoad / uint64_t(std::chrono::seconds(m_Config.MonitorInterval).count())),
 						 NextTriggerStatus);
 
-				if (!DiskSpaceGCTriggered && !TimeBasedGCTriggered)
+				if (!HighDiskSpaceUsageGCTriggered && !LowDiskSpaceGCTriggered && !TimeBasedGCTriggered)
 				{
 					WaitTime = m_Config.MonitorInterval;
 					if (RemainingTimeUntilGc < WaitTime)
diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp
index c0c2a754a..bfa0d3c49 100644
--- a/src/zenutil/zenserverprocess.cpp
+++ b/src/zenutil/zenserverprocess.cpp
@@ -619,6 +619,7 @@ ZenServerInstance::Shutdown()
 				std::error_code Ec;
 				if (SignalShutdown(Ec))
 				{
+					Stopwatch Timer;
 					ZEN_DEBUG("Waiting for zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
 					while (!m_Process.Wait(1000))
 					{
@@ -632,7 +633,10 @@ ZenServerInstance::Shutdown()
 							ZEN_WARN("Wait abandoned by exited process");
 							return 0;
 						}
-						ZEN_WARN("Waiting for zenserver process {} ({}) timed out", m_Name, m_Process.Pid());
+						ZEN_WARN("Waited for zenserver process {} ({}) to exit for {}",
+								 m_Name,
+								 m_Process.Pid(),
+								 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 					}
 					ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
 					int ExitCode = m_Process.GetExitCode();