automatic scrub on startup (#667)

- Improvement: Deeper validation of data when scrub is activated (cas/cache/project)
- Improvement: Enabled more multi threading when running scrub operations
- Improvement: Added means to force a scrub operation at startup with a new release using ZEN_DATA_FORCE_SCRUB_VERSION variable in xmake.lua

1 files changed, 133 insertions, 108 deletions

diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index ba1bce974..a4a141577 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -1810,6 +1810,8 @@ GcScheduler::Shutdown()
 	ZEN_TRACE_CPU("GcScheduler::Shutdown");
 	ZEN_MEMSCOPE(GetGcTag());
 
+	m_GcManager.SetCancelGC(true);
+
 	if (static_cast<uint32_t>(GcSchedulerStatus::kStopped) != m_Status)
 	{
 		bool GcIsRunning = m_Status == static_cast<uint32_t>(GcSchedulerStatus::kRunning);
@@ -1817,18 +1819,18 @@ GcScheduler::Shutdown()
 		{
 			ZEN_INFO("Requesting cancel running garbage collection");
 		}
-		m_GcManager.SetCancelGC(true);
 		m_Status = static_cast<uint32_t>(GcSchedulerStatus::kStopped);
-		m_GcSignal.notify_one();
+		m_GcSignal.Set();
+	}
 
-		if (m_GcThread.joinable())
+	if (m_GcThread.joinable())
+	{
+		bool GcIsRunning = m_Status == static_cast<uint32_t>(GcSchedulerStatus::kRunning);
+		if (GcIsRunning)
 		{
-			if (GcIsRunning)
-			{
-				ZEN_INFO("Waiting for garbage collection to complete");
-			}
-			m_GcThread.join();
+			ZEN_INFO("Waiting for garbage collection to complete");
 		}
+		m_GcThread.join();
 	}
 	m_DiskUsageLog.Flush();
 	m_DiskUsageLog.Close();
@@ -1839,17 +1841,17 @@ GcScheduler::TriggerGc(const GcScheduler::TriggerGcParams& Params)
 {
 	ZEN_MEMSCOPE(GetGcTag());
 	std::unique_lock Lock(m_GcMutex);
-	if (static_cast<uint32_t>(GcSchedulerStatus::kIdle) == m_Status)
+
+	if (m_TriggerGcParams || m_TriggerScrubParams)
 	{
-		m_TriggerGcParams  = Params;
-		uint32_t IdleState = static_cast<uint32_t>(GcSchedulerStatus::kIdle);
+		return false;
+	}
 
-		if (m_Status.compare_exchange_strong(/* expected */ IdleState,
-											 /* desired */ static_cast<uint32_t>(GcSchedulerStatus::kRunning)))
-		{
-			m_GcSignal.notify_one();
-			return true;
-		}
+	if (static_cast<uint32_t>(GcSchedulerStatus::kStopped) != m_Status)
+	{
+		m_TriggerGcParams = Params;
+		m_GcSignal.Set();
+		return true;
 	}
 	return false;
 }
@@ -1860,17 +1862,16 @@ GcScheduler::TriggerScrub(const TriggerScrubParams& Params)
 	ZEN_MEMSCOPE(GetGcTag());
 	std::unique_lock Lock(m_GcMutex);
 
-	if (static_cast<uint32_t>(GcSchedulerStatus::kIdle) == m_Status)
+	if (m_TriggerGcParams || m_TriggerScrubParams)
 	{
-		m_TriggerScrubParams = Params;
-		uint32_t IdleState	 = static_cast<uint32_t>(GcSchedulerStatus::kIdle);
-
-		if (m_Status.compare_exchange_strong(/* expected */ IdleState, /* desired */ static_cast<uint32_t>(GcSchedulerStatus::kRunning)))
-		{
-			m_GcSignal.notify_one();
+		return false;
+	}
 
-			return true;
-		}
+	if (static_cast<uint32_t>(GcSchedulerStatus::kStopped) != m_Status)
+	{
+		m_TriggerScrubParams = Params;
+		m_GcSignal.Set();
+		return true;
 	}
 
 	return false;
@@ -1977,8 +1978,6 @@ GcScheduler::AppendGCLog(std::string_view Id, GcClock::TimePoint StartTime, cons
 
 		MemoryView EntryBuffer(Blob.data(), Blob.size());
 		{
-			RwLock::ExclusiveLockScope _(m_GcLogLock);
-
 			GcLogFile.Open(Path, BasicFile::Mode::kWrite);
 			uint64_t AppendPos = GcLogFile.FileSize();
 
@@ -2096,13 +2095,25 @@ GcScheduler::GetState() const
 	return Result;
 }
 
+bool
+GcScheduler::IsManualTriggerPresent() const
+{
+	bool IsPending = Status() != GcSchedulerStatus::kStopped;
+	if (IsPending)
+	{
+		std::unique_lock Lock(m_GcMutex);
+		IsPending = m_TriggerGcParams || m_TriggerScrubParams;
+	}
+	return IsPending;
+}
+
 void
 GcScheduler::SchedulerThread()
 {
 	ZEN_MEMSCOPE(GetGcTag());
 	SetCurrentThreadName("GcScheduler");
 
-	std::chrono::seconds	   WaitTime{0};
+	std::chrono::seconds	   WaitTime{m_Config.Enabled ? std::chrono::seconds{0} : std::chrono::seconds::max()};
 	const std::chrono::seconds ShortWaitTime{5};
 	bool					   SilenceErrors = false;
 
@@ -2111,60 +2122,67 @@ GcScheduler::SchedulerThread()
 		(void)CheckDiskSpace();
 
 		std::chrono::seconds WaitedTime{0};
-		bool				 Timeout = false;
+
+		std::optional<TriggerGcParams>	  TriggerGcParams;
+		std::optional<TriggerScrubParams> TriggerScrubParams;
+
+		ZEN_ASSERT(WaitTime.count() >= 0);
+		while (Status() != GcSchedulerStatus::kStopped)
 		{
-			ZEN_ASSERT(WaitTime.count() >= 0);
-			std::unique_lock Lock(m_GcMutex);
-			while (!Timeout && (Status() != GcSchedulerStatus::kStopped))
-			{
-				std::chrono::seconds ShortWait	  = Min(WaitTime, ShortWaitTime);
-				bool				 ShortTimeout = std::cv_status::timeout == m_GcSignal.wait_for(Lock, ShortWait);
+			std::chrono::seconds ShortWait	  = Min(WaitTime, ShortWaitTime);
+			bool				 ShortTimeout = !m_GcSignal.Wait(gsl::narrow<int>(ShortWait.count() * 1000));
 
-				if (ShortTimeout)
+			if (ShortTimeout)
+			{
+				if (WaitTime > ShortWaitTime)
 				{
-					if (WaitTime > ShortWaitTime)
+					DiskSpace Space = CheckDiskSpace();
+					if (!AreDiskWritesAllowed())
 					{
-						DiskSpace Space = CheckDiskSpace();
-						if (!AreDiskWritesAllowed())
-						{
-							ZEN_INFO("Triggering GC due to low disk space ({}) on {}", NiceBytes(Space.Free), m_Config.RootDirectory);
-							Timeout = true;
-						}
-						WaitTime -= ShortWaitTime;
-					}
-					else
-					{
-						Timeout = true;
+						ZEN_INFO("Triggering GC due to low disk space ({}) on {}", NiceBytes(Space.Free), m_Config.RootDirectory);
+						break;
 					}
+					WaitTime -= ShortWaitTime;
 				}
 				else
 				{
-					// We got a signal
 					break;
 				}
 			}
+			else
+			{
+				m_GcSignal.Reset();
+				{
+					std::unique_lock Lock(m_GcMutex);
+					TriggerGcParams	   = m_TriggerGcParams;
+					TriggerScrubParams = m_TriggerScrubParams;
+				}
+				break;
+			}
 		}
 
+		auto TriggerCleanup = MakeGuard([&]() {
+			if (TriggerGcParams || TriggerScrubParams)
+			{
+				std::unique_lock Lock(m_GcMutex);
+				m_TriggerGcParams.reset();
+				m_TriggerScrubParams.reset();
+			}
+		});
+
 		if (Status() == GcSchedulerStatus::kStopped)
 		{
 			break;
 		}
 
-		if (!m_Config.Enabled && !m_TriggerScrubParams && !m_TriggerGcParams)
-		{
-			WaitTime = std::chrono::seconds::max();
-			continue;
-		}
-
-		if (!Timeout && Status() == GcSchedulerStatus::kIdle)
-		{
-			continue;
-		}
-
 		try
 		{
-			bool				 DoGc							   = m_Config.Enabled;
-			bool				 DoScrubbing					   = false;
+			bool ManualGcTriggered			   = false;
+			bool ManualScrubbingTriggered	   = false;
+			bool LowDiskSpaceGCTriggered	   = false;
+			bool HighDiskSpaceUsageGCTriggered = false;
+			bool TimeBasedGCTriggered		   = false;
+
 			std::chrono::seconds ScrubTimeslice					   = std::chrono::seconds::max();
 			bool				 DoDelete						   = true;
 			bool				 CollectSmallObjects			   = m_Config.CollectSmallObjects;
@@ -2189,16 +2207,33 @@ GcScheduler::SchedulerThread()
 			uint8_t NextAttachmentPassIndex =
 				ComputeAttachmentRange(m_AttachmentPassIndex, m_Config.AttachmentPassCount, AttachmentRangeMin, AttachmentRangeMax);
 
-			bool LowDiskSpaceGCTriggered	   = false;
-			bool HighDiskSpaceUsageGCTriggered = false;
-			bool TimeBasedGCTriggered		   = false;
-
 			GcClock::TimePoint Now = GcClock::Now();
 
-			if (m_TriggerGcParams)
+			if (TriggerScrubParams)
 			{
-				const auto TriggerParams = m_TriggerGcParams.value();
-				m_TriggerGcParams.reset();
+				ZEN_ASSERT_SLOW(!TriggerGcParams);
+				ZEN_INFO("Manual scrub triggered");
+				const auto TriggerParams = TriggerScrubParams.value();
+
+				ManualScrubbingTriggered = true;
+
+				if (!TriggerParams.SkipGc)
+				{
+					ManualGcTriggered = true;
+				}
+
+				if (TriggerParams.SkipCas)
+				{
+					SkipCid = true;
+				}
+
+				DoDelete	   = !TriggerParams.SkipDelete;
+				ScrubTimeslice = TriggerParams.MaxTimeslice;
+			}
+			else if (TriggerGcParams)
+			{
+				ZEN_INFO("Manual gc triggered");
+				const auto TriggerParams = TriggerGcParams.value();
 
 				CollectSmallObjects = TriggerParams.CollectSmallObjects;
 
@@ -2249,34 +2284,10 @@ GcScheduler::SchedulerThread()
 				{
 					EnableValidation = TriggerParams.EnableValidation.value();
 				}
-				DoGc = true;
-			}
-
-			if (m_TriggerScrubParams)
-			{
-				DoScrubbing = true;
-
-				if (m_TriggerScrubParams->SkipGc)
-				{
-					DoGc = false;
-				}
-
-				if (m_TriggerScrubParams->SkipCas)
-				{
-					SkipCid = true;
-				}
-
-				DoDelete	   = !m_TriggerScrubParams->SkipDelete;
-				ScrubTimeslice = m_TriggerScrubParams->MaxTimeslice;
+				ManualGcTriggered = true;
 			}
 
-			if (DoScrubbing)
-			{
-				ScrubStorage(DoDelete, SkipCid, ScrubTimeslice);
-				m_TriggerScrubParams.reset();
-			}
-
-			if (!DoGc)
+			if (!ManualScrubbingTriggered && !ManualGcTriggered && !m_Config.Enabled)
 			{
 				continue;
 			}
@@ -2288,10 +2299,12 @@ GcScheduler::SchedulerThread()
 			GcClock::TimePoint BuildStoreExpireTime =
 				MaxBuildStoreDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxBuildStoreDuration;
 
-			const GcStorageSize TotalSize = m_GcManager.TotalStorageSize();
-
-			if (Timeout && Status() == GcSchedulerStatus::kIdle)
+			if (!ManualGcTriggered && !ManualScrubbingTriggered)
 			{
+				// Check for GC triggered by time/size limits
+
+				const GcStorageSize TotalSize = m_GcManager.TotalStorageSize();
+
 				DiskSpace Space = CheckDiskSpace();
 
 				const int64_t				PressureGraphLength = 30;
@@ -2508,7 +2521,9 @@ GcScheduler::SchedulerThread()
 					}
 					continue;
 				}
+			}
 
+			{
 				uint32_t IdleState = static_cast<uint32_t>(GcSchedulerStatus::kIdle);
 				if (!m_Status.compare_exchange_strong(IdleState, static_cast<uint32_t>(GcSchedulerStatus::kRunning)))
 				{
@@ -2517,13 +2532,30 @@ GcScheduler::SchedulerThread()
 				}
 			}
 
-			if (!SkipCid)
+			auto ResetState = MakeGuard([&]() {
+				uint32_t RunningState = static_cast<uint32_t>(GcSchedulerStatus::kRunning);
+				if (!m_Status.compare_exchange_strong(RunningState, static_cast<uint32_t>(GcSchedulerStatus::kIdle)))
+				{
+					ZEN_ASSERT(m_Status == static_cast<uint32_t>(GcSchedulerStatus::kStopped));
+				}
+			});
+
+			if (ManualScrubbingTriggered)
 			{
-				m_AttachmentPassIndex = NextAttachmentPassIndex;
+				ScrubStorage(DoDelete, SkipCid, ScrubTimeslice);
+				if (!ManualGcTriggered)
+				{
+					continue;
+				}
 			}
 
 			if (PrepareDiskReserve())
 			{
+				if (!SkipCid)
+				{
+					m_AttachmentPassIndex = NextAttachmentPassIndex;
+				}
+
 				bool GcSuccess = CollectGarbage(CacheExpireTime,
 												ProjectStoreExpireTime,
 												BuildStoreExpireTime,
@@ -2596,13 +2628,6 @@ GcScheduler::SchedulerThread()
 			WaitTime				= m_Config.MonitorInterval;
 		}
 		m_GcManager.SetCancelGC(false);
-
-		uint32_t RunningState = static_cast<uint32_t>(GcSchedulerStatus::kRunning);
-		if (!m_Status.compare_exchange_strong(RunningState, static_cast<uint32_t>(GcSchedulerStatus::kIdle)))
-		{
-			ZEN_ASSERT(m_Status == static_cast<uint32_t>(GcSchedulerStatus::kStopped));
-			break;
-		}
 	}
 }