lightweight gc (#431)

- Feature: Add lightweight GC that only removes items from cache/project store without cleaning up data referenced in Cid store
  - Add `skipcid` parameter to http endpoint `admin/gc`, defaults to "false"
  - Add `--skipcid` option to `zen gc` command, defaults to false
  - Add `--gc-lightweight-interval-seconds` option to zenserver

1 files changed, 141 insertions, 47 deletions

diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index a3282dde6..f8ef5de82 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -196,6 +196,7 @@ struct GcContext::GcState
 	GcClock::TimePoint m_ProjectStoreExpireTime;
 	bool			   m_DeletionMode		 = true;
 	bool			   m_CollectSmallObjects = false;
+	bool			   m_SkipCid			 = false;
 
 	std::filesystem::path DiskReservePath;
 };
@@ -260,6 +261,18 @@ GcContext::ExpiredCacheKeys(const std::string& CacheKeyContext) const
 }
 
 bool
+GcContext::SkipCid() const
+{
+	return m_State->m_SkipCid;
+}
+
+void
+GcContext::SetSkipCid(bool NewState)
+{
+	m_State->m_SkipCid = NewState;
+}
+
+bool
 GcContext::IsDeletionMode() const
 {
 	return m_State->m_DeletionMode;
@@ -591,6 +604,11 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
 		m_Config.Interval = m_Config.MonitorInterval;
 	}
 
+	if (m_Config.LightweightInterval.count() && m_Config.LightweightInterval < m_Config.MonitorInterval)
+	{
+		m_Config.LightweightInterval = m_Config.MonitorInterval;
+	}
+
 	std::filesystem::create_directories(Config.RootDirectory);
 
 	std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
@@ -604,8 +622,9 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
 
 	CheckDiskSpace();
 
-	m_LastGcTime	   = GcClock::Now();
-	m_LastGcExpireTime = GcClock::TimePoint::min();
+	m_LastGcTime			= GcClock::Now();
+	m_LastLightweightGcTime = m_LastGcTime;
+	m_LastGcExpireTime		= GcClock::TimePoint::min();
 
 	if (CbObject SchedulerState = LoadCompactBinaryObject(Config.RootDirectory / "gc_state"))
 	{
@@ -615,7 +634,8 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
 		if (m_LastGcTime + m_Config.Interval < GcClock::Now())
 		{
 			// TODO: Trigger GC?
-			m_LastGcTime = GcClock::Now();
+			m_LastGcTime			= GcClock::Now();
+			m_LastLightweightGcTime = m_LastGcTime;
 		}
 	}
 
@@ -631,8 +651,7 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
 		},
 		0);
 
-	m_NextGcTime = NextGcTime(m_LastGcTime);
-	m_GcThread	 = std::thread(&GcScheduler::SchedulerThread, this);
+	m_GcThread = std::thread(&GcScheduler::SchedulerThread, this);
 }
 
 void
@@ -773,10 +792,17 @@ GcScheduler::SchedulerThread()
 			std::chrono::seconds ScrubTimeslice			 = std::chrono::seconds::max();
 			bool				 DoDelete				 = true;
 			bool				 CollectSmallObjects	 = m_Config.CollectSmallObjects;
+			std::chrono::seconds GcInterval				 = m_Config.Interval;
+			std::chrono::seconds LightwightGcInterval	 = m_Config.LightweightInterval;
 			std::chrono::seconds MaxCacheDuration		 = m_Config.MaxCacheDuration;
 			std::chrono::seconds MaxProjectStoreDuration = m_Config.MaxProjectStoreDuration;
 			uint64_t			 DiskSizeSoftLimit		 = m_Config.DiskSizeSoftLimit;
-			GcClock::TimePoint	 Now					 = GcClock::Now();
+			bool				 SkipCid				 = false;
+
+			bool DiskSpaceGCTriggered = false;
+			bool TimeBasedGCTriggered = false;
+
+			GcClock::TimePoint Now = GcClock::Now();
 
 			if (m_TriggerGcParams)
 			{
@@ -797,6 +823,10 @@ GcScheduler::SchedulerThread()
 				{
 					DiskSizeSoftLimit = TriggerParams.DiskSizeSoftLimit;
 				}
+				if (TriggerParams.SkipCid)
+				{
+					SkipCid = true;
+				}
 			}
 
 			if (m_TriggerScrubParams)
@@ -859,7 +889,7 @@ GcScheduler::SchedulerThread()
 				LoadGraph.resize(DiskDeltas.size(), '0');
 				if (DiskDeltas.size() > 0 && MaxLoad > 0)
 				{
-					char LoadIndicator[11] = "0123456789";
+					static const char LoadIndicator[11] = "0123456789";
 					for (size_t Index = 0; Index < DiskDeltas.size(); ++Index)
 					{
 						size_t LoadIndex = (9 * DiskDeltas[Index] + MaxLoad - 1) / MaxLoad;
@@ -884,48 +914,114 @@ GcScheduler::SchedulerThread()
 					}
 				}
 
-				const bool DiskSpaceGCTriggered = GcDiskSpaceGoal > 0;
+				std::chrono::seconds RemaingTimeUntilGc =
+					GcInterval.count() == 0 ? std::chrono::seconds::max()
+											: std::chrono::duration_cast<std::chrono::seconds>(m_LastGcTime + GcInterval - GcClock::Now());
+				if (RemaingTimeUntilGc < std::chrono::seconds::zero())
+				{
+					RemaingTimeUntilGc = std::chrono::seconds::zero();
+				}
+				std::chrono::seconds RemaingTimeUntilLightweightGc =
+					LightwightGcInterval.count() == 0
+						? std::chrono::seconds::max()
+						: std::chrono::duration_cast<std::chrono::seconds>(m_LastLightweightGcTime + LightwightGcInterval - GcClock::Now());
+				if (RemaingTimeUntilLightweightGc < std::chrono::seconds::zero())
+				{
+					RemaingTimeUntilLightweightGc = std::chrono::seconds::zero();
+				}
 
-				std::chrono::seconds RemaingTime = std::chrono::duration_cast<std::chrono::seconds>(m_NextGcTime - GcClock::Now());
+				if (GcDiskSpaceGoal > 0)
+				{
+					DiskSpaceGCTriggered = true;
+				}
+				else if (RemaingTimeUntilGc.count() == 0)
+				{
+					TimeBasedGCTriggered = true;
+				}
+				else if (RemaingTimeUntilLightweightGc.count() == 0)
+				{
+					TimeBasedGCTriggered = true;
+					SkipCid				 = true;
+				}
 
-				if (RemaingTime < std::chrono::seconds::zero())
+				std::string NextTriggerStatus;
+				if (GcInterval.count() != 0 || LightwightGcInterval.count() != 0 || DiskSizeSoftLimit != 0)
 				{
-					RemaingTime = std::chrono::seconds::zero();
+					ExtendableStringBuilder<256> Sb;
+					if (DiskSpaceGCTriggered)
+					{
+						Sb.Append(fmt::format(" Disk space exceeds {}, trying to reclaim {}.",
+											  NiceBytes(DiskSizeSoftLimit),
+											  NiceBytes(GcDiskSpaceGoal)));
+					}
+					else if (TimeBasedGCTriggered)
+					{
+						if (SkipCid)
+						{
+							Sb.Append(fmt::format(" Lightweight GC schedule triggered."));
+						}
+						else
+						{
+							Sb.Append(fmt::format(" GC schedule triggered."));
+						}
+					}
+					else
+					{
+						if (GcInterval.count() != 0)
+						{
+							Sb.Append(fmt::format(" Full GC in {}.",
+												  NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(RemaingTimeUntilGc).count()))));
+						}
+						if (LightwightGcInterval.count() != 0)
+						{
+							Sb.Append(
+								fmt::format(" Lightweight GC in {}.",
+											NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(RemaingTimeUntilLightweightGc).count()))));
+						}
+						if (DiskSizeSoftLimit != 0 && DiskSizeSoftLimit > TotalSize.DiskSize)
+						{
+							Sb.Append(fmt::format(" Disk usagage GC in {}.", NiceBytes(DiskSizeSoftLimit - TotalSize.DiskSize)));
+						}
+					}
+					NextTriggerStatus = Sb;
 				}
 
-				bool TimeBasedGCTriggered = !DiskSpaceGCTriggered && RemaingTime.count() == 0;
 				ZEN_INFO(
-					"{} in use,{} {} of total {} free disk space, disk writes last {} per {} [{}], peak {}/s. {}",
+					"{} used{}. '{}': {} in use, {} free. Disk writes last {} per {} [{}], peak {}/s.{}",
 					NiceBytes(TotalSize.DiskSize),
-					DiskSizeSoftLimit == 0 ? "" : fmt::format(" {} soft limit,", NiceBytes(DiskSizeSoftLimit)),
+					DiskSizeSoftLimit == 0 ? "" : fmt::format(", {} soft limit", NiceBytes(DiskSizeSoftLimit)),
+					m_Config.RootDirectory,
 					NiceBytes(Space.Free),
 					NiceBytes(Space.Total),
 					NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(LoadGraphTime).count())),
 					NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(LoadGraphTime).count() / PressureGraphLength)),
 					LoadGraph,
 					NiceBytes(MaxLoad * uint64_t(std::chrono::seconds(1).count()) / uint64_t(std::chrono::seconds(LoadGraphTime).count())),
-					DiskSpaceGCTriggered   ? fmt::format("Disk use threshold triggered, trying to reclaim {}. ", NiceBytes(GcDiskSpaceGoal))
-					: TimeBasedGCTriggered ? "GC schedule triggered."
-					: m_NextGcTime == GcClock::TimePoint::max()
-						? ""
-						: fmt::format("{} until next scheduled GC.",
-									  NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(RemaingTime).count()))));
+					NextTriggerStatus);
 
 				if (!DiskSpaceGCTriggered && !TimeBasedGCTriggered)
 				{
-					WaitTime = m_Config.MonitorInterval < RemaingTime ? m_Config.MonitorInterval : RemaingTime;
+					WaitTime = m_Config.MonitorInterval;
+					if (RemaingTimeUntilGc < WaitTime)
+					{
+						WaitTime = RemaingTimeUntilGc;
+					}
+					if (RemaingTimeUntilLightweightGc < WaitTime)
+					{
+						WaitTime = RemaingTimeUntilLightweightGc;
+					}
 					continue;
 				}
 
-				WaitTime		   = m_Config.MonitorInterval;
 				uint32_t IdleState = static_cast<uint32_t>(GcSchedulerStatus::kIdle);
 				if (!m_Status.compare_exchange_strong(IdleState, static_cast<uint32_t>(GcSchedulerStatus::kRunning)))
 				{
+					WaitTime = m_Config.MonitorInterval;
 					continue;
 				}
 			}
 
-			CollectGarbage(CacheExpireTime, ProjectStoreExpireTime, DoDelete, CollectSmallObjects);
+			CollectGarbage(CacheExpireTime, ProjectStoreExpireTime, DoDelete, CollectSmallObjects, SkipCid);
 
 			uint32_t RunningState = static_cast<uint32_t>(GcSchedulerStatus::kRunning);
 			if (!m_Status.compare_exchange_strong(RunningState, static_cast<uint32_t>(GcSchedulerStatus::kIdle)))
@@ -933,26 +1029,14 @@ GcScheduler::SchedulerThread()
 				ZEN_ASSERT(m_Status == static_cast<uint32_t>(GcSchedulerStatus::kStopped));
 				break;
 			}
+
+			WaitTime = std::chrono::seconds(0);
 		}
 		catch (std::exception& Ex)
 		{
 			ZEN_ERROR("scheduling garbage collection failed with: '{}'", Ex.what());
+			WaitTime = m_Config.MonitorInterval;
 		}
-
-		WaitTime = m_Config.MonitorInterval;
-	}
-}
-
-GcClock::TimePoint
-GcScheduler::NextGcTime(GcClock::TimePoint CurrentTime)
-{
-	if (m_Config.Interval.count())
-	{
-		return CurrentTime + m_Config.Interval;
-	}
-	else
-	{
-		return GcClock::TimePoint::max();
 	}
 }
 
@@ -990,12 +1074,14 @@ void
 GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 							const GcClock::TimePoint& ProjectStoreExpireTime,
 							bool					  Delete,
-							bool					  CollectSmallObjects)
+							bool					  CollectSmallObjects,
+							bool					  SkipCid)
 {
 	ZEN_TRACE_CPU("GcScheduler::CollectGarbage");
 
 	GcContext GcCtx(CacheExpireTime, ProjectStoreExpireTime);
 	GcCtx.SetDeletionMode(Delete);
+	GcCtx.SetSkipCid(SkipCid);
 	GcCtx.CollectSmallObjects(CollectSmallObjects);
 	GcCtx.DiskReservePath(m_Config.RootDirectory / "reserve.gc");
 
@@ -1027,8 +1113,9 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 		}
 	}
 
-	ZEN_INFO("garbage collection STARTING, small objects gc {}, cache cutoff time {}, project store cutoff time {}",
+	ZEN_INFO("garbage collection STARTING, small objects gc {}, CAS. Cache cutoff time {}, project store cutoff time {}",
 			 GcCtx.CollectSmallObjects() ? "ENABLED"sv : "DISABLED"sv,
+			 SkipCid ? "skip"sv : "include"sv,
 			 CacheExpireTime,
 			 ProjectStoreExpireTime);
 	{
@@ -1037,16 +1124,23 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 
 		m_GcManager.CollectGarbage(GcCtx);
 
-		if (Delete)
+		if (SkipCid)
 		{
-			GcClock::TimePoint KeepRangeStart = Min(CacheExpireTime, ProjectStoreExpireTime);
-			m_LastGcExpireTime				  = KeepRangeStart;
-			std::unique_lock Lock(m_GcMutex);
-			m_DiskUsageWindow.KeepRange(KeepRangeStart.time_since_epoch().count(), GcClock::Duration::max().count());
+			m_LastLightweightGcTime = GcClock::Now();
 		}
+		else
+		{
+			if (Delete)
+			{
+				GcClock::TimePoint KeepRangeStart = Min(CacheExpireTime, ProjectStoreExpireTime);
+				m_LastGcExpireTime				  = KeepRangeStart;
+				std::unique_lock Lock(m_GcMutex);
+				m_DiskUsageWindow.KeepRange(KeepRangeStart.time_since_epoch().count(), GcClock::Duration::max().count());
+			}
 
-		m_LastGcTime = GcClock::Now();
-		m_NextGcTime = NextGcTime(m_LastGcTime);
+			m_LastGcTime			= GcClock::Now();
+			m_LastLightweightGcTime = m_LastGcTime;
+		}
 
 		try
 		{