Add `--gc-projectstore-duration-seconds` option (#281)

* Add `--gc-projectstore-duration-seconds` option
* Cleanup lua gc options parsing
* Remove dead configuration values
* changelog

1 files changed, 51 insertions, 26 deletions

diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index a7c757877..2d7e0e02f 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -189,16 +189,19 @@ struct GcContext::GcState
 	CacheKeyContexts   m_ExpiredCacheKeys;
 	HashKeySet		   m_RetainedCids;
 	HashKeySet		   m_DeletedCids;
-	GcClock::TimePoint m_ExpireTime;
+	GcClock::TimePoint m_CacheExpireTime;
+	GcClock::TimePoint m_ProjectStoreExpireTime;
 	bool			   m_DeletionMode		 = true;
 	bool			   m_CollectSmallObjects = false;
 
 	std::filesystem::path DiskReservePath;
 };
 
-GcContext::GcContext(const GcClock::TimePoint& ExpireTime) : m_State(std::make_unique<GcState>())
+GcContext::GcContext(const GcClock::TimePoint& CacheExpireTime, const GcClock::TimePoint& ProjectStoreExpireTime)
+: m_State(std::make_unique<GcState>())
 {
-	m_State->m_ExpireTime = ExpireTime;
+	m_State->m_CacheExpireTime		  = CacheExpireTime;
+	m_State->m_ProjectStoreExpireTime = ProjectStoreExpireTime;
 }
 
 GcContext::~GcContext()
@@ -278,9 +281,15 @@ GcContext::CollectSmallObjects(bool NewState)
 }
 
 GcClock::TimePoint
-GcContext::ExpireTime() const
+GcContext::CacheExpireTime() const
 {
-	return m_State->m_ExpireTime;
+	return m_State->m_CacheExpireTime;
+}
+
+GcClock::TimePoint
+GcContext::ProjectStoreExpireTime() const
+{
+	return m_State->m_ProjectStoreExpireTime;
 }
 
 void
@@ -710,11 +719,12 @@ GcScheduler::SchedulerThread()
 			continue;
 		}
 
-		bool				 Delete				 = true;
-		bool				 CollectSmallObjects = m_Config.CollectSmallObjects;
-		std::chrono::seconds MaxCacheDuration	 = m_Config.MaxCacheDuration;
-		uint64_t			 DiskSizeSoftLimit	 = m_Config.DiskSizeSoftLimit;
-		GcClock::TimePoint	 Now				 = GcClock::Now();
+		bool				 Delete					 = true;
+		bool				 CollectSmallObjects	 = m_Config.CollectSmallObjects;
+		std::chrono::seconds MaxCacheDuration		 = m_Config.MaxCacheDuration;
+		std::chrono::seconds MaxProjectStoreDuration = m_Config.MaxProjectStoreDuration;
+		uint64_t			 DiskSizeSoftLimit		 = m_Config.DiskSizeSoftLimit;
+		GcClock::TimePoint	 Now					 = GcClock::Now();
 		if (m_TriggerGcParams)
 		{
 			const auto TriggerParams = m_TriggerGcParams.value();
@@ -725,13 +735,20 @@ GcScheduler::SchedulerThread()
 			{
 				MaxCacheDuration = TriggerParams.MaxCacheDuration;
 			}
+			if (TriggerParams.MaxProjectStoreDuration != std::chrono::seconds::max())
+			{
+				MaxProjectStoreDuration = TriggerParams.MaxProjectStoreDuration;
+			}
 			if (TriggerParams.DiskSizeSoftLimit != 0)
 			{
 				DiskSizeSoftLimit = TriggerParams.DiskSizeSoftLimit;
 			}
 		}
 
-		GcClock::TimePoint ExpireTime = MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxCacheDuration;
+		GcClock::TimePoint CacheExpireTime =
+			MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxCacheDuration;
+		GcClock::TimePoint ProjectStoreExpireTime =
+			MaxProjectStoreDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxProjectStoreDuration;
 
 		const GcStorageSize TotalSize = m_GcManager.TotalStorageSize();
 
@@ -786,9 +803,13 @@ GcScheduler::SchedulerThread()
 				std::unique_lock   Lock(m_GcMutex);
 				GcClock::Tick	   AgeTick = m_DiskUsageWindow.FindTimepointThatRemoves(GcDiskSpaceGoal, Now.time_since_epoch().count());
 				GcClock::TimePoint SizeBasedExpireTime = GcClock::TimePointFromTick(AgeTick);
-				if (SizeBasedExpireTime > ExpireTime)
+				if (SizeBasedExpireTime > CacheExpireTime)
+				{
+					CacheExpireTime = SizeBasedExpireTime;
+				}
+				if (SizeBasedExpireTime > ProjectStoreExpireTime)
 				{
-					ExpireTime = SizeBasedExpireTime;
+					ProjectStoreExpireTime = SizeBasedExpireTime;
 				}
 			}
 
@@ -832,7 +853,7 @@ GcScheduler::SchedulerThread()
 			}
 		}
 
-		CollectGarbage(ExpireTime, Delete, CollectSmallObjects);
+		CollectGarbage(CacheExpireTime, ProjectStoreExpireTime, Delete, CollectSmallObjects);
 
 		uint32_t RunningState = static_cast<uint32_t>(GcSchedulerStatus::kRunning);
 		if (!m_Status.compare_exchange_strong(RunningState, static_cast<uint32_t>(GcSchedulerStatus::kIdle)))
@@ -859,17 +880,20 @@ GcScheduler::NextGcTime(GcClock::TimePoint CurrentTime)
 }
 
 void
-GcScheduler::CollectGarbage(const GcClock::TimePoint& ExpireTime, bool Delete, bool CollectSmallObjects)
+GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
+							const GcClock::TimePoint& ProjectStoreExpireTime,
+							bool					  Delete,
+							bool					  CollectSmallObjects)
 {
-	GcContext GcCtx(ExpireTime);
+	GcContext GcCtx(CacheExpireTime, ProjectStoreExpireTime);
 	GcCtx.SetDeletionMode(Delete);
 	GcCtx.CollectSmallObjects(CollectSmallObjects);
-	//	GcCtx.MaxCacheDuration(MaxCacheDuration);
 	GcCtx.DiskReservePath(m_Config.RootDirectory / "reserve.gc");
 
-	ZEN_INFO("garbage collection STARTING, small objects gc {}, cutoff time {}",
+	ZEN_INFO("garbage collection STARTING, small objects gc {}, cache cutoff time {}, project store cutoff time {}",
 			 GcCtx.CollectSmallObjects() ? "ENABLED"sv : "DISABLED"sv,
-			 ExpireTime);
+			 CacheExpireTime,
+			 ProjectStoreExpireTime);
 	{
 		Stopwatch  Timer;
 		const auto __ = MakeGuard([&] { ZEN_INFO("garbage collection DONE in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
@@ -878,9 +902,10 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& ExpireTime, bool Delete, b
 
 		if (Delete)
 		{
-			m_LastGcExpireTime = ExpireTime;
+			GcClock::TimePoint KeepRangeStart = Min(CacheExpireTime, ProjectStoreExpireTime);
+			m_LastGcExpireTime				  = KeepRangeStart;
 			std::unique_lock Lock(m_GcMutex);
-			m_DiskUsageWindow.KeepRange(ExpireTime.time_since_epoch().count(), GcClock::Duration::max().count());
+			m_DiskUsageWindow.KeepRange(KeepRangeStart.time_since_epoch().count(), GcClock::Duration::max().count());
 		}
 
 		m_LastGcTime = GcClock::Now();
@@ -953,7 +978,7 @@ TEST_CASE("gc.basic")
 	const auto InsertResult = CidStore.AddChunk(CompressedChunk.GetCompressed().Flatten().AsIoBuffer(), CompressedChunk.DecodeRawHash());
 	CHECK(InsertResult.New);
 
-	GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
+	GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
 	GcCtx.CollectSmallObjects(true);
 
 	CidStore.Flush();
@@ -1012,7 +1037,7 @@ TEST_CASE("gc.full")
 
 	// Keep first and last
 	{
-		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
+		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
 		GcCtx.CollectSmallObjects(true);
 
 		std::vector<IoHash> KeepChunks;
@@ -1047,7 +1072,7 @@ TEST_CASE("gc.full")
 
 	// Keep last
 	{
-		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
+		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
 		GcCtx.CollectSmallObjects(true);
 		std::vector<IoHash> KeepChunks;
 		KeepChunks.push_back(ChunkHashes[8]);
@@ -1079,7 +1104,7 @@ TEST_CASE("gc.full")
 
 	// Keep mixed
 	{
-		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
+		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
 		GcCtx.CollectSmallObjects(true);
 		std::vector<IoHash> KeepChunks;
 		KeepChunks.push_back(ChunkHashes[1]);
@@ -1114,7 +1139,7 @@ TEST_CASE("gc.full")
 
 	// Keep multiple at end
 	{
-		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
+		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
 		GcCtx.CollectSmallObjects(true);
 		std::vector<IoHash> KeepChunks;
 		KeepChunks.push_back(ChunkHashes[6]);