lightweight gc (#431)

- Feature: Add lightweight GC that only removes items from cache/project store without cleaning up data referenced in Cid store
  - Add `skipcid` parameter to http endpoint `admin/gc`, defaults to "false"
  - Add `--skipcid` option to `zen gc` command, defaults to false
  - Add `--gc-lightweight-interval-seconds` option to zenserver

1 files changed, 16 insertions, 11 deletions

diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index 28e5c2ec2..2e52218f8 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -65,6 +65,9 @@ public:
 
 	std::span<const IoHash> ExpiredCacheKeys(const std::string& CacheKeyContext) const;
 
+	bool SkipCid() const;
+	void SetSkipCid(bool NewState);
+
 	bool IsDeletionMode() const;
 	void SetDeletionMode(bool NewState);
 
@@ -183,6 +186,7 @@ struct GcSchedulerConfig
 	uint64_t			  DiskReserveSize					= 1ul << 28;
 	uint64_t			  DiskSizeSoftLimit					= 0;
 	uint64_t			  MinimumFreeDiskSpaceToAllowWrites = 1ul << 28;
+	std::chrono::seconds  LightweightInterval{};
 };
 
 class DiskUsageWindow
@@ -224,6 +228,7 @@ public:
 		std::chrono::seconds MaxCacheDuration		 = std::chrono::seconds::max();
 		std::chrono::seconds MaxProjectStoreDuration = std::chrono::seconds::max();
 		uint64_t			 DiskSizeSoftLimit		 = 0;
+		bool				 SkipCid				 = false;
 	};
 
 	bool TriggerGc(const TriggerGcParams& Params);
@@ -237,23 +242,23 @@ public:
 	bool TriggerScrub(const TriggerScrubParams& Params);
 
 private:
-	void			   SchedulerThread();
-	void			   CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
-									  const GcClock::TimePoint& ProjectStoreExpireTime,
-									  bool						Delete,
-									  bool						CollectSmallObjects);
-	void			   ScrubStorage(bool DoDelete, std::chrono::seconds TimeSlice);
-	GcClock::TimePoint NextGcTime(GcClock::TimePoint CurrentTime);
-	spdlog::logger&	   Log() { return m_Log; }
-	virtual bool	   AreDiskWritesAllowed() const override { return !m_AreDiskWritesBlocked.load(); }
-	DiskSpace		   CheckDiskSpace();
+	void			SchedulerThread();
+	void			CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
+								   const GcClock::TimePoint& ProjectStoreExpireTime,
+								   bool						 Delete,
+								   bool						 CollectSmallObjects,
+								   bool						 SkipCid);
+	void			ScrubStorage(bool DoDelete, std::chrono::seconds TimeSlice);
+	spdlog::logger& Log() { return m_Log; }
+	virtual bool	AreDiskWritesAllowed() const override { return !m_AreDiskWritesBlocked.load(); }
+	DiskSpace		CheckDiskSpace();
 
 	spdlog::logger&					  m_Log;
 	GcManager&						  m_GcManager;
 	GcSchedulerConfig				  m_Config;
 	GcClock::TimePoint				  m_LastGcTime{};
+	GcClock::TimePoint				  m_LastLightweightGcTime{};
 	GcClock::TimePoint				  m_LastGcExpireTime{};
-	GcClock::TimePoint				  m_NextGcTime{};
 	std::atomic_uint32_t			  m_Status{};
 	std::thread						  m_GcThread;
 	std::mutex						  m_GcMutex;