Size based gc trigger (#197)

- Feature: Disk size triggered GC, a soft disk usage limit for cache data.
- Feature: New option `--gc-disk-size-soft-limit` (command line), `gc.cache.disksizesoftlimit` (lua config) controlling limit for soft disk usage limit. Defaults to zero which disables soft disk usage limit.
- Improvement: Disk write pressure in GC log and cleaned up clutter in GC logging.

1 files changed, 32 insertions, 7 deletions

diff --git a/zenstore/include/zenstore/gc.h b/zenstore/include/zenstore/gc.h
index 656e594af..e0354b331 100644
--- a/zenstore/include/zenstore/gc.h
+++ b/zenstore/include/zenstore/gc.h
@@ -4,6 +4,7 @@
 
 #include <zencore/iohash.h>
 #include <zencore/thread.h>
+#include <zenstore/caslog.h>
 
 #include <atomic>
 #include <chrono>
@@ -47,7 +48,7 @@ public:
 class GcContext
 {
 public:
-	GcContext(GcClock::TimePoint Time = GcClock::Now());
+	GcContext(const GcClock::TimePoint& ExpireTime);
 	~GcContext();
 
 	void AddRetainedCids(std::span<const IoHash> Cid);
@@ -69,16 +70,11 @@ public:
 	bool CollectSmallObjects() const;
 	void CollectSmallObjects(bool NewState);
 
-	GcClock::TimePoint Time() const;
-
-	GcClock::Duration MaxCacheDuration() const;
-	void			  MaxCacheDuration(GcClock::Duration Duration);
+	GcClock::TimePoint ExpireTime() const;
 
 	void	 DiskReservePath(const std::filesystem::path& Path);
 	uint64_t ClaimGCReserve();
 
-	inline bool Expired(GcClock::Tick TickCount) { return Time() - GcClock::TimePointFromTick(TickCount) > MaxCacheDuration(); }
-
 private:
 	struct GcState;
 
@@ -150,6 +146,8 @@ public:
 #endif
 
 private:
+	spdlog::logger&				Log() { return m_Log; }
+	spdlog::logger&				m_Log;
 	mutable RwLock				m_Lock;
 	std::vector<GcContributor*> m_GcContribs;
 	std::vector<GcStorage*>		m_GcStorage;
@@ -172,6 +170,27 @@ struct GcSchedulerConfig
 	bool				  CollectSmallObjects = true;
 	bool				  Enabled			  = true;
 	uint64_t			  DiskReserveSize	  = 1ul << 28;
+	uint64_t			  DiskSizeSoftLimit	  = 0;
+};
+
+class DiskUsageWindow
+{
+public:
+	struct DiskUsageEntry
+	{
+		GcClock::Tick SampleTime;
+		uint64_t	  DiskUsage;
+	};
+
+	std::vector<DiskUsageEntry> m_LogWindow;
+	inline void					Append(const DiskUsageEntry& Entry) { m_LogWindow.push_back(Entry); }
+	inline void					Append(DiskUsageEntry&& Entry) { m_LogWindow.emplace_back(std::move(Entry)); }
+	void						KeepRange(GcClock::Tick StartTick, GcClock::Tick EndTick);
+	std::vector<uint64_t>		GetDiskDeltas(GcClock::Tick StartTick,
+											  GcClock::Tick EndTick,
+											  GcClock::Tick DeltaWidth,
+											  uint64_t&		OutMaxDelta) const;
+	GcClock::Tick				FindTimepointThatRemoves(uint64_t Amount, GcClock::Tick EndTick) const;
 };
 
 /**
@@ -191,12 +210,14 @@ public:
 	{
 		bool				 CollectSmallObjects = false;
 		std::chrono::seconds MaxCacheDuration	 = std::chrono::seconds::max();
+		uint64_t			 DiskSizeSoftLimit	 = 0;
 	};
 
 	bool Trigger(const TriggerParams& Params);
 
 private:
 	void			   SchedulerThread();
+	void			   CollectGarbage(const GcClock::TimePoint& ExpireTime, bool Delete, bool CollectSmallObjects);
 	GcClock::TimePoint NextGcTime(GcClock::TimePoint CurrentTime);
 	spdlog::logger&	   Log() { return m_Log; }
 
@@ -204,12 +225,16 @@ private:
 	GcManager&					 m_GcManager;
 	GcSchedulerConfig			 m_Config;
 	GcClock::TimePoint			 m_LastGcTime{};
+	GcClock::TimePoint			 m_LastGcExpireTime{};
 	GcClock::TimePoint			 m_NextGcTime{};
 	std::atomic_uint32_t		 m_Status{};
 	std::thread					 m_GcThread;
 	std::mutex					 m_GcMutex;
 	std::condition_variable		 m_GcSignal;
 	std::optional<TriggerParams> m_TriggerParams;
+
+	TCasLogFile<DiskUsageWindow::DiskUsageEntry> m_DiskUsageLog;
+	DiskUsageWindow								 m_DiskUsageWindow;
 };
 
 void gc_forcelink();