add `flush` command and more gc status info (#483)

- Feature: New endpoint `/admin/flush ` to flush all storage - CAS, Cache and ProjectStore
- Feature: New command `zen flush` to flush all storage - CAS, Cache and ProjectStore
- Improved: Command `zen gc-status` now gives details about storage, when last GC occured, how long until next GC etc
- Changed: Cache access and write log are disabled by default

3 files changed, 147 insertions, 32 deletions

diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp
index 9446e2152..fc549a729 100644
--- a/src/zenstore/cas.cpp
+++ b/src/zenstore/cas.cpp
@@ -83,6 +83,7 @@ CasImpl::CasImpl(GcManager& Gc) : m_TinyStrategy(Gc), m_SmallStrategy(Gc), m_Lar
 
 CasImpl::~CasImpl()
 {
+	ZEN_INFO("closing Cas pool at '{}'", m_Config.RootDirectory);
 }
 
 void
@@ -259,6 +260,7 @@ CasImpl::FilterChunks(HashKeySet& InOutChunks)
 void
 CasImpl::Flush()
 {
+	ZEN_INFO("flushing CAS pool at '{}'", m_Config.RootDirectory);
 	m_SmallStrategy.Flush();
 	m_TinyStrategy.Flush();
 	m_LargeStrategy.Flush();
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 0f3f178d6..0e4c49f65 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -39,19 +39,6 @@
 #	include <random>
 #endif
 
-template<>
-struct fmt::formatter<zen::GcClock::TimePoint> : formatter<string_view>
-{
-	template<typename FormatContext>
-	auto format(const zen::GcClock::TimePoint& TimePoint, FormatContext& ctx)
-	{
-		std::time_t Time = std::chrono::system_clock::to_time_t(TimePoint);
-		char		TimeString[std::size("yyyy-mm-ddThh:mm:ss")];
-		std::strftime(std::data(TimeString), std::size(TimeString), "%FT%T", std::localtime(&Time));
-		return fmt::format_to(ctx.out(), "{}", TimeString);
-	}
-};
-
 namespace zen {
 
 using namespace std::literals;
@@ -403,11 +390,13 @@ GcManager::ScrubStorage(ScrubContext& GcCtx)
 	}
 }
 
-void
+GcStorageSize
 GcManager::CollectGarbage(GcContext& GcCtx)
 {
 	ZEN_TRACE_CPU("Gc::CollectGarbage");
 
+	GcStorageSize GCTotalSizeDiff;
+
 	RwLock::SharedLockScope _(m_Lock);
 
 	// First gather reference set
@@ -425,14 +414,13 @@ GcManager::CollectGarbage(GcContext& GcCtx)
 	{
 		ZEN_TRACE_CPU("Gc::CollectGarbage::CollectGarbage");
 
-		GcStorageSize GCTotalSizeDiff;
-		Stopwatch	  Timer;
-		const auto	  Guard = MakeGuard([&] {
-			   ZEN_INFO("collected garbage in {}. Removed {} disk space, {} memory",
-						NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
-						NiceBytes(GCTotalSizeDiff.DiskSize),
-						NiceBytes(GCTotalSizeDiff.MemorySize));
-		   });
+		Stopwatch  Timer;
+		const auto Guard = MakeGuard([&] {
+			ZEN_INFO("collected garbage in {}. Removed {} disk space, {} memory",
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+					 NiceBytes(GCTotalSizeDiff.DiskSize),
+					 NiceBytes(GCTotalSizeDiff.MemorySize));
+		});
 		for (GcStorage* Storage : m_GcStorage)
 		{
 			const auto PreSize = Storage->StorageSize();
@@ -442,6 +430,7 @@ GcManager::CollectGarbage(GcContext& GcCtx)
 			GCTotalSizeDiff.MemorySize += PreSize.MemorySize > PostSize.MemorySize ? PreSize.MemorySize - PostSize.MemorySize : 0;
 		}
 	}
+	return GCTotalSizeDiff;
 }
 
 GcStorageSize
@@ -753,6 +742,70 @@ GcScheduler::CheckDiskSpace()
 	return Space;
 }
 
+GcSchedulerState
+GcScheduler::GetState() const
+{
+	GcClock::TimePoint Now = GcClock::Now();
+
+	const GcStorageSize TotalSize = m_GcManager.TotalStorageSize();
+
+	GcSchedulerState Result{.Status = Status(), .Config = m_Config, .AreDiskWritesBlocked = m_AreDiskWritesBlocked.load()};
+
+	{
+		std::unique_lock Lock(m_GcMutex);
+		Result.LastFullGcTime		 = m_LastGcTime;
+		Result.LastFullGCDiff		 = m_LastFullGCDiff;
+		Result.LastLightweightGcTime = m_LastLightweightGcTime;
+		Result.LastLightweightGCDiff = m_LastLightweightGCDiff;
+	}
+	std::error_code Ec;
+	DiskSpace		Space = DiskSpaceInfo(Result.Config.RootDirectory, Ec);
+	if (!Ec)
+	{
+		Result.DiskSize = Space.Total;
+		Result.DiskUsed = Space.Total - Space.Free;
+		Result.DiskFree = Space.Free;
+		if (Result.Config.DiskSizeSoftLimit != 0)
+		{
+			Result.RemainingSpaceUntilFullGC =
+				Result.Config.DiskSizeSoftLimit > TotalSize.DiskSize ? (Result.Config.DiskSizeSoftLimit - TotalSize.DiskSize) : 0;
+		}
+	}
+	if (Result.Config.DiskReserveSize != 0)
+	{
+		Ec.clear();
+		Result.HasDiskReserve = std::filesystem::is_regular_file(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec;
+	}
+
+	GcClock::TimePoint CacheExpireTime =
+		Result.Config.MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - Result.Config.MaxCacheDuration;
+	GcClock::TimePoint ProjectStoreExpireTime = Result.Config.MaxProjectStoreDuration == GcClock::Duration::max()
+													? GcClock::TimePoint::min()
+													: Now - Result.Config.MaxProjectStoreDuration;
+
+	Result.RemainingTimeUntilFullGc =
+		Result.Config.Interval.count() == 0
+			? std::chrono::seconds::max()
+			: std::chrono::duration_cast<std::chrono::seconds>(Result.LastFullGcTime + Result.Config.Interval - Now);
+
+	if (Result.RemainingTimeUntilFullGc < std::chrono::seconds::zero())
+	{
+		Result.RemainingTimeUntilFullGc = std::chrono::seconds::zero();
+	}
+
+	Result.RemainingTimeUntilLightweightGc =
+		Result.Config.LightweightInterval.count() == 0
+			? std::chrono::seconds::max()
+			: std::chrono::duration_cast<std::chrono::seconds>(Result.LastLightweightGcTime + Result.Config.LightweightInterval - Now);
+
+	if (Result.RemainingTimeUntilLightweightGc < std::chrono::seconds::zero())
+	{
+		Result.RemainingTimeUntilLightweightGc = std::chrono::seconds::zero();
+	}
+
+	return Result;
+}
+
 void
 GcScheduler::SchedulerThread()
 {
@@ -1159,11 +1212,24 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 		Stopwatch  Timer;
 		const auto __ = MakeGuard([&] { ZEN_INFO("garbage collection DONE in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
 
-		m_GcManager.CollectGarbage(GcCtx);
+		GcStorageSize Diff = m_GcManager.CollectGarbage(GcCtx);
 
 		if (SkipCid)
 		{
 			m_LastLightweightGcTime = GcClock::Now();
+
+			std::chrono::seconds ElapsedSeconds =
+				std::chrono::duration_cast<std::chrono::seconds>(std::chrono::milliseconds(Timer.GetElapsedTimeMs()));
+			if (SkipCid)
+			{
+				m_LastLightweightGcDuration = ElapsedSeconds;
+				m_LastLightweightGCDiff		= Diff;
+			}
+			else
+			{
+				m_LastFullGcDuration = ElapsedSeconds;
+				m_LastFullGCDiff	 = Diff;
+			}
 		}
 		else
 		{
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index 2e52218f8..aaa156ebf 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -7,6 +7,10 @@
 #include <zenstore/caslog.h>
 #include <zenstore/scrubcontext.h>
 
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
 #include <atomic>
 #include <chrono>
 #include <condition_variable>
@@ -149,8 +153,8 @@ public:
 	void AddGcStorage(GcStorage* Contributor);
 	void RemoveGcStorage(GcStorage* Contributor);
 
-	void CollectGarbage(GcContext& GcCtx);
-	void ScrubStorage(ScrubContext& GcCtx);
+	GcStorageSize CollectGarbage(GcContext& GcCtx);
+	void		  ScrubStorage(ScrubContext& GcCtx);
 
 	GcStorageSize TotalStorageSize() const;
 
@@ -189,6 +193,29 @@ struct GcSchedulerConfig
 	std::chrono::seconds  LightweightInterval{};
 };
 
+struct GcSchedulerState
+{
+	GcSchedulerStatus Status;
+
+	GcSchedulerConfig Config;
+
+	bool				 AreDiskWritesBlocked = false;
+	bool				 HasDiskReserve		  = false;
+	uint64_t			 DiskSize			  = 0;
+	uint64_t			 DiskUsed			  = 0;
+	uint64_t			 DiskFree			  = 0;
+	GcClock::TimePoint	 LastFullGcTime{};
+	GcClock::TimePoint	 LastLightweightGcTime{};
+	std::chrono::seconds RemainingTimeUntilLightweightGc;
+	std::chrono::seconds RemainingTimeUntilFullGc;
+	uint64_t			 RemainingSpaceUntilFullGC = 0;
+
+	std::chrono::seconds LastFullGcDuration{};
+	GcStorageSize		 LastFullGCDiff;
+	std::chrono::seconds LastLightweightGcDuration{};
+	GcStorageSize		 LastLightweightGCDiff;
+};
+
 class DiskUsageWindow
 {
 public:
@@ -221,6 +248,7 @@ public:
 	void			  Initialize(const GcSchedulerConfig& Config);
 	void			  Shutdown();
 	GcSchedulerStatus Status() const { return static_cast<GcSchedulerStatus>(m_Status.load()); }
+	GcSchedulerState  GetState() const;
 
 	struct TriggerGcParams
 	{
@@ -253,15 +281,21 @@ private:
 	virtual bool	AreDiskWritesAllowed() const override { return !m_AreDiskWritesBlocked.load(); }
 	DiskSpace		CheckDiskSpace();
 
-	spdlog::logger&					  m_Log;
-	GcManager&						  m_GcManager;
-	GcSchedulerConfig				  m_Config;
-	GcClock::TimePoint				  m_LastGcTime{};
-	GcClock::TimePoint				  m_LastLightweightGcTime{};
-	GcClock::TimePoint				  m_LastGcExpireTime{};
+	spdlog::logger&	   m_Log;
+	GcManager&		   m_GcManager;
+	GcSchedulerConfig  m_Config;
+	GcClock::TimePoint m_LastGcTime{};
+	GcClock::TimePoint m_LastLightweightGcTime{};
+	GcClock::TimePoint m_LastGcExpireTime{};
+
+	std::chrono::seconds m_LastFullGcDuration{};
+	GcStorageSize		 m_LastFullGCDiff;
+	std::chrono::seconds m_LastLightweightGcDuration{};
+	GcStorageSize		 m_LastLightweightGCDiff;
+
 	std::atomic_uint32_t			  m_Status{};
 	std::thread						  m_GcThread;
-	std::mutex						  m_GcMutex;
+	mutable std::mutex				  m_GcMutex;
 	std::condition_variable			  m_GcSignal;
 	std::optional<TriggerGcParams>	  m_TriggerGcParams;
 	std::optional<TriggerScrubParams> m_TriggerScrubParams;
@@ -274,3 +308,16 @@ private:
 void gc_forcelink();
 
 }  // namespace zen
+
+template<>
+struct fmt::formatter<zen::GcClock::TimePoint> : formatter<string_view>
+{
+	template<typename FormatContext>
+	auto format(const zen::GcClock::TimePoint& TimePoint, FormatContext& ctx)
+	{
+		std::time_t Time = std::chrono::system_clock::to_time_t(TimePoint);
+		char		TimeString[std::size("yyyy-mm-ddThh:mm:ss")];
+		std::strftime(std::data(TimeString), std::size(TimeString), "%FT%T", std::localtime(&Time));
+		return fmt::format_to(ctx.out(), "{}", TimeString);
+	}
+};