add `flush` command and more gc status info (#483)

- Feature: New endpoint `/admin/flush ` to flush all storage - CAS, Cache and ProjectStore
- Feature: New command `zen flush` to flush all storage - CAS, Cache and ProjectStore
- Improved: Command `zen gc-status` now gives details about storage, when last GC occured, how long until next GC etc
- Changed: Cache access and write log are disabled by default

13 files changed, 345 insertions, 68 deletions

diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp
index 9d8348b43..9713d8c87 100644
--- a/src/zen/cmds/admin_cmd.cpp
+++ b/src/zen/cmds/admin_cmd.cpp
@@ -374,4 +374,54 @@ LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	return 0;
 }
 
+//////////////////////////////////////////////////////////////////////////
+
+FlushCommand::FlushCommand()
+{
+	m_Options.add_options()("h,help", "Print help");
+	m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+}
+
+FlushCommand::~FlushCommand() = default;
+
+int
+FlushCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+	ZEN_UNUSED(GlobalOptions);
+
+	if (!ParseOptions(argc, argv))
+	{
+		return 0;
+	}
+
+	m_HostName = ResolveTargetHostSpec(m_HostName);
+
+	if (m_HostName.empty())
+	{
+		throw OptionParseException("unable to resolve server specification");
+	}
+
+	zen::HttpClient Http(m_HostName);
+
+	if (zen::HttpClient::Response Response = Http.Post("/admin/flush"sv))
+	{
+		ZEN_CONSOLE("OK: {}", Response.ToText());
+
+		return 0;
+	}
+	else if (int StatusCode = (int)Response.StatusCode)
+	{
+		ZEN_ERROR("flush failed: {}: {} ({})",
+				  (int)Response.StatusCode,
+				  ReasonStringForHttpResultCode((int)Response.StatusCode),
+				  Response.AsText());
+	}
+	else
+	{
+		ZEN_ERROR("flush failed: {}", Response.AsText());
+	}
+
+	return 1;
+}
+
 }  // namespace zen
diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h
index af375e91b..d1f8079b9 100644
--- a/src/zen/cmds/admin_cmd.h
+++ b/src/zen/cmds/admin_cmd.h
@@ -93,4 +93,20 @@ private:
 	std::string		 m_SetLogLevel;
 };
 
+/** Flush storage
+ */
+class FlushCommand : public ZenCmdBase
+{
+public:
+	FlushCommand();
+	~FlushCommand();
+
+	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+	virtual cxxopts::Options& Options() override { return m_Options; }
+
+private:
+	cxxopts::Options m_Options{"flush", "Flush zen storage"};
+	std::string		 m_HostName;
+};
+
 }  // namespace zen
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index 949a5f4ac..cfed591a8 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -209,6 +209,7 @@ main(int argc, char** argv)
 	DropCommand				 DropCmd;
 	DropProjectCommand		 ProjectDropCmd;
 	ExportOplogCommand		 ExportOplogCmd;
+	FlushCommand			 FlushCmd;
 	GcCommand				 GcCmd;
 	GcStatusCommand			 GcStatusCmd;
 	HashCommand				 HashCmd;
@@ -286,6 +287,7 @@ main(int argc, char** argv)
 		{"attach",				&AttachCmd,				"Add a sponsor process to a running zen service"},
 		{"version",				&VersionCmd,			"Get zen server version"},
 		{"vfs",					&VfsCmd,				"Manage virtual file system"},
+		{"flush",				&FlushCmd,				"Flush storage"},
 #if ZEN_WITH_TESTS
 		{"runtests",			&RunTestsCmd,			"Run zen tests"},
 #endif
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index 60599a992..5313a7592 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -3,8 +3,10 @@
 #include "admin.h"
 
 #include <zencore/compactbinarybuilder.h>
+#include <zencore/fmtutils.h>
 #include <zencore/jobqueue.h>
 #include <zencore/string.h>
+
 #if ZEN_WITH_TRACE
 #	include <zencore/trace.h>
 #endif	// ZEN_WITH_TRACE
@@ -13,8 +15,11 @@
 #	include <mimalloc.h>
 #endif
 
+#include <zenstore/cidstore.h>
 #include <zenstore/gc.h>
+
 #include "cache/structuredcachestore.h"
+#include "projectstore/projectstore.h"
 
 #include <chrono>
 
@@ -26,11 +31,15 @@ namespace zen {
 
 HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 								   JobQueue&	   BackgroundJobQueue,
-								   ZenCacheStore&  CacheStore,
+								   ZenCacheStore*  CacheStore,
+								   CidStore*	   CidStore,
+								   ProjectStore*   ProjectStore,
 								   const LogPaths& LogPaths)
 : m_GcScheduler(Scheduler)
 , m_BackgroundJobQueue(BackgroundJobQueue)
 , m_CacheStore(CacheStore)
+, m_CidStore(CidStore)
+, m_ProjectStore(ProjectStore)
 , m_LogPaths(LogPaths)
 {
 	using namespace std::literals;
@@ -188,10 +197,57 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 	m_Router.RegisterRoute(
 		"gc",
 		[this](HttpRouterRequest& Req) {
-			const GcSchedulerStatus Status = m_GcScheduler.Status();
+			const GcSchedulerState State = m_GcScheduler.GetState();
+
+			auto SecondsToString = [](std::chrono::seconds Secs) {
+				return NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(Secs).count()));
+			};
 
 			CbObjectWriter Response;
-			Response << "Status"sv << (GcSchedulerStatus::kIdle == Status ? "Idle"sv : "Running"sv);
+			Response << "Status"sv << (GcSchedulerStatus::kIdle == State.Status ? "Idle"sv : "Running"sv);
+			Response.BeginObject("Config");
+			{
+				Response << "RootDirectory" << State.Config.RootDirectory.string();
+				Response << "MonitorInterval" << SecondsToString(State.Config.MonitorInterval);
+				Response << "Interval" << SecondsToString(State.Config.Interval);
+				Response << "MaxCacheDuration" << SecondsToString(State.Config.MaxCacheDuration);
+				Response << "MaxProjectStoreDuration" << SecondsToString(State.Config.MaxProjectStoreDuration);
+				Response << "CollectSmallObjects" << State.Config.CollectSmallObjects;
+				Response << "Enabled" << State.Config.Enabled;
+				Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize);
+				Response << "DiskSizeSoftLimit" << NiceBytes(State.Config.DiskSizeSoftLimit);
+				Response << "MinimumFreeDiskSpaceToAllowWrites" << NiceBytes(State.Config.MinimumFreeDiskSpaceToAllowWrites);
+				Response << "LightweightInterval" << SecondsToString(State.Config.LightweightInterval);
+			}
+			Response.EndObject();
+			Response << "AreDiskWritesBlocked" << State.AreDiskWritesBlocked;
+			Response << "HasDiskReserve" << State.HasDiskReserve;
+			Response << "DiskSize" << NiceBytes(State.DiskSize);
+			Response << "DiskUsed" << NiceBytes(State.DiskUsed);
+			Response << "DiskFree" << NiceBytes(State.DiskFree);
+
+			Response.BeginObject("FullGC");
+			{
+				Response << "LastTime" << fmt::format("{}", State.LastFullGcTime);
+				Response << "TimeToNext" << SecondsToString(State.RemainingTimeUntilFullGc);
+				if (State.Config.DiskSizeSoftLimit != 0)
+				{
+					Response << "SpaceToNext" << NiceBytes(State.RemainingSpaceUntilFullGC);
+				}
+				Response << "LastDuration" << SecondsToString(State.LastFullGcDuration);
+				Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize);
+				Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize);
+			}
+			Response.EndObject();
+			Response.BeginObject("LightweightGC");
+			{
+				Response << "LastTime" << fmt::format("{}", State.LastLightweightGcTime);
+				Response << "TimeToNext" << SecondsToString(State.RemainingTimeUntilLightweightGc);
+				Response << "LastDuration" << SecondsToString(State.LastLightweightGcDuration);
+				Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize);
+				Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize);
+			}
+			Response.EndObject();
 			Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save());
 		},
 		HttpVerb::kGet);
@@ -379,8 +435,9 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 			Obj.AddString("loglevel", std::string_view(LogLevel.data(), LogLevel.size()));
 			Obj.AddString("Logfile", PathToUtf8(m_LogPaths.AbsLogPath));
 			Obj.BeginObject("cache");
+			if (m_CacheStore)
 			{
-				const ZenCacheStore::Configuration& CacheConfig = m_CacheStore.GetConfiguration();
+				const ZenCacheStore::Configuration& CacheConfig = m_CacheStore->GetConfiguration();
 				Obj.AddString("Logfile", PathToUtf8(m_LogPaths.CacheLogPath));
 				Obj.AddBool("Write", CacheConfig.Logging.EnableWriteLog);
 				Obj.AddBool("Access", CacheConfig.Logging.EnableAccessLog);
@@ -398,27 +455,30 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 	m_Router.RegisterRoute(
 		"logs",
 		[this](HttpRouterRequest& Req) {
-			HttpServerRequest&						HttpReq			  = Req.ServerRequest();
-			const HttpServerRequest::QueryParams	Params			  = HttpReq.GetQueryParams();
-			bool									SetCacheLogConfig = false;
-			ExtendableStringBuilder<256>			StringBuilder;
-			ZenCacheStore::Configuration::LogConfig LoggingConfig = m_CacheStore.GetConfiguration().Logging;
-			if (std::string Param(Params.GetValue("cacheenablewritelog")); Param.empty() == false)
-			{
-				LoggingConfig.EnableWriteLog = StrCaseCompare(Param.c_str(), "true") == 0;
-				SetCacheLogConfig			 = true;
-			}
-			if (std::string Param(Params.GetValue("cacheenableaccesslog")); Param.empty() == false)
-			{
-				LoggingConfig.EnableAccessLog = StrCaseCompare(Param.c_str(), "true") == 0;
-				SetCacheLogConfig			  = true;
-			}
-			if (SetCacheLogConfig)
+			HttpServerRequest&					 HttpReq		   = Req.ServerRequest();
+			const HttpServerRequest::QueryParams Params			   = HttpReq.GetQueryParams();
+			bool								 SetCacheLogConfig = false;
+			ExtendableStringBuilder<256>		 StringBuilder;
+			if (m_CacheStore)
 			{
-				m_CacheStore.SetLoggingConfig(LoggingConfig);
-				StringBuilder.Append(fmt::format("cache write log: {}, cache access log: {}",
-												 LoggingConfig.EnableWriteLog ? "true" : "false",
-												 LoggingConfig.EnableAccessLog ? "true" : "false"));
+				ZenCacheStore::Configuration::LogConfig LoggingConfig = m_CacheStore->GetConfiguration().Logging;
+				if (std::string Param(Params.GetValue("cacheenablewritelog")); Param.empty() == false)
+				{
+					LoggingConfig.EnableWriteLog = StrCaseCompare(Param.c_str(), "true") == 0;
+					SetCacheLogConfig			 = true;
+				}
+				if (std::string Param(Params.GetValue("cacheenableaccesslog")); Param.empty() == false)
+				{
+					LoggingConfig.EnableAccessLog = StrCaseCompare(Param.c_str(), "true") == 0;
+					SetCacheLogConfig			  = true;
+				}
+				if (SetCacheLogConfig)
+				{
+					m_CacheStore->SetLoggingConfig(LoggingConfig);
+					StringBuilder.Append(fmt::format("cache write log: {}, cache access log: {}",
+													 LoggingConfig.EnableWriteLog ? "true" : "false",
+													 LoggingConfig.EnableAccessLog ? "true" : "false"));
+				}
 			}
 			if (std::string Param(Params.GetValue("loglevel")); Param.empty() == false)
 			{
@@ -441,6 +501,25 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 			return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, StringBuilder.ToView());
 		},
 		HttpVerb::kPost);
+	m_Router.RegisterRoute(
+		"flush",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest& HttpReq = Req.ServerRequest();
+			if (m_CidStore)
+			{
+				m_CidStore->Flush();
+			}
+			if (m_CacheStore)
+			{
+				m_CacheStore->Flush();
+			}
+			if (m_ProjectStore)
+			{
+				m_ProjectStore->Flush();
+			}
+			HttpReq.WriteResponse(HttpResponseCode::OK);
+		},
+		HttpVerb::kPost);
 }
 
 HttpAdminService::~HttpAdminService()
diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h
index 12a47f29e..9d8bdfe50 100644
--- a/src/zenserver/admin/admin.h
+++ b/src/zenserver/admin/admin.h
@@ -10,6 +10,8 @@ namespace zen {
 class GcScheduler;
 class JobQueue;
 class ZenCacheStore;
+class CidStore;
+class ProjectStore;
 
 class HttpAdminService : public zen::HttpService
 {
@@ -20,7 +22,12 @@ public:
 		std::filesystem::path HttpLogPath;
 		std::filesystem::path CacheLogPath;
 	};
-	HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue, ZenCacheStore& CacheStore, const LogPaths& LogPaths);
+	HttpAdminService(GcScheduler&	 Scheduler,
+					 JobQueue&		 BackgroundJobQueue,
+					 ZenCacheStore*	 CacheStore,
+					 CidStore*		 CidStore,
+					 ProjectStore*	 ProjectStore,
+					 const LogPaths& LogPaths);
 	~HttpAdminService();
 
 	virtual const char* BaseUri() const override;
@@ -30,7 +37,9 @@ private:
 	HttpRequestRouter m_Router;
 	GcScheduler&	  m_GcScheduler;
 	JobQueue&		  m_BackgroundJobQueue;
-	ZenCacheStore&	  m_CacheStore;
+	ZenCacheStore*	  m_CacheStore;
+	CidStore*		  m_CidStore;
+	ProjectStore*	  m_ProjectStore;
 	LogPaths		  m_LogPaths;
 };
 
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 48463fcd8..05b4c2e58 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -278,8 +278,7 @@ ZenCacheStore::ZenCacheStore(GcManager&				 Gc,
 							 JobQueue&				 JobQueue,
 							 const Configuration&	 Configuration,
 							 const DiskWriteBlocker* InDiskWriteBlocker)
-: m_Log(logging::Get("z$"))
-, m_DiskWriteBlocker(InDiskWriteBlocker)
+: m_DiskWriteBlocker(InDiskWriteBlocker)
 , m_Gc(Gc)
 , m_JobQueue(JobQueue)
 , m_Configuration(Configuration)
@@ -288,7 +287,7 @@ ZenCacheStore::ZenCacheStore(GcManager&				 Gc,
 	SetLoggingConfig(m_Configuration.Logging);
 	CreateDirectories(m_Configuration.BasePath);
 
-	ZEN_INFO("Initializing at '{}'", m_Configuration.BasePath);
+	ZEN_INFO("initializing cache store at '{}'", m_Configuration.BasePath);
 
 	DirectoryContent DirContent;
 	GetDirectoryContent(m_Configuration.BasePath, DirectoryContent::IncludeDirsFlag, DirContent);
@@ -329,6 +328,7 @@ ZenCacheStore::ZenCacheStore(GcManager&				 Gc,
 
 ZenCacheStore::~ZenCacheStore()
 {
+	ZEN_INFO("closing cache store at '{}'", m_Configuration.BasePath);
 	SetLoggingConfig({.EnableWriteLog = false, .EnableAccessLog = false});
 	m_Namespaces.clear();
 }
@@ -338,6 +338,10 @@ ZenCacheStore::LogWorker()
 {
 	SetCurrentThreadName("ZenCacheStore::LogWorker");
 
+	spdlog::logger& ZCacheLog(logging::Get("z$"));
+
+	auto Log = [&ZCacheLog]() { return ZCacheLog; };
+
 	std::vector<AccessLogItem> Items;
 	while (true)
 	{
@@ -556,6 +560,7 @@ ZenCacheStore::DropNamespace(std::string_view InNamespace)
 void
 ZenCacheStore::Flush()
 {
+	ZEN_INFO("flushing cache store at '{}'", m_Configuration.BasePath);
 	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Flush(); });
 }
 
diff --git a/src/zenserver/cache/structuredcachestore.h b/src/zenserver/cache/structuredcachestore.h
index 672858fa0..c67b23e93 100644
--- a/src/zenserver/cache/structuredcachestore.h
+++ b/src/zenserver/cache/structuredcachestore.h
@@ -225,8 +225,6 @@ private:
 
 	typedef std::unordered_map<std::string, std::unique_ptr<ZenCacheNamespace>> NamespaceMap;
 
-	spdlog::logger&									m_Log;
-	spdlog::logger&									Log() { return m_Log; }
 	const DiskWriteBlocker*							m_DiskWriteBlocker = nullptr;
 	mutable RwLock									m_NamespacesLock;
 	NamespaceMap									m_Namespaces;
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 2566d8a3c..e08d36583 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -1165,14 +1165,14 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 					   "",
 					   "cache-write-log",
 					   "Whether cache write log is enabled",
-					   cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.WriteLogEnabled)->default_value("true"),
+					   cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.WriteLogEnabled)->default_value("false"),
 					   "");
 
 	options.add_option("cache",
 					   "",
 					   "cache-access-log",
 					   "Whether cache access log is enabled",
-					   cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.AccessLogEnabled)->default_value("true"),
+					   cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.AccessLogEnabled)->default_value("false"),
 					   "");
 
 	options.add_option("cache",
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 8faad58b6..0010f09f5 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1672,13 +1672,13 @@ ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcMa
 , m_ProjectBasePath(BasePath)
 , m_DiskWriteBlocker(Gc.GetDiskWriteBlocker())
 {
-	ZEN_INFO("initializing project store at '{}'", BasePath);
+	ZEN_INFO("initializing project store at '{}'", m_ProjectBasePath);
 	// m_Log.set_level(spdlog::level::debug);
 }
 
 ProjectStore::~ProjectStore()
 {
-	ZEN_INFO("closing project store ('{}')", m_ProjectBasePath);
+	ZEN_INFO("closing project store at '{}'", m_ProjectBasePath);
 }
 
 std::filesystem::path
@@ -1719,6 +1719,7 @@ ProjectStore::IterateProjects(std::function<void(Project& Prj)>&& Fn)
 void
 ProjectStore::Flush()
 {
+	ZEN_INFO("flushing project store at '{}'", m_ProjectBasePath);
 	std::vector<Ref<Project>> Projects;
 	{
 		RwLock::SharedLockScope _(m_ProjectsLock);
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 60950dbe3..152a47584 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -289,7 +289,9 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 	m_AdminService =
 		std::make_unique<HttpAdminService>(m_GcScheduler,
 										   *m_JobQueue,
-										   *m_CacheStore,
+										   m_CacheStore.Get(),
+										   m_CidStore.get(),
+										   m_ProjectStore,
 										   HttpAdminService::LogPaths{.AbsLogPath	= ServerOptions.AbsLogFile,
 																	  .HttpLogPath	= ServerOptions.DataDir / "logs" / "http.log",
 																	  .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"});
diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp
index 9446e2152..fc549a729 100644
--- a/src/zenstore/cas.cpp
+++ b/src/zenstore/cas.cpp
@@ -83,6 +83,7 @@ CasImpl::CasImpl(GcManager& Gc) : m_TinyStrategy(Gc), m_SmallStrategy(Gc), m_Lar
 
 CasImpl::~CasImpl()
 {
+	ZEN_INFO("closing Cas pool at '{}'", m_Config.RootDirectory);
 }
 
 void
@@ -259,6 +260,7 @@ CasImpl::FilterChunks(HashKeySet& InOutChunks)
 void
 CasImpl::Flush()
 {
+	ZEN_INFO("flushing CAS pool at '{}'", m_Config.RootDirectory);
 	m_SmallStrategy.Flush();
 	m_TinyStrategy.Flush();
 	m_LargeStrategy.Flush();
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 0f3f178d6..0e4c49f65 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -39,19 +39,6 @@
 #	include <random>
 #endif
 
-template<>
-struct fmt::formatter<zen::GcClock::TimePoint> : formatter<string_view>
-{
-	template<typename FormatContext>
-	auto format(const zen::GcClock::TimePoint& TimePoint, FormatContext& ctx)
-	{
-		std::time_t Time = std::chrono::system_clock::to_time_t(TimePoint);
-		char		TimeString[std::size("yyyy-mm-ddThh:mm:ss")];
-		std::strftime(std::data(TimeString), std::size(TimeString), "%FT%T", std::localtime(&Time));
-		return fmt::format_to(ctx.out(), "{}", TimeString);
-	}
-};
-
 namespace zen {
 
 using namespace std::literals;
@@ -403,11 +390,13 @@ GcManager::ScrubStorage(ScrubContext& GcCtx)
 	}
 }
 
-void
+GcStorageSize
 GcManager::CollectGarbage(GcContext& GcCtx)
 {
 	ZEN_TRACE_CPU("Gc::CollectGarbage");
 
+	GcStorageSize GCTotalSizeDiff;
+
 	RwLock::SharedLockScope _(m_Lock);
 
 	// First gather reference set
@@ -425,14 +414,13 @@ GcManager::CollectGarbage(GcContext& GcCtx)
 	{
 		ZEN_TRACE_CPU("Gc::CollectGarbage::CollectGarbage");
 
-		GcStorageSize GCTotalSizeDiff;
-		Stopwatch	  Timer;
-		const auto	  Guard = MakeGuard([&] {
-			   ZEN_INFO("collected garbage in {}. Removed {} disk space, {} memory",
-						NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
-						NiceBytes(GCTotalSizeDiff.DiskSize),
-						NiceBytes(GCTotalSizeDiff.MemorySize));
-		   });
+		Stopwatch  Timer;
+		const auto Guard = MakeGuard([&] {
+			ZEN_INFO("collected garbage in {}. Removed {} disk space, {} memory",
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+					 NiceBytes(GCTotalSizeDiff.DiskSize),
+					 NiceBytes(GCTotalSizeDiff.MemorySize));
+		});
 		for (GcStorage* Storage : m_GcStorage)
 		{
 			const auto PreSize = Storage->StorageSize();
@@ -442,6 +430,7 @@ GcManager::CollectGarbage(GcContext& GcCtx)
 			GCTotalSizeDiff.MemorySize += PreSize.MemorySize > PostSize.MemorySize ? PreSize.MemorySize - PostSize.MemorySize : 0;
 		}
 	}
+	return GCTotalSizeDiff;
 }
 
 GcStorageSize
@@ -753,6 +742,70 @@ GcScheduler::CheckDiskSpace()
 	return Space;
 }
 
+GcSchedulerState
+GcScheduler::GetState() const
+{
+	GcClock::TimePoint Now = GcClock::Now();
+
+	const GcStorageSize TotalSize = m_GcManager.TotalStorageSize();
+
+	GcSchedulerState Result{.Status = Status(), .Config = m_Config, .AreDiskWritesBlocked = m_AreDiskWritesBlocked.load()};
+
+	{
+		std::unique_lock Lock(m_GcMutex);
+		Result.LastFullGcTime		 = m_LastGcTime;
+		Result.LastFullGCDiff		 = m_LastFullGCDiff;
+		Result.LastLightweightGcTime = m_LastLightweightGcTime;
+		Result.LastLightweightGCDiff = m_LastLightweightGCDiff;
+	}
+	std::error_code Ec;
+	DiskSpace		Space = DiskSpaceInfo(Result.Config.RootDirectory, Ec);
+	if (!Ec)
+	{
+		Result.DiskSize = Space.Total;
+		Result.DiskUsed = Space.Total - Space.Free;
+		Result.DiskFree = Space.Free;
+		if (Result.Config.DiskSizeSoftLimit != 0)
+		{
+			Result.RemainingSpaceUntilFullGC =
+				Result.Config.DiskSizeSoftLimit > TotalSize.DiskSize ? (Result.Config.DiskSizeSoftLimit - TotalSize.DiskSize) : 0;
+		}
+	}
+	if (Result.Config.DiskReserveSize != 0)
+	{
+		Ec.clear();
+		Result.HasDiskReserve = std::filesystem::is_regular_file(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec;
+	}
+
+	GcClock::TimePoint CacheExpireTime =
+		Result.Config.MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - Result.Config.MaxCacheDuration;
+	GcClock::TimePoint ProjectStoreExpireTime = Result.Config.MaxProjectStoreDuration == GcClock::Duration::max()
+													? GcClock::TimePoint::min()
+													: Now - Result.Config.MaxProjectStoreDuration;
+
+	Result.RemainingTimeUntilFullGc =
+		Result.Config.Interval.count() == 0
+			? std::chrono::seconds::max()
+			: std::chrono::duration_cast<std::chrono::seconds>(Result.LastFullGcTime + Result.Config.Interval - Now);
+
+	if (Result.RemainingTimeUntilFullGc < std::chrono::seconds::zero())
+	{
+		Result.RemainingTimeUntilFullGc = std::chrono::seconds::zero();
+	}
+
+	Result.RemainingTimeUntilLightweightGc =
+		Result.Config.LightweightInterval.count() == 0
+			? std::chrono::seconds::max()
+			: std::chrono::duration_cast<std::chrono::seconds>(Result.LastLightweightGcTime + Result.Config.LightweightInterval - Now);
+
+	if (Result.RemainingTimeUntilLightweightGc < std::chrono::seconds::zero())
+	{
+		Result.RemainingTimeUntilLightweightGc = std::chrono::seconds::zero();
+	}
+
+	return Result;
+}
+
 void
 GcScheduler::SchedulerThread()
 {
@@ -1159,11 +1212,24 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 		Stopwatch  Timer;
 		const auto __ = MakeGuard([&] { ZEN_INFO("garbage collection DONE in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
 
-		m_GcManager.CollectGarbage(GcCtx);
+		GcStorageSize Diff = m_GcManager.CollectGarbage(GcCtx);
 
 		if (SkipCid)
 		{
 			m_LastLightweightGcTime = GcClock::Now();
+
+			std::chrono::seconds ElapsedSeconds =
+				std::chrono::duration_cast<std::chrono::seconds>(std::chrono::milliseconds(Timer.GetElapsedTimeMs()));
+			if (SkipCid)
+			{
+				m_LastLightweightGcDuration = ElapsedSeconds;
+				m_LastLightweightGCDiff		= Diff;
+			}
+			else
+			{
+				m_LastFullGcDuration = ElapsedSeconds;
+				m_LastFullGCDiff	 = Diff;
+			}
 		}
 		else
 		{
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index 2e52218f8..aaa156ebf 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -7,6 +7,10 @@
 #include <zenstore/caslog.h>
 #include <zenstore/scrubcontext.h>
 
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
 #include <atomic>
 #include <chrono>
 #include <condition_variable>
@@ -149,8 +153,8 @@ public:
 	void AddGcStorage(GcStorage* Contributor);
 	void RemoveGcStorage(GcStorage* Contributor);
 
-	void CollectGarbage(GcContext& GcCtx);
-	void ScrubStorage(ScrubContext& GcCtx);
+	GcStorageSize CollectGarbage(GcContext& GcCtx);
+	void		  ScrubStorage(ScrubContext& GcCtx);
 
 	GcStorageSize TotalStorageSize() const;
 
@@ -189,6 +193,29 @@ struct GcSchedulerConfig
 	std::chrono::seconds  LightweightInterval{};
 };
 
+struct GcSchedulerState
+{
+	GcSchedulerStatus Status;
+
+	GcSchedulerConfig Config;
+
+	bool				 AreDiskWritesBlocked = false;
+	bool				 HasDiskReserve		  = false;
+	uint64_t			 DiskSize			  = 0;
+	uint64_t			 DiskUsed			  = 0;
+	uint64_t			 DiskFree			  = 0;
+	GcClock::TimePoint	 LastFullGcTime{};
+	GcClock::TimePoint	 LastLightweightGcTime{};
+	std::chrono::seconds RemainingTimeUntilLightweightGc;
+	std::chrono::seconds RemainingTimeUntilFullGc;
+	uint64_t			 RemainingSpaceUntilFullGC = 0;
+
+	std::chrono::seconds LastFullGcDuration{};
+	GcStorageSize		 LastFullGCDiff;
+	std::chrono::seconds LastLightweightGcDuration{};
+	GcStorageSize		 LastLightweightGCDiff;
+};
+
 class DiskUsageWindow
 {
 public:
@@ -221,6 +248,7 @@ public:
 	void			  Initialize(const GcSchedulerConfig& Config);
 	void			  Shutdown();
 	GcSchedulerStatus Status() const { return static_cast<GcSchedulerStatus>(m_Status.load()); }
+	GcSchedulerState  GetState() const;
 
 	struct TriggerGcParams
 	{
@@ -253,15 +281,21 @@ private:
 	virtual bool	AreDiskWritesAllowed() const override { return !m_AreDiskWritesBlocked.load(); }
 	DiskSpace		CheckDiskSpace();
 
-	spdlog::logger&					  m_Log;
-	GcManager&						  m_GcManager;
-	GcSchedulerConfig				  m_Config;
-	GcClock::TimePoint				  m_LastGcTime{};
-	GcClock::TimePoint				  m_LastLightweightGcTime{};
-	GcClock::TimePoint				  m_LastGcExpireTime{};
+	spdlog::logger&	   m_Log;
+	GcManager&		   m_GcManager;
+	GcSchedulerConfig  m_Config;
+	GcClock::TimePoint m_LastGcTime{};
+	GcClock::TimePoint m_LastLightweightGcTime{};
+	GcClock::TimePoint m_LastGcExpireTime{};
+
+	std::chrono::seconds m_LastFullGcDuration{};
+	GcStorageSize		 m_LastFullGCDiff;
+	std::chrono::seconds m_LastLightweightGcDuration{};
+	GcStorageSize		 m_LastLightweightGCDiff;
+
 	std::atomic_uint32_t			  m_Status{};
 	std::thread						  m_GcThread;
-	std::mutex						  m_GcMutex;
+	mutable std::mutex				  m_GcMutex;
 	std::condition_variable			  m_GcSignal;
 	std::optional<TriggerGcParams>	  m_TriggerGcParams;
 	std::optional<TriggerScrubParams> m_TriggerScrubParams;
@@ -274,3 +308,16 @@ private:
 void gc_forcelink();
 
 }  // namespace zen
+
+template<>
+struct fmt::formatter<zen::GcClock::TimePoint> : formatter<string_view>
+{
+	template<typename FormatContext>
+	auto format(const zen::GcClock::TimePoint& TimePoint, FormatContext& ctx)
+	{
+		std::time_t Time = std::chrono::system_clock::to_time_t(TimePoint);
+		char		TimeString[std::size("yyyy-mm-ddThh:mm:ss")];
+		std::strftime(std::data(TimeString), std::size(TimeString), "%FT%T", std::localtime(&Time));
+		return fmt::format_to(ctx.out(), "{}", TimeString);
+	}
+};