add `flush` command and more gc status info (#483)

- Feature: New endpoint `/admin/flush ` to flush all storage - CAS, Cache and ProjectStore
- Feature: New command `zen flush` to flush all storage - CAS, Cache and ProjectStore
- Improved: Command `zen gc-status` now gives details about storage, when last GC occured, how long until next GC etc
- Changed: Cache access and write log are disabled by default

7 files changed, 130 insertions, 36 deletions

diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index 60599a992..5313a7592 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -3,8 +3,10 @@
 #include "admin.h"
 
 #include <zencore/compactbinarybuilder.h>
+#include <zencore/fmtutils.h>
 #include <zencore/jobqueue.h>
 #include <zencore/string.h>
+
 #if ZEN_WITH_TRACE
 #	include <zencore/trace.h>
 #endif	// ZEN_WITH_TRACE
@@ -13,8 +15,11 @@
 #	include <mimalloc.h>
 #endif
 
+#include <zenstore/cidstore.h>
 #include <zenstore/gc.h>
+
 #include "cache/structuredcachestore.h"
+#include "projectstore/projectstore.h"
 
 #include <chrono>
 
@@ -26,11 +31,15 @@ namespace zen {
 
 HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 								   JobQueue&	   BackgroundJobQueue,
-								   ZenCacheStore&  CacheStore,
+								   ZenCacheStore*  CacheStore,
+								   CidStore*	   CidStore,
+								   ProjectStore*   ProjectStore,
 								   const LogPaths& LogPaths)
 : m_GcScheduler(Scheduler)
 , m_BackgroundJobQueue(BackgroundJobQueue)
 , m_CacheStore(CacheStore)
+, m_CidStore(CidStore)
+, m_ProjectStore(ProjectStore)
 , m_LogPaths(LogPaths)
 {
 	using namespace std::literals;
@@ -188,10 +197,57 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 	m_Router.RegisterRoute(
 		"gc",
 		[this](HttpRouterRequest& Req) {
-			const GcSchedulerStatus Status = m_GcScheduler.Status();
+			const GcSchedulerState State = m_GcScheduler.GetState();
+
+			auto SecondsToString = [](std::chrono::seconds Secs) {
+				return NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(Secs).count()));
+			};
 
 			CbObjectWriter Response;
-			Response << "Status"sv << (GcSchedulerStatus::kIdle == Status ? "Idle"sv : "Running"sv);
+			Response << "Status"sv << (GcSchedulerStatus::kIdle == State.Status ? "Idle"sv : "Running"sv);
+			Response.BeginObject("Config");
+			{
+				Response << "RootDirectory" << State.Config.RootDirectory.string();
+				Response << "MonitorInterval" << SecondsToString(State.Config.MonitorInterval);
+				Response << "Interval" << SecondsToString(State.Config.Interval);
+				Response << "MaxCacheDuration" << SecondsToString(State.Config.MaxCacheDuration);
+				Response << "MaxProjectStoreDuration" << SecondsToString(State.Config.MaxProjectStoreDuration);
+				Response << "CollectSmallObjects" << State.Config.CollectSmallObjects;
+				Response << "Enabled" << State.Config.Enabled;
+				Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize);
+				Response << "DiskSizeSoftLimit" << NiceBytes(State.Config.DiskSizeSoftLimit);
+				Response << "MinimumFreeDiskSpaceToAllowWrites" << NiceBytes(State.Config.MinimumFreeDiskSpaceToAllowWrites);
+				Response << "LightweightInterval" << SecondsToString(State.Config.LightweightInterval);
+			}
+			Response.EndObject();
+			Response << "AreDiskWritesBlocked" << State.AreDiskWritesBlocked;
+			Response << "HasDiskReserve" << State.HasDiskReserve;
+			Response << "DiskSize" << NiceBytes(State.DiskSize);
+			Response << "DiskUsed" << NiceBytes(State.DiskUsed);
+			Response << "DiskFree" << NiceBytes(State.DiskFree);
+
+			Response.BeginObject("FullGC");
+			{
+				Response << "LastTime" << fmt::format("{}", State.LastFullGcTime);
+				Response << "TimeToNext" << SecondsToString(State.RemainingTimeUntilFullGc);
+				if (State.Config.DiskSizeSoftLimit != 0)
+				{
+					Response << "SpaceToNext" << NiceBytes(State.RemainingSpaceUntilFullGC);
+				}
+				Response << "LastDuration" << SecondsToString(State.LastFullGcDuration);
+				Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize);
+				Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize);
+			}
+			Response.EndObject();
+			Response.BeginObject("LightweightGC");
+			{
+				Response << "LastTime" << fmt::format("{}", State.LastLightweightGcTime);
+				Response << "TimeToNext" << SecondsToString(State.RemainingTimeUntilLightweightGc);
+				Response << "LastDuration" << SecondsToString(State.LastLightweightGcDuration);
+				Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize);
+				Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize);
+			}
+			Response.EndObject();
 			Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save());
 		},
 		HttpVerb::kGet);
@@ -379,8 +435,9 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 			Obj.AddString("loglevel", std::string_view(LogLevel.data(), LogLevel.size()));
 			Obj.AddString("Logfile", PathToUtf8(m_LogPaths.AbsLogPath));
 			Obj.BeginObject("cache");
+			if (m_CacheStore)
 			{
-				const ZenCacheStore::Configuration& CacheConfig = m_CacheStore.GetConfiguration();
+				const ZenCacheStore::Configuration& CacheConfig = m_CacheStore->GetConfiguration();
 				Obj.AddString("Logfile", PathToUtf8(m_LogPaths.CacheLogPath));
 				Obj.AddBool("Write", CacheConfig.Logging.EnableWriteLog);
 				Obj.AddBool("Access", CacheConfig.Logging.EnableAccessLog);
@@ -398,27 +455,30 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 	m_Router.RegisterRoute(
 		"logs",
 		[this](HttpRouterRequest& Req) {
-			HttpServerRequest&						HttpReq			  = Req.ServerRequest();
-			const HttpServerRequest::QueryParams	Params			  = HttpReq.GetQueryParams();
-			bool									SetCacheLogConfig = false;
-			ExtendableStringBuilder<256>			StringBuilder;
-			ZenCacheStore::Configuration::LogConfig LoggingConfig = m_CacheStore.GetConfiguration().Logging;
-			if (std::string Param(Params.GetValue("cacheenablewritelog")); Param.empty() == false)
-			{
-				LoggingConfig.EnableWriteLog = StrCaseCompare(Param.c_str(), "true") == 0;
-				SetCacheLogConfig			 = true;
-			}
-			if (std::string Param(Params.GetValue("cacheenableaccesslog")); Param.empty() == false)
-			{
-				LoggingConfig.EnableAccessLog = StrCaseCompare(Param.c_str(), "true") == 0;
-				SetCacheLogConfig			  = true;
-			}
-			if (SetCacheLogConfig)
+			HttpServerRequest&					 HttpReq		   = Req.ServerRequest();
+			const HttpServerRequest::QueryParams Params			   = HttpReq.GetQueryParams();
+			bool								 SetCacheLogConfig = false;
+			ExtendableStringBuilder<256>		 StringBuilder;
+			if (m_CacheStore)
 			{
-				m_CacheStore.SetLoggingConfig(LoggingConfig);
-				StringBuilder.Append(fmt::format("cache write log: {}, cache access log: {}",
-												 LoggingConfig.EnableWriteLog ? "true" : "false",
-												 LoggingConfig.EnableAccessLog ? "true" : "false"));
+				ZenCacheStore::Configuration::LogConfig LoggingConfig = m_CacheStore->GetConfiguration().Logging;
+				if (std::string Param(Params.GetValue("cacheenablewritelog")); Param.empty() == false)
+				{
+					LoggingConfig.EnableWriteLog = StrCaseCompare(Param.c_str(), "true") == 0;
+					SetCacheLogConfig			 = true;
+				}
+				if (std::string Param(Params.GetValue("cacheenableaccesslog")); Param.empty() == false)
+				{
+					LoggingConfig.EnableAccessLog = StrCaseCompare(Param.c_str(), "true") == 0;
+					SetCacheLogConfig			  = true;
+				}
+				if (SetCacheLogConfig)
+				{
+					m_CacheStore->SetLoggingConfig(LoggingConfig);
+					StringBuilder.Append(fmt::format("cache write log: {}, cache access log: {}",
+													 LoggingConfig.EnableWriteLog ? "true" : "false",
+													 LoggingConfig.EnableAccessLog ? "true" : "false"));
+				}
 			}
 			if (std::string Param(Params.GetValue("loglevel")); Param.empty() == false)
 			{
@@ -441,6 +501,25 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 			return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, StringBuilder.ToView());
 		},
 		HttpVerb::kPost);
+	m_Router.RegisterRoute(
+		"flush",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest& HttpReq = Req.ServerRequest();
+			if (m_CidStore)
+			{
+				m_CidStore->Flush();
+			}
+			if (m_CacheStore)
+			{
+				m_CacheStore->Flush();
+			}
+			if (m_ProjectStore)
+			{
+				m_ProjectStore->Flush();
+			}
+			HttpReq.WriteResponse(HttpResponseCode::OK);
+		},
+		HttpVerb::kPost);
 }
 
 HttpAdminService::~HttpAdminService()
diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h
index 12a47f29e..9d8bdfe50 100644
--- a/src/zenserver/admin/admin.h
+++ b/src/zenserver/admin/admin.h
@@ -10,6 +10,8 @@ namespace zen {
 class GcScheduler;
 class JobQueue;
 class ZenCacheStore;
+class CidStore;
+class ProjectStore;
 
 class HttpAdminService : public zen::HttpService
 {
@@ -20,7 +22,12 @@ public:
 		std::filesystem::path HttpLogPath;
 		std::filesystem::path CacheLogPath;
 	};
-	HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue, ZenCacheStore& CacheStore, const LogPaths& LogPaths);
+	HttpAdminService(GcScheduler&	 Scheduler,
+					 JobQueue&		 BackgroundJobQueue,
+					 ZenCacheStore*	 CacheStore,
+					 CidStore*		 CidStore,
+					 ProjectStore*	 ProjectStore,
+					 const LogPaths& LogPaths);
 	~HttpAdminService();
 
 	virtual const char* BaseUri() const override;
@@ -30,7 +37,9 @@ private:
 	HttpRequestRouter m_Router;
 	GcScheduler&	  m_GcScheduler;
 	JobQueue&		  m_BackgroundJobQueue;
-	ZenCacheStore&	  m_CacheStore;
+	ZenCacheStore*	  m_CacheStore;
+	CidStore*		  m_CidStore;
+	ProjectStore*	  m_ProjectStore;
 	LogPaths		  m_LogPaths;
 };
 
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 48463fcd8..05b4c2e58 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -278,8 +278,7 @@ ZenCacheStore::ZenCacheStore(GcManager&				 Gc,
 							 JobQueue&				 JobQueue,
 							 const Configuration&	 Configuration,
 							 const DiskWriteBlocker* InDiskWriteBlocker)
-: m_Log(logging::Get("z$"))
-, m_DiskWriteBlocker(InDiskWriteBlocker)
+: m_DiskWriteBlocker(InDiskWriteBlocker)
 , m_Gc(Gc)
 , m_JobQueue(JobQueue)
 , m_Configuration(Configuration)
@@ -288,7 +287,7 @@ ZenCacheStore::ZenCacheStore(GcManager&				 Gc,
 	SetLoggingConfig(m_Configuration.Logging);
 	CreateDirectories(m_Configuration.BasePath);
 
-	ZEN_INFO("Initializing at '{}'", m_Configuration.BasePath);
+	ZEN_INFO("initializing cache store at '{}'", m_Configuration.BasePath);
 
 	DirectoryContent DirContent;
 	GetDirectoryContent(m_Configuration.BasePath, DirectoryContent::IncludeDirsFlag, DirContent);
@@ -329,6 +328,7 @@ ZenCacheStore::ZenCacheStore(GcManager&				 Gc,
 
 ZenCacheStore::~ZenCacheStore()
 {
+	ZEN_INFO("closing cache store at '{}'", m_Configuration.BasePath);
 	SetLoggingConfig({.EnableWriteLog = false, .EnableAccessLog = false});
 	m_Namespaces.clear();
 }
@@ -338,6 +338,10 @@ ZenCacheStore::LogWorker()
 {
 	SetCurrentThreadName("ZenCacheStore::LogWorker");
 
+	spdlog::logger& ZCacheLog(logging::Get("z$"));
+
+	auto Log = [&ZCacheLog]() { return ZCacheLog; };
+
 	std::vector<AccessLogItem> Items;
 	while (true)
 	{
@@ -556,6 +560,7 @@ ZenCacheStore::DropNamespace(std::string_view InNamespace)
 void
 ZenCacheStore::Flush()
 {
+	ZEN_INFO("flushing cache store at '{}'", m_Configuration.BasePath);
 	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.Flush(); });
 }
 
diff --git a/src/zenserver/cache/structuredcachestore.h b/src/zenserver/cache/structuredcachestore.h
index 672858fa0..c67b23e93 100644
--- a/src/zenserver/cache/structuredcachestore.h
+++ b/src/zenserver/cache/structuredcachestore.h
@@ -225,8 +225,6 @@ private:
 
 	typedef std::unordered_map<std::string, std::unique_ptr<ZenCacheNamespace>> NamespaceMap;
 
-	spdlog::logger&									m_Log;
-	spdlog::logger&									Log() { return m_Log; }
 	const DiskWriteBlocker*							m_DiskWriteBlocker = nullptr;
 	mutable RwLock									m_NamespacesLock;
 	NamespaceMap									m_Namespaces;
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 2566d8a3c..e08d36583 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -1165,14 +1165,14 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 					   "",
 					   "cache-write-log",
 					   "Whether cache write log is enabled",
-					   cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.WriteLogEnabled)->default_value("true"),
+					   cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.WriteLogEnabled)->default_value("false"),
 					   "");
 
 	options.add_option("cache",
 					   "",
 					   "cache-access-log",
 					   "Whether cache access log is enabled",
-					   cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.AccessLogEnabled)->default_value("true"),
+					   cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.AccessLogEnabled)->default_value("false"),
 					   "");
 
 	options.add_option("cache",
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 8faad58b6..0010f09f5 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1672,13 +1672,13 @@ ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcMa
 , m_ProjectBasePath(BasePath)
 , m_DiskWriteBlocker(Gc.GetDiskWriteBlocker())
 {
-	ZEN_INFO("initializing project store at '{}'", BasePath);
+	ZEN_INFO("initializing project store at '{}'", m_ProjectBasePath);
 	// m_Log.set_level(spdlog::level::debug);
 }
 
 ProjectStore::~ProjectStore()
 {
-	ZEN_INFO("closing project store ('{}')", m_ProjectBasePath);
+	ZEN_INFO("closing project store at '{}'", m_ProjectBasePath);
 }
 
 std::filesystem::path
@@ -1719,6 +1719,7 @@ ProjectStore::IterateProjects(std::function<void(Project& Prj)>&& Fn)
 void
 ProjectStore::Flush()
 {
+	ZEN_INFO("flushing project store at '{}'", m_ProjectBasePath);
 	std::vector<Ref<Project>> Projects;
 	{
 		RwLock::SharedLockScope _(m_ProjectsLock);
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 60950dbe3..152a47584 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -289,7 +289,9 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 	m_AdminService =
 		std::make_unique<HttpAdminService>(m_GcScheduler,
 										   *m_JobQueue,
-										   *m_CacheStore,
+										   m_CacheStore.Get(),
+										   m_CidStore.get(),
+										   m_ProjectStore,
 										   HttpAdminService::LogPaths{.AbsLogPath	= ServerOptions.AbsLogFile,
 																	  .HttpLogPath	= ServerOptions.DataDir / "logs" / "http.log",
 																	  .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"});