Add runtime status/control of logging (#419)

- Feature: New endpoint `/admin/logs` to query status of logging and log file locations and cache logging
  - `enablewritelog`=`true`/`false` parameter to control cache write logging
  - `enableaccesslog`=`true`/`false` parameter to control cache access logging
  - `loglevel` = `trace`/`debug`/`info`/`warning`/`error`
- Feature: New zen command `logs` to query/control zen logging
  - No arguments gives status of logging and paths to log files
  - `--cache-write-log` `enable`/`disable` to control cache write logging
  - `--cache-access-log` `enable`/`disable` to control cache access logging
  - `--loglevel` `trace`/`debug`/`info`/`warning`/`error` to set debug level

11 files changed, 277 insertions, 29 deletions

diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp
index 0aef968a9..b48207bec 100644
--- a/src/zen/cmds/admin_cmd.cpp
+++ b/src/zen/cmds/admin_cmd.cpp
@@ -267,4 +267,106 @@ JobCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	return 0;
 }
 
+////////////////////////////////////////////
+
+LoggingCommand::LoggingCommand()
+{
+	m_Options.add_options()("h,help", "Print help");
+	m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+	m_Options.add_option("", "", "cache-write-log", "Enable cache write logging", cxxopts::value(m_CacheWriteLog), "<enable/disable>");
+	m_Options.add_option("", "", "cache-access-log", "Enable cache access logging", cxxopts::value(m_CacheAccessLog), "<enable/disable>");
+	m_Options
+		.add_option("", "", "set-log-level", "Set zenserver log level", cxxopts::value(m_SetLogLevel), "<trace/debug/info/warning/error>");
+}
+
+LoggingCommand::~LoggingCommand() = default;
+
+int
+LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+	ZEN_UNUSED(GlobalOptions);
+
+	using namespace std::literals;
+
+	if (!ParseOptions(argc, argv))
+	{
+		return 0;
+	}
+
+	m_HostName = ResolveTargetHostSpec(m_HostName);
+
+	if (m_HostName.empty())
+	{
+		throw OptionParseException("unable to resolve server specification");
+	}
+
+	HttpClient Http(m_HostName);
+
+	HttpClient::KeyValueMap Parameters;
+
+	if (!m_CacheWriteLog.empty())
+	{
+		if (m_CacheWriteLog == "enable")
+		{
+			(*Parameters)["cacheenablewritelog"] = "true";
+		}
+		else if (m_CacheWriteLog == "disable")
+		{
+			(*Parameters)["cacheenablewritelog"] = "false";
+		}
+		else
+		{
+			ZEN_ERROR("Invalid value for parameter 'cache-write-log'. Use 'enable' or 'disable'");
+			return 1;
+		}
+	}
+
+	if (!m_CacheAccessLog.empty())
+	{
+		if (m_CacheAccessLog == "enable")
+		{
+			(*Parameters)["cacheenableaccesslog"] = "true";
+		}
+		else if (m_CacheAccessLog == "disable")
+		{
+			(*Parameters)["cacheenableaccesslog"] = "false";
+		}
+		else
+		{
+			ZEN_ERROR("Invalid value for parameter 'cache-access-log'. Use 'enable' or 'disable'");
+			return 1;
+		}
+	}
+
+	if (!m_SetLogLevel.empty())
+	{
+		(*Parameters)["loglevel"] = m_SetLogLevel;
+	}
+
+	if ((*Parameters).empty())
+	{
+		if (HttpClient::Response Result = Http.Get("/admin/logs", HttpClient::Accept(ZenContentType::kJSON)))
+		{
+			ZEN_CONSOLE("{}", Result.AsText());
+		}
+		else
+		{
+			Result.ThrowError("failed fetching log info"sv);
+			return 1;
+		}
+		return 0;
+	}
+	if (HttpClient::Response Result = Http.Post("/admin/logs", HttpClient::KeyValueMap{}, Parameters))
+	{
+		ZEN_CONSOLE("{}", Result.AsText());
+	}
+	else
+	{
+		Result.ThrowError("failed setting log info"sv);
+		return 1;
+	}
+
+	return 0;
+}
+
 }  // namespace zen
diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h
index 6caab7138..873c230d9 100644
--- a/src/zen/cmds/admin_cmd.h
+++ b/src/zen/cmds/admin_cmd.h
@@ -73,4 +73,23 @@ private:
 	bool			 m_Cancel = 0;
 };
 
+////////////////////////////////////////////
+
+class LoggingCommand : public ZenCmdBase
+{
+public:
+	LoggingCommand();
+	~LoggingCommand();
+
+	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+	virtual cxxopts::Options& Options() override { return m_Options; }
+
+private:
+	cxxopts::Options m_Options{"logs", "Show/control zen logging"};
+	std::string		 m_HostName;
+	std::string		 m_CacheWriteLog;
+	std::string		 m_CacheAccessLog;
+	std::string		 m_SetLogLevel;
+};
+
 }  // namespace zen
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index 23dce850f..9c664e0dc 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -226,6 +226,7 @@ main(int argc, char** argv)
 	ServeCommand			 ServeCmd;
 	SnapshotOplogCommand	 SnapshotOplogCmd;
 	StatusCommand			 StatusCmd;
+	LoggingCommand			 LoggingCmd;
 	JobCommand				 JobCmd;
 	TopCommand				 TopCmd;
 	TraceCommand			 TraceCmd;
@@ -277,6 +278,7 @@ main(int argc, char** argv)
 		{"scrub",				&ScrubCmd,				"Scrub zen storage (verify data integrity)"},
 		{"serve",				&ServeCmd,				"Serve files from a directory"},
 		{"status",				&StatusCmd,				"Show zen status"},
+		{"logs",				&LoggingCmd,			"Show/control zen logging"},
 		{"jobs",				&JobCmd,				"Show/cancel zen background jobs"},
 		{"top",					&TopCmd,				"Monitor zen server activity"},
 		{"trace",				&TraceCmd,				"Control zen realtime tracing"},
diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp
index 5ae5f78d6..caefce5f4 100644
--- a/src/zenhttp/httpclient.cpp
+++ b/src/zenhttp/httpclient.cpp
@@ -587,11 +587,11 @@ HttpClient::Delete(std::string_view Url, const KeyValueMap& AdditionalHeader)
 }
 
 HttpClient::Response
-HttpClient::Post(std::string_view Url, const KeyValueMap& AdditionalHeader)
+HttpClient::Post(std::string_view Url, const KeyValueMap& AdditionalHeader, const KeyValueMap& Parameters)
 {
 	ZEN_TRACE_CPU("HttpClient::PostNoPayload");
 
-	Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {});
+	Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, Parameters);
 
 	return CommonResponse(Sess.Post());
 }
diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h
index 2c81f4a23..044a3162e 100644
--- a/src/zenhttp/include/zenhttp/httpclient.h
+++ b/src/zenhttp/include/zenhttp/httpclient.h
@@ -128,7 +128,7 @@ public:
 	[[nodiscard]] Response Get(std::string_view Url, const KeyValueMap& AdditionalHeader = {}, const KeyValueMap& Parameters = {});
 	[[nodiscard]] Response Head(std::string_view Url, const KeyValueMap& AdditionalHeader = {});
 	[[nodiscard]] Response Delete(std::string_view Url, const KeyValueMap& AdditionalHeader = {});
-	[[nodiscard]] Response Post(std::string_view Url, const KeyValueMap& AdditionalHeader = {});
+	[[nodiscard]] Response Post(std::string_view Url, const KeyValueMap& AdditionalHeader = {}, const KeyValueMap& Parameters = {});
 	[[nodiscard]] Response Post(std::string_view Url, const IoBuffer& Payload, const KeyValueMap& AdditionalHeader = {});
 	[[nodiscard]] Response Post(std::string_view Url, CbObject Payload, const KeyValueMap& AdditionalHeader = {});
 	[[nodiscard]] Response Post(std::string_view Url, CbPackage Payload, const KeyValueMap& AdditionalHeader = {});
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index 05ae0a24a..083086f50 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -10,14 +10,24 @@
 #endif	// ZEN_WITH_TRACE
 
 #include <zenstore/gc.h>
+#include "cache/structuredcachestore.h"
 
 #include <chrono>
 
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <spdlog/spdlog.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
 namespace zen {
 
-HttpAdminService::HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue)
+HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
+								   JobQueue&	   BackgroundJobQueue,
+								   ZenCacheStore&  CacheStore,
+								   const LogPaths& LogPaths)
 : m_GcScheduler(Scheduler)
 , m_BackgroundJobQueue(BackgroundJobQueue)
+, m_CacheStore(CacheStore)
+, m_LogPaths(LogPaths)
 {
 	using namespace std::literals;
 
@@ -315,6 +325,77 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJ
 		},
 		HttpVerb::kPost);
 #endif	// ZEN_WITH_TRACE
+
+	m_Router.RegisterRoute(
+		"logs",
+		[this](HttpRouterRequest& Req) {
+			CbObjectWriter		  Obj;
+			spdlog::string_view_t LogLevel = spdlog::level::to_string_view(spdlog::get_level());
+			Obj.AddString("loglevel", std::string_view(LogLevel.data(), LogLevel.size()));
+			Obj.AddString("Logfile", PathToUtf8(m_LogPaths.AbsLogPath));
+			Obj.BeginObject("cache");
+			{
+				const ZenCacheStore::Configuration& CacheConfig = m_CacheStore.GetConfiguration();
+				Obj.AddString("Logfile", PathToUtf8(m_LogPaths.CacheLogPath));
+				Obj.AddBool("Write", CacheConfig.Logging.EnableWriteLog);
+				Obj.AddBool("Access", CacheConfig.Logging.EnableAccessLog);
+			}
+			Obj.EndObject();
+			Obj.BeginObject("http");
+			{
+				Obj.AddString("Logfile", PathToUtf8(m_LogPaths.HttpLogPath));
+			}
+			Obj.EndObject();
+			Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
+		},
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
+		"logs",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest&						HttpReq			  = Req.ServerRequest();
+			const HttpServerRequest::QueryParams	Params			  = HttpReq.GetQueryParams();
+			bool									SetCacheLogConfig = false;
+			ExtendableStringBuilder<256>			StringBuilder;
+			ZenCacheStore::Configuration::LogConfig LoggingConfig = m_CacheStore.GetConfiguration().Logging;
+			if (std::string Param(Params.GetValue("cacheenablewritelog")); Param.empty() == false)
+			{
+				LoggingConfig.EnableWriteLog = StrCaseCompare(Param.c_str(), "true") == 0;
+				SetCacheLogConfig			 = true;
+			}
+			if (std::string Param(Params.GetValue("cacheenableaccesslog")); Param.empty() == false)
+			{
+				LoggingConfig.EnableAccessLog = StrCaseCompare(Param.c_str(), "true") == 0;
+				SetCacheLogConfig			  = true;
+			}
+			if (SetCacheLogConfig)
+			{
+				m_CacheStore.SetLoggingConfig(LoggingConfig);
+				StringBuilder.Append(fmt::format("cache write log: {}, cache access log: {}",
+												 LoggingConfig.EnableWriteLog ? "true" : "false",
+												 LoggingConfig.EnableAccessLog ? "true" : "false"));
+			}
+			if (std::string Param(Params.GetValue("loglevel")); Param.empty() == false)
+			{
+				spdlog::level::level_enum NewLevel = spdlog::level::from_str(Param);
+				spdlog::string_view_t	  LogLevel = spdlog::level::to_string_view(NewLevel);
+				if (LogLevel != Param)
+				{
+					return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest,
+															 HttpContentType::kText,
+															 fmt::format("Invalid log level '{}'", Param));
+				}
+				spdlog::set_level(NewLevel);
+				if (StringBuilder.Size() > 0)
+				{
+					StringBuilder.Append(", ");
+				}
+				StringBuilder.Append("loglevel: ");
+				StringBuilder.Append(Param);
+			}
+			return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, StringBuilder.ToView());
+		},
+		HttpVerb::kPost);
 }
 
 HttpAdminService::~HttpAdminService()
diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h
index 3152f87ab..12a47f29e 100644
--- a/src/zenserver/admin/admin.h
+++ b/src/zenserver/admin/admin.h
@@ -9,11 +9,18 @@ namespace zen {
 
 class GcScheduler;
 class JobQueue;
+class ZenCacheStore;
 
 class HttpAdminService : public zen::HttpService
 {
 public:
-	HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue);
+	struct LogPaths
+	{
+		std::filesystem::path AbsLogPath;
+		std::filesystem::path HttpLogPath;
+		std::filesystem::path CacheLogPath;
+	};
+	HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue, ZenCacheStore& CacheStore, const LogPaths& LogPaths);
 	~HttpAdminService();
 
 	virtual const char* BaseUri() const override;
@@ -23,6 +30,8 @@ private:
 	HttpRequestRouter m_Router;
 	GcScheduler&	  m_GcScheduler;
 	JobQueue&		  m_BackgroundJobQueue;
+	ZenCacheStore&	  m_CacheStore;
+	LogPaths		  m_LogPaths;
 };
 
 }  // namespace zen
diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index 32e8c1f98..11ac81dcb 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -717,8 +717,12 @@ HttpStructuredCacheService::HandleCacheRequest(HttpServerRequest& Request)
 					BasePathString << Info.Config.BasePath.u8string();
 					ResponseWriter.AddString("BasePath"sv, BasePathString.ToView());
 					ResponseWriter.AddBool("AllowAutomaticCreationOfNamespaces", Info.Config.AllowAutomaticCreationOfNamespaces);
-					ResponseWriter.AddBool("EnableWriteLog", Info.Config.EnableWriteLog);
-					ResponseWriter.AddBool("EnableAccessLog", Info.Config.EnableAccessLog);
+					ResponseWriter.BeginObject("Logging");
+					{
+						ResponseWriter.AddBool("EnableWriteLog", Info.Config.Logging.EnableWriteLog);
+						ResponseWriter.AddBool("EnableAccessLog", Info.Config.Logging.EnableAccessLog);
+					}
+					ResponseWriter.EndObject();
 				}
 				ResponseWriter.EndObject();
 
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 4499b05f7..809df1a94 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -238,10 +238,7 @@ ZenCacheStore::ZenCacheStore(GcManager& Gc, const Configuration& Configuration,
 , m_Configuration(Configuration)
 , m_ExitLogging(false)
 {
-	if (m_Configuration.EnableAccessLog || m_Configuration.EnableWriteLog)
-	{
-		m_AsyncLoggingThread = std::thread(&ZenCacheStore::LogWorker, this);
-	}
+	SetLoggingConfig(m_Configuration.Logging);
 	CreateDirectories(m_Configuration.BasePath);
 
 	ZEN_INFO("Initializing at '{}'", m_Configuration.BasePath);
@@ -281,12 +278,7 @@ ZenCacheStore::ZenCacheStore(GcManager& Gc, const Configuration& Configuration,
 
 ZenCacheStore::~ZenCacheStore()
 {
-	m_ExitLogging.store(true);
-	m_LogEvent.Set();
-	if (m_AsyncLoggingThread.joinable())
-	{
-		m_AsyncLoggingThread.join();
-	}
+	SetLoggingConfig({.EnableWriteLog = false, .EnableAccessLog = false});
 	m_Namespaces.clear();
 }
 
@@ -381,7 +373,7 @@ ZenCacheStore::Get(const CacheRequestContext& Context,
 	{
 		bool Result = Store->Get(Bucket, HashKey, OutValue);
 
-		if (m_Configuration.EnableAccessLog)
+		if (m_AccessLogEnabled)
 		{
 			ZEN_TRACE_CPU("Z$::Get::AccessLog");
 			bool Signal = false;
@@ -421,7 +413,7 @@ ZenCacheStore::Put(const CacheRequestContext& Context,
 {
 	ZEN_TRACE_CPU("Z$::Put");
 
-	if (m_Configuration.EnableWriteLog)
+	if (m_WriteLogEnabled)
 	{
 		ZEN_TRACE_CPU("Z$::Get::WriteLog");
 		bool Signal = false;
@@ -616,6 +608,31 @@ ZenCacheStore::StorageSize() const
 	return Size;
 }
 
+void
+ZenCacheStore::SetLoggingConfig(const Configuration::LogConfig& Loggingconfig)
+{
+	if (!Loggingconfig.EnableAccessLog && !Loggingconfig.EnableWriteLog)
+	{
+		m_AccessLogEnabled.store(false);
+		m_WriteLogEnabled.store(false);
+		m_ExitLogging.store(true);
+		m_LogEvent.Set();
+		if (m_AsyncLoggingThread.joinable())
+		{
+			m_AsyncLoggingThread.join();
+		}
+		m_Configuration.Logging = Loggingconfig;
+		return;
+	}
+	if (!m_AccessLogEnabled.load() && !m_WriteLogEnabled.load())
+	{
+		m_AsyncLoggingThread = std::thread(&ZenCacheStore::LogWorker, this);
+	}
+	m_WriteLogEnabled.store(Loggingconfig.EnableWriteLog);
+	m_AccessLogEnabled.store(Loggingconfig.EnableAccessLog);
+	m_Configuration.Logging = Loggingconfig;
+}
+
 ZenCacheStore::Info
 ZenCacheStore::GetInfo() const
 {
diff --git a/src/zenserver/cache/structuredcachestore.h b/src/zenserver/cache/structuredcachestore.h
index 239efe68f..e7b64babe 100644
--- a/src/zenserver/cache/structuredcachestore.h
+++ b/src/zenserver/cache/structuredcachestore.h
@@ -122,8 +122,11 @@ public:
 	{
 		std::filesystem::path BasePath;
 		bool				  AllowAutomaticCreationOfNamespaces = false;
-		bool				  EnableWriteLog					 = true;
-		bool				  EnableAccessLog					 = true;
+		struct LogConfig
+		{
+			bool EnableWriteLog	 = true;
+			bool EnableAccessLog = true;
+		} Logging;
 	};
 
 	struct Info
@@ -159,6 +162,8 @@ public:
 
 	GcStorageSize StorageSize() const;
 
+	Configuration								 GetConfiguration() const { return m_Configuration; }
+	void										 SetLoggingConfig(const Configuration::LogConfig& Loggingconfig);
 	Info										 GetInfo() const;
 	std::optional<ZenCacheNamespace::Info>		 GetNamespaceInfo(std::string_view Namespace);
 	std::optional<ZenCacheNamespace::BucketInfo> GetBucketInfo(std::string_view Namespace, std::string_view Bucket);
@@ -201,6 +206,8 @@ private:
 	std::atomic_bool		   m_ExitLogging;
 	Event					   m_LogEvent;
 	std::thread				   m_AsyncLoggingThread;
+	std::atomic_bool		   m_WriteLogEnabled;
+	std::atomic_bool		   m_AccessLogEnabled;
 };
 
 void z$_forcelink();
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 9c607f1d3..c36e20b30 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -433,7 +433,13 @@ public:
 		m_GcScheduler.Initialize(GcConfig);
 
 		// Create and register admin interface last to make sure all is properly initialized
-		m_AdminService = std::make_unique<HttpAdminService>(m_GcScheduler, *m_JobQueue);
+		m_AdminService =
+			std::make_unique<HttpAdminService>(m_GcScheduler,
+											   *m_JobQueue,
+											   *m_CacheStore,
+											   HttpAdminService::LogPaths{.AbsLogPath	= ServerOptions.AbsLogFile,
+																		  .HttpLogPath	= ServerOptions.DataDir / "logs" / "http.log",
+																		  .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"});
 		m_Http->RegisterService(*m_AdminService);
 
 		return EffectiveBasePort;
@@ -905,12 +911,13 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 	using namespace std::literals;
 
 	ZEN_INFO("instantiating structured cache service");
-	m_CacheStore = new ZenCacheStore(m_GcManager,
-									 ZenCacheStore::Configuration{.BasePath							  = m_DataRoot / "cache",
-																  .AllowAutomaticCreationOfNamespaces = true,
-																  .EnableWriteLog  = ServerOptions.StructuredCacheWriteLogEnabled,
-																  .EnableAccessLog = ServerOptions.StructuredCacheAccessLogEnabled},
-									 m_GcManager.GetDiskWriteBlocker());
+	m_CacheStore =
+		new ZenCacheStore(m_GcManager,
+						  ZenCacheStore::Configuration{.BasePath						   = m_DataRoot / "cache",
+													   .AllowAutomaticCreationOfNamespaces = true,
+													   .Logging = {.EnableWriteLog	= ServerOptions.StructuredCacheWriteLogEnabled,
+																   .EnableAccessLog = ServerOptions.StructuredCacheAccessLogEnabled}},
+						  m_GcManager.GetDiskWriteBlocker());
 
 	const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig;
 
@@ -1136,7 +1143,7 @@ ZenEntryPoint::Run()
 		sentry_options_set_dsn(SentryOptions, "https://[email protected]/5919284");
 		sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str());
 		sentry_options_set_logger(SentryOptions, SentryLogFunction, this);
-		std::string SentryAttachmentPath = m_ServerOptions.AbsLogFile.string();
+		std::string SentryAttachmentPath = PathToUtf8(m_ServerOptions.AbsLogFile);
 		if (SentryAttachmentPath.starts_with("\\\\?\\"))
 		{
 			SentryAttachmentPath = SentryAttachmentPath.substr(4);