Merge branch 'main' of https://github.com/EpicGames/zen

6 files changed, 268 insertions, 53 deletions

diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index cef2ba403..083086f50 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -5,15 +5,29 @@
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/jobqueue.h>
 #include <zencore/string.h>
+#if ZEN_WITH_TRACE
+#	include <zencore/trace.h>
+#endif	// ZEN_WITH_TRACE
+
 #include <zenstore/gc.h>
+#include "cache/structuredcachestore.h"
 
 #include <chrono>
 
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <spdlog/spdlog.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
 namespace zen {
 
-HttpAdminService::HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue)
+HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
+								   JobQueue&	   BackgroundJobQueue,
+								   ZenCacheStore&  CacheStore,
+								   const LogPaths& LogPaths)
 : m_GcScheduler(Scheduler)
 , m_BackgroundJobQueue(BackgroundJobQueue)
+, m_CacheStore(CacheStore)
+, m_LogPaths(LogPaths)
 {
 	using namespace std::literals;
 
@@ -248,6 +262,140 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJ
 			Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
 		},
 		HttpVerb::kPost);
+#if ZEN_WITH_TRACE
+	m_Router.RegisterRoute(
+		"trace",
+		[this](HttpRouterRequest& Req) {
+			bool Enabled = IsTracing();
+			return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Enabled ? "enabled" : "disabled");
+		},
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
+		"trace/start",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest&					 HttpReq = Req.ServerRequest();
+			const HttpServerRequest::QueryParams Params	 = HttpReq.GetQueryParams();
+			TraceType							 Type	 = TraceType::None;
+			std::string							 HostOrPath;
+			if (auto Param = Params.GetValue("file"); Param.empty() == false)
+			{
+				Type	   = TraceType::File;
+				HostOrPath = Param;
+			}
+			if (auto Param = Params.GetValue("host"); Param.empty() == false)
+			{
+				Type	   = TraceType::Network;
+				HostOrPath = Param;
+			}
+			if (Type == TraceType::None)
+			{
+				return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest,
+														 HttpContentType::kText,
+														 "Invalid trace type, use `file` or `host`"sv);
+			}
+			if (IsTracing())
+			{
+				return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest,
+														 HttpContentType::kText,
+														 "Tracing is already enabled"sv);
+			}
+			TraceStart(HostOrPath.c_str(), Type);
+			return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing started");
+		},
+		HttpVerb::kPost);
+
+	m_Router.RegisterRoute(
+		"trace/stop",
+		[this](HttpRouterRequest& Req) {
+			if (!IsTracing())
+			{
+				return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Tracing is not enabled"sv);
+			}
+			if (TraceStop())
+			{
+				return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing stopped");
+			}
+			else
+			{
+				return Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError,
+														 HttpContentType::kText,
+														 "Failed stopping trace");
+			}
+		},
+		HttpVerb::kPost);
+#endif	// ZEN_WITH_TRACE
+
+	m_Router.RegisterRoute(
+		"logs",
+		[this](HttpRouterRequest& Req) {
+			CbObjectWriter		  Obj;
+			spdlog::string_view_t LogLevel = spdlog::level::to_string_view(spdlog::get_level());
+			Obj.AddString("loglevel", std::string_view(LogLevel.data(), LogLevel.size()));
+			Obj.AddString("Logfile", PathToUtf8(m_LogPaths.AbsLogPath));
+			Obj.BeginObject("cache");
+			{
+				const ZenCacheStore::Configuration& CacheConfig = m_CacheStore.GetConfiguration();
+				Obj.AddString("Logfile", PathToUtf8(m_LogPaths.CacheLogPath));
+				Obj.AddBool("Write", CacheConfig.Logging.EnableWriteLog);
+				Obj.AddBool("Access", CacheConfig.Logging.EnableAccessLog);
+			}
+			Obj.EndObject();
+			Obj.BeginObject("http");
+			{
+				Obj.AddString("Logfile", PathToUtf8(m_LogPaths.HttpLogPath));
+			}
+			Obj.EndObject();
+			Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
+		},
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
+		"logs",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest&						HttpReq			  = Req.ServerRequest();
+			const HttpServerRequest::QueryParams	Params			  = HttpReq.GetQueryParams();
+			bool									SetCacheLogConfig = false;
+			ExtendableStringBuilder<256>			StringBuilder;
+			ZenCacheStore::Configuration::LogConfig LoggingConfig = m_CacheStore.GetConfiguration().Logging;
+			if (std::string Param(Params.GetValue("cacheenablewritelog")); Param.empty() == false)
+			{
+				LoggingConfig.EnableWriteLog = StrCaseCompare(Param.c_str(), "true") == 0;
+				SetCacheLogConfig			 = true;
+			}
+			if (std::string Param(Params.GetValue("cacheenableaccesslog")); Param.empty() == false)
+			{
+				LoggingConfig.EnableAccessLog = StrCaseCompare(Param.c_str(), "true") == 0;
+				SetCacheLogConfig			  = true;
+			}
+			if (SetCacheLogConfig)
+			{
+				m_CacheStore.SetLoggingConfig(LoggingConfig);
+				StringBuilder.Append(fmt::format("cache write log: {}, cache access log: {}",
+												 LoggingConfig.EnableWriteLog ? "true" : "false",
+												 LoggingConfig.EnableAccessLog ? "true" : "false"));
+			}
+			if (std::string Param(Params.GetValue("loglevel")); Param.empty() == false)
+			{
+				spdlog::level::level_enum NewLevel = spdlog::level::from_str(Param);
+				spdlog::string_view_t	  LogLevel = spdlog::level::to_string_view(NewLevel);
+				if (LogLevel != Param)
+				{
+					return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest,
+															 HttpContentType::kText,
+															 fmt::format("Invalid log level '{}'", Param));
+				}
+				spdlog::set_level(NewLevel);
+				if (StringBuilder.Size() > 0)
+				{
+					StringBuilder.Append(", ");
+				}
+				StringBuilder.Append("loglevel: ");
+				StringBuilder.Append(Param);
+			}
+			return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, StringBuilder.ToView());
+		},
+		HttpVerb::kPost);
 }
 
 HttpAdminService::~HttpAdminService()
diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h
index 3152f87ab..12a47f29e 100644
--- a/src/zenserver/admin/admin.h
+++ b/src/zenserver/admin/admin.h
@@ -9,11 +9,18 @@ namespace zen {
 
 class GcScheduler;
 class JobQueue;
+class ZenCacheStore;
 
 class HttpAdminService : public zen::HttpService
 {
 public:
-	HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue);
+	struct LogPaths
+	{
+		std::filesystem::path AbsLogPath;
+		std::filesystem::path HttpLogPath;
+		std::filesystem::path CacheLogPath;
+	};
+	HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue, ZenCacheStore& CacheStore, const LogPaths& LogPaths);
 	~HttpAdminService();
 
 	virtual const char* BaseUri() const override;
@@ -23,6 +30,8 @@ private:
 	HttpRequestRouter m_Router;
 	GcScheduler&	  m_GcScheduler;
 	JobQueue&		  m_BackgroundJobQueue;
+	ZenCacheStore&	  m_CacheStore;
+	LogPaths		  m_LogPaths;
 };
 
 }  // namespace zen
diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index 32e8c1f98..11ac81dcb 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -717,8 +717,12 @@ HttpStructuredCacheService::HandleCacheRequest(HttpServerRequest& Request)
 					BasePathString << Info.Config.BasePath.u8string();
 					ResponseWriter.AddString("BasePath"sv, BasePathString.ToView());
 					ResponseWriter.AddBool("AllowAutomaticCreationOfNamespaces", Info.Config.AllowAutomaticCreationOfNamespaces);
-					ResponseWriter.AddBool("EnableWriteLog", Info.Config.EnableWriteLog);
-					ResponseWriter.AddBool("EnableAccessLog", Info.Config.EnableAccessLog);
+					ResponseWriter.BeginObject("Logging");
+					{
+						ResponseWriter.AddBool("EnableWriteLog", Info.Config.Logging.EnableWriteLog);
+						ResponseWriter.AddBool("EnableAccessLog", Info.Config.Logging.EnableAccessLog);
+					}
+					ResponseWriter.EndObject();
 				}
 				ResponseWriter.EndObject();
 
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 4499b05f7..809df1a94 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -238,10 +238,7 @@ ZenCacheStore::ZenCacheStore(GcManager& Gc, const Configuration& Configuration,
 , m_Configuration(Configuration)
 , m_ExitLogging(false)
 {
-	if (m_Configuration.EnableAccessLog || m_Configuration.EnableWriteLog)
-	{
-		m_AsyncLoggingThread = std::thread(&ZenCacheStore::LogWorker, this);
-	}
+	SetLoggingConfig(m_Configuration.Logging);
 	CreateDirectories(m_Configuration.BasePath);
 
 	ZEN_INFO("Initializing at '{}'", m_Configuration.BasePath);
@@ -281,12 +278,7 @@ ZenCacheStore::ZenCacheStore(GcManager& Gc, const Configuration& Configuration,
 
 ZenCacheStore::~ZenCacheStore()
 {
-	m_ExitLogging.store(true);
-	m_LogEvent.Set();
-	if (m_AsyncLoggingThread.joinable())
-	{
-		m_AsyncLoggingThread.join();
-	}
+	SetLoggingConfig({.EnableWriteLog = false, .EnableAccessLog = false});
 	m_Namespaces.clear();
 }
 
@@ -381,7 +373,7 @@ ZenCacheStore::Get(const CacheRequestContext& Context,
 	{
 		bool Result = Store->Get(Bucket, HashKey, OutValue);
 
-		if (m_Configuration.EnableAccessLog)
+		if (m_AccessLogEnabled)
 		{
 			ZEN_TRACE_CPU("Z$::Get::AccessLog");
 			bool Signal = false;
@@ -421,7 +413,7 @@ ZenCacheStore::Put(const CacheRequestContext& Context,
 {
 	ZEN_TRACE_CPU("Z$::Put");
 
-	if (m_Configuration.EnableWriteLog)
+	if (m_WriteLogEnabled)
 	{
 		ZEN_TRACE_CPU("Z$::Get::WriteLog");
 		bool Signal = false;
@@ -616,6 +608,31 @@ ZenCacheStore::StorageSize() const
 	return Size;
 }
 
+void
+ZenCacheStore::SetLoggingConfig(const Configuration::LogConfig& Loggingconfig)
+{
+	if (!Loggingconfig.EnableAccessLog && !Loggingconfig.EnableWriteLog)
+	{
+		m_AccessLogEnabled.store(false);
+		m_WriteLogEnabled.store(false);
+		m_ExitLogging.store(true);
+		m_LogEvent.Set();
+		if (m_AsyncLoggingThread.joinable())
+		{
+			m_AsyncLoggingThread.join();
+		}
+		m_Configuration.Logging = Loggingconfig;
+		return;
+	}
+	if (!m_AccessLogEnabled.load() && !m_WriteLogEnabled.load())
+	{
+		m_AsyncLoggingThread = std::thread(&ZenCacheStore::LogWorker, this);
+	}
+	m_WriteLogEnabled.store(Loggingconfig.EnableWriteLog);
+	m_AccessLogEnabled.store(Loggingconfig.EnableAccessLog);
+	m_Configuration.Logging = Loggingconfig;
+}
+
 ZenCacheStore::Info
 ZenCacheStore::GetInfo() const
 {
diff --git a/src/zenserver/cache/structuredcachestore.h b/src/zenserver/cache/structuredcachestore.h
index 239efe68f..e7b64babe 100644
--- a/src/zenserver/cache/structuredcachestore.h
+++ b/src/zenserver/cache/structuredcachestore.h
@@ -122,8 +122,11 @@ public:
 	{
 		std::filesystem::path BasePath;
 		bool				  AllowAutomaticCreationOfNamespaces = false;
-		bool				  EnableWriteLog					 = true;
-		bool				  EnableAccessLog					 = true;
+		struct LogConfig
+		{
+			bool EnableWriteLog	 = true;
+			bool EnableAccessLog = true;
+		} Logging;
 	};
 
 	struct Info
@@ -159,6 +162,8 @@ public:
 
 	GcStorageSize StorageSize() const;
 
+	Configuration								 GetConfiguration() const { return m_Configuration; }
+	void										 SetLoggingConfig(const Configuration::LogConfig& Loggingconfig);
 	Info										 GetInfo() const;
 	std::optional<ZenCacheNamespace::Info>		 GetNamespaceInfo(std::string_view Namespace);
 	std::optional<ZenCacheNamespace::BucketInfo> GetBucketInfo(std::string_view Namespace, std::string_view Bucket);
@@ -201,6 +206,8 @@ private:
 	std::atomic_bool		   m_ExitLogging;
 	Event					   m_LogEvent;
 	std::thread				   m_AsyncLoggingThread;
+	std::atomic_bool		   m_WriteLogEnabled;
+	std::atomic_bool		   m_AccessLogEnabled;
 };
 
 void z$_forcelink();
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 921e3038d..40a797c21 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -161,13 +161,31 @@ namespace utils {
 
 		void sink_it_(const spdlog::details::log_msg& msg) override
 		{
-			std::string	   Message = fmt::format("{}\n{}({}) [{}]", msg.payload, msg.source.filename, msg.source.line, msg.source.funcname);
-			sentry_value_t event   = sentry_value_new_message_event(
-				  /*   level */ MapToSentryLevel[msg.level],
-				  /*  logger */ nullptr,
-				  /* message */ Message.c_str());
-			sentry_event_value_add_stacktrace(event, NULL, 0);
-			sentry_capture_event(event);
+			try
+			{
+				std::string Message =
+					fmt::format("{}\n{}({}) [{}]", msg.payload, msg.source.filename, msg.source.line, msg.source.funcname);
+				sentry_value_t event = sentry_value_new_message_event(
+					/*   level */ MapToSentryLevel[msg.level],
+					/*  logger */ nullptr,
+					/* message */ Message.c_str());
+				sentry_event_value_add_stacktrace(event, NULL, 0);
+				sentry_capture_event(event);
+			}
+			catch (std::exception&)
+			{
+				// If our logging with Message formatting fails we do a non-allocating version and just post the msg.payload raw
+				char   TmpBuffer[256];
+				size_t MaxCopy = Min<size_t>(msg.payload.size(), size_t(255));
+				memcpy(TmpBuffer, msg.payload.data(), MaxCopy);
+				TmpBuffer[MaxCopy]	 = '\0';
+				sentry_value_t event = sentry_value_new_message_event(
+					/*   level */ SENTRY_LEVEL_ERROR,
+					/*  logger */ nullptr,
+					/* message */ TmpBuffer);
+				sentry_event_value_add_stacktrace(event, NULL, 0);
+				sentry_capture_event(event);
+			}
 		}
 		void flush_() override {}
 	};
@@ -181,13 +199,26 @@ namespace utils {
 																  const char* FunctionName,
 																  const char* Msg)
 		{
-			std::string	   Message = fmt::format("ASSERT {}:({}) [{}]\n\"{}\"", Filename, LineNumber, FunctionName, Msg);
-			sentry_value_t event   = sentry_value_new_message_event(
-				  /*   level */ SENTRY_LEVEL_ERROR,
-				  /*  logger */ nullptr,
-				  /* message */ Message.c_str());
-			sentry_event_value_add_stacktrace(event, NULL, 0);
-			sentry_capture_event(event);
+			try
+			{
+				std::string	   Message = fmt::format("ASSERT {}:({}) [{}]\n\"{}\"", Filename, LineNumber, FunctionName, Msg);
+				sentry_value_t event   = sentry_value_new_message_event(
+					  /*   level */ SENTRY_LEVEL_ERROR,
+					  /*  logger */ nullptr,
+					  /* message */ Message.c_str());
+				sentry_event_value_add_stacktrace(event, NULL, 0);
+				sentry_capture_event(event);
+			}
+			catch (std::exception&)
+			{
+				// If our logging with Message formatting fails we do a non-allocating version and just post the Msg raw
+				sentry_value_t event = sentry_value_new_message_event(
+					/*   level */ SENTRY_LEVEL_ERROR,
+					/*  logger */ nullptr,
+					/* message */ Msg);
+				sentry_event_value_add_stacktrace(event, NULL, 0);
+				sentry_capture_event(event);
+			}
 		}
 		AssertImpl* PrevAssertImpl;
 	};
@@ -433,7 +464,13 @@ public:
 		m_GcScheduler.Initialize(GcConfig);
 
 		// Create and register admin interface last to make sure all is properly initialized
-		m_AdminService = std::make_unique<HttpAdminService>(m_GcScheduler, *m_JobQueue);
+		m_AdminService =
+			std::make_unique<HttpAdminService>(m_GcScheduler,
+											   *m_JobQueue,
+											   *m_CacheStore,
+											   HttpAdminService::LogPaths{.AbsLogPath	= ServerOptions.AbsLogFile,
+																		  .HttpLogPath	= ServerOptions.DataDir / "logs" / "http.log",
+																		  .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"});
 		m_Http->RegisterService(*m_AdminService);
 
 		return EffectiveBasePort;
@@ -905,12 +942,13 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 	using namespace std::literals;
 
 	ZEN_INFO("instantiating structured cache service");
-	m_CacheStore = new ZenCacheStore(m_GcManager,
-									 ZenCacheStore::Configuration{.BasePath							  = m_DataRoot / "cache",
-																  .AllowAutomaticCreationOfNamespaces = true,
-																  .EnableWriteLog  = ServerOptions.StructuredCacheWriteLogEnabled,
-																  .EnableAccessLog = ServerOptions.StructuredCacheAccessLogEnabled},
-									 m_GcManager.GetDiskWriteBlocker());
+	m_CacheStore =
+		new ZenCacheStore(m_GcManager,
+						  ZenCacheStore::Configuration{.BasePath						   = m_DataRoot / "cache",
+													   .AllowAutomaticCreationOfNamespaces = true,
+													   .Logging = {.EnableWriteLog	= ServerOptions.StructuredCacheWriteLogEnabled,
+																   .EnableAccessLog = ServerOptions.StructuredCacheAccessLogEnabled}},
+						  m_GcManager.GetDiskWriteBlocker());
 
 	const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig;
 
@@ -1136,7 +1174,7 @@ ZenEntryPoint::Run()
 		sentry_options_set_dsn(SentryOptions, "https://[email protected]/5919284");
 		sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str());
 		sentry_options_set_logger(SentryOptions, SentryLogFunction, this);
-		std::string SentryAttachmentPath = m_ServerOptions.AbsLogFile.string();
+		std::string SentryAttachmentPath = PathToUtf8(m_ServerOptions.AbsLogFile);
 		if (SentryAttachmentPath.starts_with("\\\\?\\"))
 		{
 			SentryAttachmentPath = SentryAttachmentPath.substr(4);
@@ -1186,8 +1224,8 @@ ZenEntryPoint::Run()
 	auto _ = zen::MakeGuard([&SentryAssert, SentryErrorCode] {
 		if (SentryErrorCode == 0)
 		{
-			SentryAssert.reset();
 			zen::logging::SetErrorLog(std::shared_ptr<spdlog::logger>());
+			SentryAssert.reset();
 			sentry_close();
 		}
 	});
@@ -1420,14 +1458,6 @@ test_main(int argc, char** argv)
 }
 #endif
 
-#if ZEN_WITH_TRACE
-static void
-StopTrace()
-{
-	TraceShutdown();
-}
-#endif	// ZEN_WITH_TRACE
-
 int
 main(int argc, char* argv[])
 {
@@ -1466,17 +1496,17 @@ main(int argc, char* argv[])
 #if ZEN_WITH_TRACE
 		if (ServerOptions.TraceHost.size())
 		{
-			TraceInit(ServerOptions.TraceHost.c_str(), TraceType::Network);
+			TraceStart(ServerOptions.TraceHost.c_str(), TraceType::Network);
 		}
 		else if (ServerOptions.TraceFile.size())
 		{
-			TraceInit(ServerOptions.TraceFile.c_str(), TraceType::File);
+			TraceStart(ServerOptions.TraceFile.c_str(), TraceType::File);
 		}
 		else
 		{
-			TraceInit(nullptr, TraceType::None);
+			TraceInit();
 		}
-		atexit(StopTrace);
+		atexit(TraceShutdown);
 #endif	// ZEN_WITH_TRACE
 
 #if ZEN_PLATFORM_WINDOWS