add trace command to enable/disable tracing at runtime (#416)

* add trace command to enable/disable tracing at runtime
* rework tracing init/start/stop
* changelog

7 files changed, 248 insertions, 38 deletions

diff --git a/src/zen/cmds/trace.cpp b/src/zen/cmds/trace.cpp
new file mode 100644
index 000000000..f8968a680
--- /dev/null
+++ b/src/zen/cmds/trace.cpp
@@ -0,0 +1,93 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "trace.h"
+#include <zencore/logging.h>
+#include <zenhttp/httpclient.h>
+#include <zenhttp/httpcommon.h>
+
+using namespace std::literals;
+
+namespace zen {
+
+TraceCommand::TraceCommand()
+{
+	m_Options.add_options()("h,help", "Print help");
+	m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+	m_Options.add_option("", "s", "stop", "Stop tracing", cxxopts::value(m_Stop)->default_value("false"), "<stop>");
+	m_Options.add_option("", "", "host", "Start tracing to host", cxxopts::value(m_TraceHost), "<hostip>");
+	m_Options.add_option("", "", "file", "Start tracing to file", cxxopts::value(m_TraceFile), "<filepath>");
+}
+
+TraceCommand::~TraceCommand() = default;
+
+int
+TraceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+	ZEN_UNUSED(GlobalOptions);
+
+	if (!ParseOptions(argc, argv))
+	{
+		return 0;
+	}
+
+	m_HostName = ResolveTargetHostSpec(m_HostName);
+
+	if (m_HostName.empty())
+	{
+		throw OptionParseException("unable to resolve server specification");
+	}
+
+	zen::HttpClient Http(m_HostName);
+
+	if (m_Stop)
+	{
+		if (zen::HttpClient::Response Response = Http.Post("/admin/trace/stop"sv))
+		{
+			ZEN_CONSOLE("OK: {}", Response.ToText());
+			return 0;
+		}
+		else
+		{
+			ZEN_ERROR("trace stop failed: {}", Response.AsText());
+			return 1;
+		}
+	}
+
+	std::string StartArg;
+	if (!m_TraceHost.empty())
+	{
+		StartArg = fmt::format("host={}", m_TraceHost);
+	}
+	else if (!m_TraceFile.empty())
+	{
+		StartArg = fmt::format("file={}", m_TraceFile);
+	}
+
+	if (!StartArg.empty())
+	{
+		if (zen::HttpClient::Response Response = Http.Post(fmt::format("/admin/trace/start?{}"sv, StartArg)))
+		{
+			ZEN_CONSOLE("OK: {}", Response.ToText());
+			return 0;
+		}
+		else
+		{
+			ZEN_ERROR("trace start failed: {}", Response.AsText());
+			return 1;
+		}
+	}
+
+	if (zen::HttpClient::Response Response = Http.Get("/admin/trace"sv))
+	{
+		ZEN_CONSOLE("OK: {}", Response.ToText());
+		return 0;
+	}
+	else
+	{
+		ZEN_ERROR("trace status failed: {}", Response.AsText());
+	}
+
+	return 1;
+}
+
+}  // namespace zen
diff --git a/src/zen/cmds/trace.h b/src/zen/cmds/trace.h
new file mode 100644
index 000000000..7b2d15fb1
--- /dev/null
+++ b/src/zen/cmds/trace.h
@@ -0,0 +1,28 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "../zen.h"
+
+namespace zen {
+
+/** Scrub storage
+ */
+class TraceCommand : public ZenCmdBase
+{
+public:
+	TraceCommand();
+	~TraceCommand();
+
+	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+	virtual cxxopts::Options& Options() override { return m_Options; }
+
+private:
+	cxxopts::Options m_Options{"trace", "Control zen realtime tracing"};
+	std::string		 m_HostName;
+	bool			 m_Stop;
+	std::string		 m_TraceHost;
+	std::string		 m_TraceFile;
+};
+
+}  // namespace zen
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index db67fbae0..8449c7a43 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -18,6 +18,7 @@
 #include "cmds/serve.h"
 #include "cmds/status.h"
 #include "cmds/top.h"
+#include "cmds/trace.h"
 #include "cmds/up.h"
 #include "cmds/version.h"
 #include "cmds/vfs_cmd.h"
@@ -228,6 +229,7 @@ main(int argc, char** argv)
 	StatusCommand			 StatusCmd;
 	JobCommand				 JobCmd;
 	TopCommand				 TopCmd;
+	TraceCommand			 TraceCmd;
 	UpCommand				 UpCmd;
 	AttachCommand			 AttachCmd;
 	VersionCommand			 VersionCmd;
@@ -278,6 +280,7 @@ main(int argc, char** argv)
 		{"status",				&StatusCmd,				"Show zen status"},
 		{"jobs",				&JobCmd,				"Show/cancel zen background jobs"},
 		{"top",					&TopCmd,				"Monitor zen server activity"},
+		{"trace",				&TraceCmd,				"Control zen realtime tracing"},
 		{"up",					&UpCmd,					"Bring zen server up"},
 		{"attach",				&AttachCmd,				"Add a sponsor process to a running zen service"},
 		{"version",				&VersionCmd,			"Get zen server version"},
diff --git a/src/zencore/include/zencore/trace.h b/src/zencore/include/zencore/trace.h
index 3bc2f7f02..665df5808 100644
--- a/src/zencore/include/zencore/trace.h
+++ b/src/zencore/include/zencore/trace.h
@@ -25,8 +25,11 @@ enum class TraceType
 	None
 };
 
-void TraceInit(const char* HostOrPath, TraceType Type);
+void TraceInit();
 void TraceShutdown();
+bool IsTracing();
+void TraceStart(const char* HostOrPath, TraceType Type);
+bool TraceStop();
 
 #else
 
diff --git a/src/zencore/trace.cpp b/src/zencore/trace.cpp
index 6da5d28d6..d71ca0984 100644
--- a/src/zencore/trace.cpp
+++ b/src/zencore/trace.cpp
@@ -9,10 +9,48 @@
 #	include <zencore/trace.h>
 
 void
-TraceInit(const char* HostOrPath, TraceType Type)
+TraceInit()
 {
-	bool EnableEvents = true;
+	static std::atomic_bool gInited	 = false;
+	bool					Expected = false;
+	if (!gInited.compare_exchange_strong(Expected, true))
+	{
+		return;
+	}
+
+	trace::FInitializeDesc Desc = {
+		.bUseImportantCache = true,
+	};
+	trace::Initialize(Desc);
 
+	trace::ThreadRegister("main", /* system id */ 0, /* sort id */ 0);
+	trace::DescribeSession("zenserver",
+#	if ZEN_BUILD_DEBUG
+						   trace::Build::Debug,
+#	else
+						   trace::Build::Development,
+#	endif
+						   "",
+						   ZEN_CFG_VERSION_BUILD_STRING);
+}
+
+void
+TraceShutdown()
+{
+	(void)TraceStop();
+	trace::Shutdown();
+}
+
+bool
+IsTracing()
+{
+	return trace::IsTracing();
+}
+
+void
+TraceStart(const char* HostOrPath, TraceType Type)
+{
+	TraceInit();
 	switch (Type)
 	{
 		case TraceType::Network:
@@ -24,34 +62,20 @@ TraceInit(const char* HostOrPath, TraceType Type)
 			break;
 
 		case TraceType::None:
-			EnableEvents = false;
 			break;
 	}
-
-	trace::FInitializeDesc Desc = {
-		.bUseImportantCache = false,
-	};
-	trace::Initialize(Desc);
-
-	if (EnableEvents)
-	{
-		trace::ToggleChannel("cpu", true);
-		trace::ThreadRegister("main", /* system id */ 0, /* sort id */ 0);
-		trace::DescribeSession("zenserver",
-#	if ZEN_BUILD_DEBUG
-							   trace::Build::Debug,
-#	else
-							   trace::Build::Development,
-#	endif
-							   "",
-							   ZEN_CFG_VERSION_BUILD_STRING);
-	}
+	trace::ToggleChannel("cpu", true);
 }
 
-void
-TraceShutdown()
+bool
+TraceStop()
 {
-	trace::Shutdown();
+	trace::ToggleChannel("cpu", false);
+	if (trace::Stop())
+	{
+		return true;
+	}
+	return false;
 }
 
 #endif	// ZEN_WITH_TRACE
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index cef2ba403..05ae0a24a 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -5,6 +5,10 @@
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/jobqueue.h>
 #include <zencore/string.h>
+#if ZEN_WITH_TRACE
+#	include <zencore/trace.h>
+#endif	// ZEN_WITH_TRACE
+
 #include <zenstore/gc.h>
 
 #include <chrono>
@@ -248,6 +252,69 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJ
 			Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
 		},
 		HttpVerb::kPost);
+#if ZEN_WITH_TRACE
+	m_Router.RegisterRoute(
+		"trace",
+		[this](HttpRouterRequest& Req) {
+			bool Enabled = IsTracing();
+			return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Enabled ? "enabled" : "disabled");
+		},
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
+		"trace/start",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest&					 HttpReq = Req.ServerRequest();
+			const HttpServerRequest::QueryParams Params	 = HttpReq.GetQueryParams();
+			TraceType							 Type	 = TraceType::None;
+			std::string							 HostOrPath;
+			if (auto Param = Params.GetValue("file"); Param.empty() == false)
+			{
+				Type	   = TraceType::File;
+				HostOrPath = Param;
+			}
+			if (auto Param = Params.GetValue("host"); Param.empty() == false)
+			{
+				Type	   = TraceType::Network;
+				HostOrPath = Param;
+			}
+			if (Type == TraceType::None)
+			{
+				return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest,
+														 HttpContentType::kText,
+														 "Invalid trace type, use `file` or `host`"sv);
+			}
+			if (IsTracing())
+			{
+				return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest,
+														 HttpContentType::kText,
+														 "Tracing is already enabled"sv);
+			}
+			TraceStart(HostOrPath.c_str(), Type);
+			return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing started");
+		},
+		HttpVerb::kPost);
+
+	m_Router.RegisterRoute(
+		"trace/stop",
+		[this](HttpRouterRequest& Req) {
+			if (!IsTracing())
+			{
+				return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Tracing is not enabled"sv);
+			}
+			if (TraceStop())
+			{
+				return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing stopped");
+			}
+			else
+			{
+				return Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError,
+														 HttpContentType::kText,
+														 "Failed stopping trace");
+			}
+		},
+		HttpVerb::kPost);
+#endif	// ZEN_WITH_TRACE
 }
 
 HttpAdminService::~HttpAdminService()
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 921e3038d..9c607f1d3 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -1420,14 +1420,6 @@ test_main(int argc, char** argv)
 }
 #endif
 
-#if ZEN_WITH_TRACE
-static void
-StopTrace()
-{
-	TraceShutdown();
-}
-#endif	// ZEN_WITH_TRACE
-
 int
 main(int argc, char* argv[])
 {
@@ -1466,17 +1458,17 @@ main(int argc, char* argv[])
 #if ZEN_WITH_TRACE
 		if (ServerOptions.TraceHost.size())
 		{
-			TraceInit(ServerOptions.TraceHost.c_str(), TraceType::Network);
+			TraceStart(ServerOptions.TraceHost.c_str(), TraceType::Network);
 		}
 		else if (ServerOptions.TraceFile.size())
 		{
-			TraceInit(ServerOptions.TraceFile.c_str(), TraceType::File);
+			TraceStart(ServerOptions.TraceFile.c_str(), TraceType::File);
 		}
 		else
 		{
-			TraceInit(nullptr, TraceType::None);
+			TraceInit();
 		}
-		atexit(StopTrace);
+		atexit(TraceShutdown);
 #endif	// ZEN_WITH_TRACE
 
 #if ZEN_PLATFORM_WINDOWS