clean up trace options parsing (#473)

* clean up trace command line options
explicitly shut down worker pools
* some additional startup trace scopes

20 files changed, 220 insertions, 148 deletions

diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index 64cdbf5c4..6376bbad2 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -39,6 +39,7 @@
 #include <zenhttp/httpcommon.h>
 #include <zenutil/environmentoptions.h>
 #include <zenutil/logging.h>
+#include <zenutil/workerpools.h>
 #include <zenutil/zenserverprocess.h>
 
 #include <zencore/memory/fmalloc.h>
@@ -636,6 +637,15 @@ main(int argc, char** argv)
 	using namespace zen;
 	using namespace std::literals;
 
+#if ZEN_WITH_TRACE
+	TraceInit("zen");
+	TraceOptions TraceCommandlineOptions;
+	if (GetTraceOptionsFromCommandline(TraceCommandlineOptions))
+	{
+		TraceConfigure(TraceCommandlineOptions);
+	}
+#endif	// ZEN_WITH_TRACE
+
 	AttachCommand			 AttachCmd;
 	BenchCommand			 BenchCmd;
 	BuildsCommand			 BuildsCmd;
@@ -851,30 +861,32 @@ main(int argc, char** argv)
 	Options.add_options()("c, command", "Sub command", cxxopts::value<std::string>(SubCommand));
 
 #if ZEN_WITH_TRACE
-	std::string TraceChannels;
-	std::string TraceHost;
-	std::string TraceFile;
+	// We only have this in options for command line help purposes - we parse these argument separately earlier using
+	// GetTraceOptionsFromCommandline()
 
 	Options.add_option("ue-trace",
 					   "",
 					   "trace",
 					   "Specify which trace channels should be enabled",
-					   cxxopts::value<std::string>(TraceChannels)->default_value(""),
+					   cxxopts::value<std::string>(TraceCommandlineOptions.Channels)->default_value(""),
 					   "");
 
 	Options.add_option("ue-trace",
 					   "",
 					   "tracehost",
 					   "Hostname to send the trace to",
-					   cxxopts::value<std::string>(TraceHost)->default_value(""),
+					   cxxopts::value<std::string>(TraceCommandlineOptions.Host)->default_value(""),
 					   "");
 
-	Options
-		.add_option("ue-trace", "", "tracefile", "Path to write a trace to", cxxopts::value<std::string>(TraceFile)->default_value(""), "");
+	Options.add_option("ue-trace",
+					   "",
+					   "tracefile",
+					   "Path to write a trace to",
+					   cxxopts::value<std::string>(TraceCommandlineOptions.File)->default_value(""),
+					   "");
 #endif	// ZEN_WITH_TRACE
 
 #if ZEN_USE_SENTRY
-
 	SentryIntegration::Config SentryConfig;
 
 	bool NoSentry = false;
@@ -998,31 +1010,10 @@ main(int argc, char** argv)
 
 		//////////////////////////////////////////////////////////////////////////
 
-		auto _ = zen::MakeGuard([] { zen::ShutdownLogging(); });
-
-#if ZEN_WITH_TRACE
-		if (TraceHost.size())
-		{
-			TraceStart("zen", TraceHost.c_str(), TraceType::Network);
-		}
-		else if (TraceFile.size())
-		{
-			TraceStart("zen", TraceFile.c_str(), TraceType::File);
-		}
-		else
-		{
-			TraceInit("zen");
-		}
-#endif	// ZEN_WITH_TRACE
-
-#if ZEN_WITH_MEMTRACK
-		FMalloc* TraceMalloc = MemoryTrace_Create(GMalloc);
-		if (TraceMalloc != GMalloc)
-		{
-			GMalloc = TraceMalloc;
-			MemoryTrace_Initialize();
-		}
-#endif
+		auto _ = zen::MakeGuard([] {
+			ShutdownWorkerPools();
+			ShutdownLogging();
+		});
 
 		for (const CommandInfo& CmdInfo : Commands)
 		{
diff --git a/src/zencore/include/zencore/memory/memorytrace.h b/src/zencore/include/zencore/memory/memorytrace.h
index d1ab1f914..6be7adb89 100644
--- a/src/zencore/include/zencore/memory/memorytrace.h
+++ b/src/zencore/include/zencore/memory/memorytrace.h
@@ -94,7 +94,7 @@ ENUM_CLASS_FLAGS(EMemoryTraceInit);
 UE_TRACE_CHANNEL_EXTERN(MemAllocChannel);
 
 ////////////////////////////////////////////////////////////////////////////////
-class FMalloc* MemoryTrace_Create(class FMalloc* InMalloc);
+class FMalloc* MemoryTrace_Create(class FMalloc* InMalloc, const TraceOptions& Options);
 void		   MemoryTrace_Initialize();
 void		   MemoryTrace_Shutdown();
 
diff --git a/src/zencore/include/zencore/trace.h b/src/zencore/include/zencore/trace.h
index 2ca2b7c81..99a565151 100644
--- a/src/zencore/include/zencore/trace.h
+++ b/src/zencore/include/zencore/trace.h
@@ -21,19 +21,21 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen {
 
-enum class TraceType
+struct TraceOptions
 {
-	File,
-	Network,
-	None
+	std::string Host;
+	std::string File;
+	std::string Channels;
 };
 
 void TraceInit(std::string_view ProgramName);
 void TraceShutdown();
 bool IsTracing();
-void TraceStart(std::string_view ProgramName, const char* HostOrPath, TraceType Type);
 bool TraceStop();
 
+bool GetTraceOptionsFromCommandline(TraceOptions& OutOptions);
+void TraceConfigure(const TraceOptions& Options);
+
 }
 
 #else
diff --git a/src/zencore/memory/memory.cpp b/src/zencore/memory/memory.cpp
index f236796ad..ae1b9abce 100644
--- a/src/zencore/memory/memory.cpp
+++ b/src/zencore/memory/memory.cpp
@@ -87,14 +87,19 @@ InitGMalloc()
 		}
 	};
 
-	constexpr std::string_view MallocOption = "--malloc="sv;
+	constexpr std::string_view MallocOption = "--malloc"sv;
 
 	std::function<void(const std::string_view&)> ProcessArg = [&](const std::string_view& Arg) {
 		if (Arg.starts_with(MallocOption))
 		{
-			const std::string_view OptionArgs = Arg.substr(MallocOption.size());
+			std::string_view::value_type DelimChar = Arg[MallocOption.length()];
 
-			IterateCommaSeparatedValue(OptionArgs, ProcessMallocArg);
+			if (DelimChar == ' ' || DelimChar == '=')
+			{
+				const std::string_view OptionArgs = Arg.substr(MallocOption.size() + 1);
+
+				IterateCommaSeparatedValue(OptionArgs, ProcessMallocArg);
+			}
 		}
 	};
 
diff --git a/src/zencore/memtrack/memorytrace.cpp b/src/zencore/memtrack/memorytrace.cpp
index e4ae8148e..8f723866d 100644
--- a/src/zencore/memtrack/memorytrace.cpp
+++ b/src/zencore/memtrack/memorytrace.cpp
@@ -294,7 +294,7 @@ static FUndestructed<FTraceMalloc> GTraceMalloc;
 
 ////////////////////////////////////////////////////////////////////////////////
 static EMemoryTraceInit
-MemoryTrace_ShouldEnable()
+MemoryTrace_ShouldEnable(const TraceOptions& Options)
 {
 	EMemoryTraceInit Mode = EMemoryTraceInit::Disabled;
 
@@ -328,19 +328,7 @@ MemoryTrace_ShouldEnable()
 		}
 	};
 
-	constexpr std::string_view TraceOption = "--trace="sv;
-
-	std::function<void(const std::string_view&)> ProcessArg = [&](const std::string_view& Arg) {
-		if (Arg.starts_with(TraceOption))
-		{
-			const std::string_view OptionArgs = Arg.substr(TraceOption.size());
-
-			IterateCommaSeparatedValue(OptionArgs, ProcessTraceArg);
-		}
-	};
-
-	IterateCommandlineArgs(ProcessArg);
-
+	IterateCommaSeparatedValue(Options.Channels, ProcessTraceArg);
 	return Mode;
 }
 
@@ -388,17 +376,17 @@ MemoryTrace_CreateInternal(FMalloc* InMalloc, EMemoryTraceInit Mode)
 
 ////////////////////////////////////////////////////////////////////////////////
 FMalloc*
-MemoryTrace_CreateInternal(FMalloc* InMalloc)
+MemoryTrace_CreateInternal(FMalloc* InMalloc, const TraceOptions& Options)
 {
-	const EMemoryTraceInit Mode = MemoryTrace_ShouldEnable();
+	const EMemoryTraceInit Mode = MemoryTrace_ShouldEnable(Options);
 	return MemoryTrace_CreateInternal(InMalloc, Mode);
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 FMalloc*
-MemoryTrace_Create(FMalloc* InMalloc)
+MemoryTrace_Create(FMalloc* InMalloc, const TraceOptions& Options)
 {
-	FMalloc* OutMalloc = MemoryTrace_CreateInternal(InMalloc);
+	FMalloc* OutMalloc = MemoryTrace_CreateInternal(InMalloc, Options);
 
 	if (OutMalloc != InMalloc)
 	{
diff --git a/src/zencore/trace.cpp b/src/zencore/trace.cpp
index ef7cbf596..fe8fb9a5d 100644
--- a/src/zencore/trace.cpp
+++ b/src/zencore/trace.cpp
@@ -11,19 +11,18 @@
 #	define TRACE_IMPLEMENT 1
 #	include <zencore/trace.h>
 
+#	include <zencore/memory/fmalloc.h>
 #	include <zencore/memory/memorytrace.h>
 
 namespace zen {
 
 void
-TraceConfigure()
+TraceConfigure(const TraceOptions& Options)
 {
 	// Configure channels based on command line options
 
 	using namespace std::literals;
 
-	constexpr std::string_view TraceOption = "--trace="sv;
-
 	std::function<void(const std::string_view&)> ProcessChannelList;
 
 	auto ProcessTraceArg = [&](const std::string_view& Arg) {
@@ -63,25 +62,32 @@ TraceConfigure()
 
 	ProcessChannelList = [&](const std::string_view& OptionArgs) { IterateCommaSeparatedValue(OptionArgs, ProcessTraceArg); };
 
-	bool TraceOptionPresent = false;
-
-	std::function<void(const std::string_view&)> ProcessArg = [&](const std::string_view& Arg) {
-		if (Arg.starts_with(TraceOption))
-		{
-			const std::string_view OptionArgs = Arg.substr(TraceOption.size());
-
-			TraceOptionPresent = true;
-
-			ProcessChannelList(OptionArgs);
-		}
-	};
+	if (Options.Channels.empty())
+	{
+		ProcessTraceArg("default"sv);
+	}
+	else
+	{
+		ProcessChannelList(Options.Channels);
+	}
 
-	IterateCommandlineArgs(ProcessArg);
+	if (Options.Host.size())
+	{
+		trace::SendTo(Options.Host.c_str());
+	}
+	else if (Options.File.size())
+	{
+		trace::WriteTo(Options.File.c_str());
+	}
 
-	if (!TraceOptionPresent)
+#	if ZEN_WITH_MEMTRACK
+	FMalloc* TraceMalloc = MemoryTrace_Create(GMalloc, Options);
+	if (TraceMalloc != GMalloc)
 	{
-		ProcessTraceArg("default"sv);
+		GMalloc = TraceMalloc;
+		MemoryTrace_Initialize();
 	}
+#	endif
 }
 
 void
@@ -122,8 +128,6 @@ TraceInit(std::string_view ProgramName)
 		trace::Update();
 		TraceShutdown();
 	});
-
-	TraceConfigure();
 }
 
 void
@@ -139,25 +143,6 @@ IsTracing()
 	return trace::IsTracing();
 }
 
-void
-TraceStart(std::string_view ProgramName, const char* HostOrPath, TraceType Type)
-{
-	TraceInit(ProgramName);
-	switch (Type)
-	{
-		case TraceType::Network:
-			trace::SendTo(HostOrPath);
-			break;
-
-		case TraceType::File:
-			trace::WriteTo(HostOrPath);
-			break;
-
-		case TraceType::None:
-			break;
-	}
-}
-
 bool
 TraceStop()
 {
@@ -168,6 +153,57 @@ TraceStop()
 	return false;
 }
 
+bool
+GetTraceOptionsFromCommandline(TraceOptions& OutOptions)
+{
+	bool HasOptions = false;
+
+#	if ZEN_WITH_TRACE
+	using namespace std::literals;
+
+	auto MatchesArg = [](std::string_view Option, std::string_view Arg) -> std::optional<std::string_view> {
+		if (Arg.starts_with(Option))
+		{
+			std::string_view::value_type DelimChar = Arg[Option.length()];
+			if (DelimChar == ' ' || DelimChar == '=')
+			{
+				return Arg.substr(Option.size() + 1);
+			}
+		}
+		return {};
+	};
+
+	constexpr std::string_view TraceOption	   = "--trace"sv;
+	constexpr std::string_view TraceHostOption = "--tracehost"sv;
+	constexpr std::string_view TraceFileOption = "--tracefile"sv;
+
+	std::function<void(const std::string_view&)> ProcessArg = [&](const std::string_view& Arg) {
+		if (auto Host = MatchesArg(TraceHostOption, Arg); Host.has_value())
+		{
+			OutOptions.Host = Host.value();
+			HasOptions		= true;
+		}
+		else if (auto File = MatchesArg(TraceFileOption, Arg); File.has_value())
+		{
+			OutOptions.File = File.value();
+			HasOptions		= true;
+		}
+		else if (auto Channels = MatchesArg(TraceOption, Arg); Channels.has_value())
+		{
+			if (!OutOptions.Channels.empty())
+			{
+				OutOptions.Channels = ","sv;
+			}
+			OutOptions.Channels += Channels.value();
+			HasOptions = true;
+		}
+	};
+
+	IterateCommandlineArgs(ProcessArg);
+#	endif	// ZEN_WITH_TRACE
+	return HasOptions;
+}
+
 }  // namespace zen
 
 #endif	// ZEN_WITH_TRACE
diff --git a/src/zenhttp/auth/authmgr.cpp b/src/zenhttp/auth/authmgr.cpp
index 8f7befc80..6c1a66a99 100644
--- a/src/zenhttp/auth/authmgr.cpp
+++ b/src/zenhttp/auth/authmgr.cpp
@@ -9,6 +9,7 @@
 #include <zencore/crypto.h>
 #include <zencore/filesystem.h>
 #include <zencore/logging.h>
+#include <zencore/trace.h>
 #include <zenhttp/auth/oidc.h>
 
 #include <condition_variable>
@@ -29,6 +30,8 @@ namespace details {
 							   const AesIV128Bit&		   IV,
 							   std::optional<std::string>& Reason)
 	{
+		ZEN_TRACE_CPU("AuthMgr::ReadEncryptedFile");
+
 		FileContents Result = ReadFile(Path);
 
 		if (Result.ErrorCode)
@@ -62,6 +65,8 @@ namespace details {
 							const AesIV128Bit&			IV,
 							std::optional<std::string>& Reason)
 	{
+		ZEN_TRACE_CPU("AuthMgr::WriteEncryptedFile");
+
 		if (FileData.GetSize() == 0)
 		{
 			return;
@@ -100,6 +105,8 @@ public:
 
 	virtual void AddOpenIdProvider(const AddOpenIdProviderParams& Params) final
 	{
+		ZEN_TRACE_CPU("AuthMgr::AddOpenIdProvider");
+
 		if (Params.Name.empty())
 		{
 			ZEN_WARN("add OpenID provider FAILED, reason 'invalid name'");
@@ -159,6 +166,8 @@ public:
 
 	virtual bool AddOpenIdToken(const AddOpenIdTokenParams& Params) final
 	{
+		ZEN_TRACE_CPU("AuthMgr::AddOpenIdToken");
+
 		if (Params.ProviderName.empty())
 		{
 			ZEN_WARN("trying add OpenID token with invalid provider name");
@@ -251,6 +260,8 @@ private:
 
 	OidcClient::RefreshTokenResult RefreshOpenIdToken(std::string_view ProviderName, std::string_view RefreshToken)
 	{
+		ZEN_TRACE_CPU("AuthMgr::RefreshOpenIdToken");
+
 		RefPtr<OidcClient> Client = GetOpenIdProvider(ProviderName).HttpClient;
 		if (!Client)
 		{
@@ -268,6 +279,7 @@ private:
 
 	void LoadState()
 	{
+		ZEN_TRACE_CPU("AuthMgrImpl::LoadState");
 		try
 		{
 			std::optional<std::string> Reason;
@@ -340,6 +352,8 @@ private:
 
 	void SaveState()
 	{
+		ZEN_TRACE_CPU("AuthMgr::SaveState");
+
 		try
 		{
 			CbObjectWriter AuthState;
diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp
index c1b7294c9..5392140d1 100644
--- a/src/zenhttp/servers/httpasio.cpp
+++ b/src/zenhttp/servers/httpasio.cpp
@@ -1159,6 +1159,7 @@ HttpAsioServer::RegisterService(HttpService& Service)
 int
 HttpAsioServer::Initialize(int BasePort, std::filesystem::path DataDir)
 {
+	ZEN_TRACE_CPU("HttpAsioServer::Initialize");
 	m_Impl->Initialize(DataDir);
 
 	m_BasePort = m_Impl->Start(gsl::narrow<uint16_t>(BasePort), m_ForceLoopback, m_ThreadCount);
@@ -1219,6 +1220,7 @@ HttpAsioServer::RequestExit()
 Ref<HttpServer>
 CreateHttpAsioServer(bool ForceLoopback, unsigned int ThreadCount)
 {
+	ZEN_TRACE_CPU("CreateHttpAsioServer");
 	ZEN_MEMSCOPE(GetHttpasioTag());
 
 	return Ref<HttpServer>{new HttpAsioServer(ForceLoopback, ThreadCount)};
diff --git a/src/zenhttp/servers/httpmulti.cpp b/src/zenhttp/servers/httpmulti.cpp
index f4dc1e15b..b8b7931a9 100644
--- a/src/zenhttp/servers/httpmulti.cpp
+++ b/src/zenhttp/servers/httpmulti.cpp
@@ -3,6 +3,7 @@
 #include "httpmulti.h"
 
 #include <zencore/logging.h>
+#include <zencore/trace.h>
 
 #if ZEN_PLATFORM_WINDOWS
 #	include <conio.h>
@@ -30,6 +31,8 @@ HttpMultiServer::RegisterService(HttpService& Service)
 int
 HttpMultiServer::Initialize(int BasePort, std::filesystem::path DataDir)
 {
+	ZEN_TRACE_CPU("HttpMultiServer::Initialize");
+
 	ZEN_UNUSED(DataDir);
 	ZEN_ASSERT(!m_IsInitialized);
 
diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp
index ec12b3755..155f3fa02 100644
--- a/src/zenhttp/servers/httpplugin.cpp
+++ b/src/zenhttp/servers/httpplugin.cpp
@@ -700,6 +700,8 @@ HttpPluginServerImpl::CreateConnectionHandler(TransportConnection* Connection)
 int
 HttpPluginServerImpl::Initialize(int BasePort, std::filesystem::path DataDir)
 {
+	ZEN_TRACE_CPU("HttpPluginServerImpl::Initialize");
+
 	ZEN_MEMSCOPE(GetHttppluginTag());
 	m_RequestTracer.Initialize(DataDir);
 
diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp
index 62dab02c4..e57fa8a30 100644
--- a/src/zenhttp/servers/httpsys.cpp
+++ b/src/zenhttp/servers/httpsys.cpp
@@ -2078,6 +2078,8 @@ InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesT
 int
 HttpSysServer::Initialize(int BasePort, std::filesystem::path DataDir)
 {
+	ZEN_TRACE_CPU("HttpSysServer::Initialize");
+
 	ZEN_UNUSED(DataDir);
 	if (int EffectivePort = InitializeServer(BasePort))
 	{
@@ -2106,6 +2108,7 @@ HttpSysServer::RegisterService(HttpService& Service)
 Ref<HttpServer>
 CreateHttpSysServer(HttpSysConfig Config)
 {
+	ZEN_TRACE_CPU("CreateHttpSysServer");
 	ZEN_MEMSCOPE(GetHttpsysTag());
 
 	return Ref<HttpServer>(new HttpSysServer(Config));
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index 8c2e6d771..f6ad14422 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -594,31 +594,35 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 		[this](HttpRouterRequest& Req) {
 			HttpServerRequest&					 HttpReq = Req.ServerRequest();
 			const HttpServerRequest::QueryParams Params	 = HttpReq.GetQueryParams();
-			TraceType							 Type	 = TraceType::None;
-			std::string							 HostOrPath;
-			if (auto Param = Params.GetValue("file"); Param.empty() == false)
+			TraceOptions						 TraceOptions;
+
+			if (!IsTracing())
 			{
-				Type	   = TraceType::File;
-				HostOrPath = Param;
+				TraceInit("zenserver");
 			}
-			if (auto Param = Params.GetValue("host"); Param.empty() == false)
+
+			if (auto Channels = Params.GetValue("channels"); Channels.empty() == false)
 			{
-				Type	   = TraceType::Network;
-				HostOrPath = Param;
+				TraceOptions.Channels = Channels;
 			}
-			if (Type == TraceType::None)
+
+			if (auto File = Params.GetValue("file"); File.empty() == false)
 			{
-				return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest,
-														 HttpContentType::kText,
-														 "Invalid trace type, use `file` or `host`"sv);
+				TraceOptions.File = File;
 			}
-			if (IsTracing())
+			else if (auto Host = Params.GetValue("host"); Host.empty() == false)
+			{
+				TraceOptions.Host = Host;
+			}
+			else
 			{
 				return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest,
 														 HttpContentType::kText,
-														 "Tracing is already enabled"sv);
+														 "Invalid trace type, use `file` or `host`"sv);
 			}
-			TraceStart("zenserver", HostOrPath.c_str(), Type);
+
+			TraceConfigure(TraceOptions);
+
 			return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing started");
 		},
 		HttpVerb::kPost);
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 23bb3ad78..4049ae815 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -461,6 +461,8 @@ ParseConfigFile(const std::filesystem::path& Path,
 				const cxxopts::ParseResult&	 CmdLineResult,
 				std::string_view			 OutputConfigFile)
 {
+	ZEN_TRACE_CPU("ParseConfigFile");
+
 	using namespace std::literals;
 
 	LuaConfig::Options LuaOptions;
@@ -508,8 +510,9 @@ ParseConfigFile(const std::filesystem::path& Path,
 
 #if ZEN_WITH_TRACE
 	////// trace
-	LuaOptions.AddOption("trace.host"sv, ServerOptions.TraceHost, "tracehost"sv);
-	LuaOptions.AddOption("trace.file"sv, ServerOptions.TraceFile, "tracefile"sv);
+	LuaOptions.AddOption("trace.channels"sv, ServerOptions.TraceOptions.Channels, "trace"sv);
+	LuaOptions.AddOption("trace.host"sv, ServerOptions.TraceOptions.Host, "tracehost"sv);
+	LuaOptions.AddOption("trace.file"sv, ServerOptions.TraceOptions.File, "tracefile"sv);
 #endif
 
 	////// stats
@@ -924,25 +927,28 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 					   "<httpsys request logging>");
 
 #if ZEN_WITH_TRACE
+	// We only have this in options for command line help purposes - we parse these argument separately earlier using
+	// GetTraceOptionsFromCommandline()
+
 	options.add_option("ue-trace",
 					   "",
 					   "trace",
 					   "Specify which trace channels should be enabled",
-					   cxxopts::value<std::string>(ServerOptions.TraceChannels)->default_value(""),
+					   cxxopts::value<std::string>(ServerOptions.TraceOptions.Channels)->default_value(""),
 					   "");
 
 	options.add_option("ue-trace",
 					   "",
 					   "tracehost",
 					   "Hostname to send the trace to",
-					   cxxopts::value<std::string>(ServerOptions.TraceHost)->default_value(""),
+					   cxxopts::value<std::string>(ServerOptions.TraceOptions.Host)->default_value(""),
 					   "");
 
 	options.add_option("ue-trace",
 					   "",
 					   "tracefile",
 					   "Path to write a trace to",
-					   cxxopts::value<std::string>(ServerOptions.TraceFile)->default_value(""),
+					   cxxopts::value<std::string>(ServerOptions.TraceOptions.File)->default_value(""),
 					   "");
 #endif	// ZEN_WITH_TRACE
 
@@ -1342,6 +1348,14 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 			exit(0);
 		}
 
+		if (!ServerOptions.HasTraceCommandlineOptions)
+		{
+			// Apply any Lua settings if we don't have them set from the command line
+			TraceConfigure(ServerOptions.TraceOptions);
+		}
+
+		ZEN_TRACE_CPU("ConfigParse");
+
 		for (int i = 0; i < logging::level::LogLevelCount; ++i)
 		{
 			logging::ConfigureLogLevels(logging::level::LogLevel(i), ServerOptions.Loggers[i]);
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index 4a022a807..9e8787957 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include <zencore/logbase.h>
+#include <zencore/trace.h>
 #include <zencore/zencore.h>
 #include <zenhttp/httpserver.h>
 #include <filesystem>
@@ -208,9 +209,8 @@ struct ZenServerOptions
 	std::string				 Loggers[zen::logging::level::LogLevelCount];
 	std::string				 ScrubOptions;
 #if ZEN_WITH_TRACE
-	std::string TraceChannels;	// Trace channels to enable
-	std::string TraceHost;		// Host name or IP address to send trace data to
-	std::string TraceFile;		// Path of a file to write a trace
+	bool		 HasTraceCommandlineOptions = false;
+	TraceOptions TraceOptions;
 #endif
 	std::string MemoryOptions;	// Memory allocation options
 	std::string CommandLine;
diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp
index dfa710ae0..2b157581f 100644
--- a/src/zenserver/frontend/frontend.cpp
+++ b/src/zenserver/frontend/frontend.cpp
@@ -8,6 +8,7 @@
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/string.h>
+#include <zencore/trace.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #if ZEN_PLATFORM_WINDOWS
@@ -31,6 +32,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpSt
 : m_Directory(Directory)
 , m_StatusService(StatusService)
 {
+	ZEN_TRACE_CPU("HttpFrontendService::HttpFrontendService");
 	std::filesystem::path SelfPath = GetRunningExecutablePath();
 
 #if ZEN_EMBED_HTML_ZIP
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index 6d9a478be..553562473 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -393,6 +393,18 @@ main(int argc, char* argv[])
 	try
 	{
 		ZenServerOptions ServerOptions;
+
+		{
+#if ZEN_WITH_TRACE
+			TraceInit("zenserver");
+			ServerOptions.HasTraceCommandlineOptions = GetTraceOptionsFromCommandline(ServerOptions.TraceOptions);
+			if (ServerOptions.HasTraceCommandlineOptions)
+			{
+				TraceConfigure(ServerOptions.TraceOptions);
+			}
+#endif	// ZEN_WITH_TRACE
+		}
+
 		ParseCliOptions(argc, argv, ServerOptions);
 
 		if (ServerOptions.Detach)
@@ -435,30 +447,6 @@ main(int argc, char* argv[])
 			CopyTree(ServerOptions.BaseSnapshotDir, ServerOptions.DataDir, {.EnableClone = true});
 		}
 
-#if ZEN_WITH_TRACE
-		if (ServerOptions.TraceHost.size())
-		{
-			TraceStart("zenserver", ServerOptions.TraceHost.c_str(), TraceType::Network);
-		}
-		else if (ServerOptions.TraceFile.size())
-		{
-			TraceStart("zenserver", ServerOptions.TraceFile.c_str(), TraceType::File);
-		}
-		else
-		{
-			TraceInit("zenserver");
-		}
-#endif	// ZEN_WITH_TRACE
-
-#if ZEN_WITH_MEMTRACK
-		FMalloc* TraceMalloc = MemoryTrace_Create(GMalloc);
-		if (TraceMalloc != GMalloc)
-		{
-			GMalloc = TraceMalloc;
-			MemoryTrace_Initialize();
-		}
-#endif
-
 		ZEN_MEMSCOPE(GetZenserverTag());
 
 #if ZEN_PLATFORM_WINDOWS
diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp
index 8faf12165..b1e73c7df 100644
--- a/src/zenserver/objectstore/objectstore.cpp
+++ b/src/zenserver/objectstore/objectstore.cpp
@@ -9,6 +9,7 @@
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/string.h>
+#include <zencore/trace.h>
 #include "zencore/compactbinary.h"
 #include "zencore/compactbinarybuilder.h"
 #include "zenhttp/httpcommon.h"
@@ -259,6 +260,8 @@ HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request)
 void
 HttpObjectStoreService::Inititalize()
 {
+	ZEN_TRACE_CPU("HttpObjectStoreService::Inititalize");
+
 	namespace fs = std::filesystem;
 	ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory);
 
diff --git a/src/zenserver/stats/statsreporter.cpp b/src/zenserver/stats/statsreporter.cpp
index 5d5ef4bfa..a1926eba4 100644
--- a/src/zenserver/stats/statsreporter.cpp
+++ b/src/zenserver/stats/statsreporter.cpp
@@ -3,6 +3,7 @@
 #include "statsreporter.h"
 
 #include <zencore/logging.h>
+#include <zencore/trace.h>
 #include <zennet/statsdclient.h>
 
 namespace zen {
@@ -18,6 +19,7 @@ StatsReporter::~StatsReporter()
 void
 StatsReporter::Initialize(const ZenStatsConfig& Config)
 {
+	ZEN_TRACE_CPU("StatsReporter::Initialize");
 	RwLock::ExclusiveLockScope _(m_Lock);
 
 	if (Config.Enabled)
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 5cab54acc..71b52817c 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -128,6 +128,8 @@ ZenServer::OnReady()
 int
 ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry)
 {
+	ZEN_TRACE_CPU("ZenServer::Initialize");
+
 	ZEN_MEMSCOPE(GetZenserverTag());
 	const std::string MutexName = fmt::format("zen_{}", ServerOptions.BasePort);
 
@@ -189,6 +191,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 
 	// Setup authentication manager
 	{
+		ZEN_TRACE_CPU("Zenserver::InitAuth");
 		std::string EncryptionKey = ServerOptions.EncryptionKey;
 
 		if (EncryptionKey.empty())
@@ -379,6 +382,8 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 void
 ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
 {
+	ZEN_TRACE_CPU("ZenServer::InitializeState");
+
 	EnqueueSigIntTimer();
 
 	// Check root manifest to deal with schema versioning
@@ -543,6 +548,8 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
 void
 ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 {
+	ZEN_TRACE_CPU("ZenServer::InitializeStructuredCache");
+
 	using namespace std::literals;
 
 	ZEN_INFO("instantiating structured cache service");
@@ -815,6 +822,7 @@ ZenServer::RequestExit(int ExitCode)
 void
 ZenServer::Cleanup()
 {
+	ZEN_TRACE_CPU("ZenServer::Cleanup");
 	ZEN_INFO(ZEN_APP_NAME " cleaning up");
 	try
 	{
@@ -838,8 +846,6 @@ ZenServer::Cleanup()
 
 		Flush();
 
-		ShutdownWorkerPools();
-
 		m_AdminService.reset();
 		m_VfsService.reset();
 		m_ObjStoreService.reset();
@@ -863,6 +869,9 @@ ZenServer::Cleanup()
 		m_AuthService.reset();
 		m_AuthMgr.reset();
 		m_Http = {};
+
+		ShutdownWorkerPools();
+
 		m_JobQueue.reset();
 	}
 	catch (const std::exception& Ex)
@@ -1042,6 +1051,8 @@ ZenServer::CheckOwnerPid()
 void
 ZenServer::Flush()
 {
+	ZEN_TRACE_CPU("ZenServer::Flush");
+
 	for (auto& It : m_CidStores)
 	{
 		It.second->Flush();
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index a15a2e084..a7ef401d5 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -1708,6 +1708,7 @@ GcScheduler::~GcScheduler()
 void
 GcScheduler::Initialize(const GcSchedulerConfig& Config)
 {
+	ZEN_TRACE_CPU("GcScheduler::Initialize");
 	ZEN_MEMSCOPE(GetGcTag());
 
 	using namespace std::chrono;
@@ -1774,6 +1775,7 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
 void
 GcScheduler::Shutdown()
 {
+	ZEN_TRACE_CPU("GcScheduler::Shutdown");
 	ZEN_MEMSCOPE(GetGcTag());
 
 	if (static_cast<uint32_t>(GcSchedulerStatus::kStopped) != m_Status)