hub instance malloc trace (#946)

`--hub-instance-malloc` selects the memory allocator for child instances
`--hub-instance-trace` sets trace channels for child instances
`--hub-instance-tracehost` sets the trace streaming host for child instances
`--hub-instance-tracefile` sets the trace output file for child instances
add {moduleid} and {port} placeholder support for tracefile

8 files changed, 91 insertions, 0 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fd8d78238..b577630d5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,11 @@
 - Improvement: `--consul-register-hub` option to disable hub parent service Consul registration while keeping instance registration active
 - Bugfix: Added logic to shared memory instance state management to ensure unclean shutdown followed by restart with identical pid doesn't lead to errors. Particularly likely to happen when running on k8s
 - Improvement: Dashboard service stats panes are now always visible on service pages (projects, cache, builds, workspaces) rather than hidden until data arrives
+- Feature: Hub instance forwarding options for memory allocator and tracing
+  - `--hub-instance-malloc` selects the memory allocator for child instances
+  - `--hub-instance-trace` sets trace channels for child instances
+  - `--hub-instance-tracehost` sets the trace streaming host for child instances
+  - `--hub-instance-tracefile` sets the trace output file for child instances; supports `{moduleid}` and `{port}` placeholders
 - Bugfix: Dashboard stats tiles no longer flicker on page load when WebSocket updates arrive before all stats are available
 
 ## 5.8.3
diff --git a/docs/hub.md b/docs/hub.md
index 310087284..27cf98512 100644
--- a/docs/hub.md
+++ b/docs/hub.md
@@ -182,6 +182,10 @@ persist across hibernation and are removed only on deprovision.
 | `--hub-instance-corelimit`         | `hub.instance.corelimit`       | `0`                                  | Concurrency limit for child instances. `0` is automatic. |
 | `--hub-instance-provision-threads` | `hub.instance.provisionthreads` | `max(cpu/4, 2)`                     | Thread count for the instance provisioning worker pool. Controls parallel I/O during provision and deprovision operations. Set to `0` for synchronous operation. |
 | `--hub-instance-config`            | `hub.instance.config`          | _(none)_                             | Path to a Lua config file passed to every spawned child instance. Use this to configure storage paths, cache sizes, and other storage server settings. See the zenserver configuration documentation. |
+| `--hub-instance-malloc`            | `hub.instance.malloc`          | _(none)_                             | Memory allocator for child instances (`ansi`, `stomp`, `rpmalloc`, `mimalloc`). When unset, instances use their compiled-in default. |
+| `--hub-instance-trace`             | `hub.instance.trace`           | _(none)_                             | Trace channel specification for child instances (e.g. `default`, `cpu,log`, `memory`). When set, instances start with tracing enabled on the specified channels. |
+| `--hub-instance-tracehost`         | `hub.instance.tracehost`       | _(none)_                             | Trace host for child instances. Instances stream trace data to this host. |
+| `--hub-instance-tracefile`         | `hub.instance.tracefile`       | _(none)_                             | Trace file path for child instances. Supports `{moduleid}` and `{port}` placeholders, resolved per instance. Without placeholders all instances write to the same file. |
 
 ---
 
diff --git a/src/zenserver/hub/hub.cpp b/src/zenserver/hub/hub.cpp
index ac6da9141..0cb25fdd6 100644
--- a/src/zenserver/hub/hub.cpp
+++ b/src/zenserver/hub/hub.cpp
@@ -332,6 +332,10 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo)
 														 .HttpThreadCount			   = m_Config.InstanceHttpThreadCount,
 														 .CoreLimit					   = m_Config.InstanceCoreLimit,
 														 .ConfigPath				   = m_Config.InstanceConfigPath,
+														 .Malloc					   = m_Config.InstanceMalloc,
+														 .Trace						   = m_Config.InstanceTrace,
+														 .TraceHost					   = m_Config.InstanceTraceHost,
+														 .TraceFile					   = m_Config.InstanceTraceFile,
 														 .OptionalWorkerPool		   = m_Config.OptionalHydrationWorkerPool},
 					ModuleId);
 
diff --git a/src/zenserver/hub/hub.h b/src/zenserver/hub/hub.h
index 97a238c01..040f34af5 100644
--- a/src/zenserver/hub/hub.h
+++ b/src/zenserver/hub/hub.h
@@ -67,6 +67,10 @@ public:
 
 		uint32_t			  InstanceHttpThreadCount = 0;	// Automatic
 		int					  InstanceCoreLimit		  = 0;	// Automatic
+		std::string			  InstanceMalloc;
+		std::string			  InstanceTrace;
+		std::string			  InstanceTraceHost;
+		std::string			  InstanceTraceFile;
 		std::filesystem::path InstanceConfigPath;
 		std::string			  HydrationTargetSpecification;
 		CbObject			  HydrationOptions;
diff --git a/src/zenserver/hub/storageserverinstance.cpp b/src/zenserver/hub/storageserverinstance.cpp
index b31a64e56..af2c19113 100644
--- a/src/zenserver/hub/storageserverinstance.cpp
+++ b/src/zenserver/hub/storageserverinstance.cpp
@@ -48,6 +48,36 @@ StorageServerInstance::SpawnServerProcess()
 	{
 		AdditionalOptions << " --config=\"" << MakeSafeAbsolutePath(m_Config.ConfigPath).string() << "\"";
 	}
+	if (!m_Config.Malloc.empty())
+	{
+		AdditionalOptions << " --malloc=" << m_Config.Malloc;
+	}
+	if (!m_Config.Trace.empty())
+	{
+		AdditionalOptions << " --trace=" << m_Config.Trace;
+	}
+	if (!m_Config.TraceHost.empty())
+	{
+		AdditionalOptions << " --tracehost=" << m_Config.TraceHost;
+	}
+	if (!m_Config.TraceFile.empty())
+	{
+		constexpr std::string_view ModuleIdPattern = "{moduleid}";
+		constexpr std::string_view PortPattern	   = "{port}";
+
+		std::string ResolvedTraceFile = m_Config.TraceFile;
+		for (size_t Pos = ResolvedTraceFile.find(ModuleIdPattern); Pos != std::string::npos;
+			 Pos		= ResolvedTraceFile.find(ModuleIdPattern, Pos))
+		{
+			ResolvedTraceFile.replace(Pos, ModuleIdPattern.length(), m_ModuleId);
+		}
+		std::string PortStr = fmt::format("{}", m_Config.BasePort);
+		for (size_t Pos = ResolvedTraceFile.find(PortPattern); Pos != std::string::npos; Pos = ResolvedTraceFile.find(PortPattern, Pos))
+		{
+			ResolvedTraceFile.replace(Pos, PortPattern.length(), PortStr);
+		}
+		AdditionalOptions << " --tracefile=\"" << ResolvedTraceFile << "\"";
+	}
 
 	m_ServerInstance.SpawnServerAndWaitUntilReady(m_Config.BasePort, AdditionalOptions.ToView());
 	ZEN_DEBUG("Storage server instance for module '{}' started, listening on port {}", m_ModuleId, m_Config.BasePort);
diff --git a/src/zenserver/hub/storageserverinstance.h b/src/zenserver/hub/storageserverinstance.h
index e2c5eb562..80f8a5016 100644
--- a/src/zenserver/hub/storageserverinstance.h
+++ b/src/zenserver/hub/storageserverinstance.h
@@ -34,6 +34,10 @@ public:
 		uint32_t			  HttpThreadCount = 0;	// Automatic
 		int					  CoreLimit		  = 0;	// Automatic
 		std::filesystem::path ConfigPath;
+		std::string			  Malloc;
+		std::string			  Trace;
+		std::string			  TraceHost;
+		std::string			  TraceFile;
 
 		WorkerThreadPool* OptionalWorkerPool = nullptr;
 	};
diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp
index a32c1f1d1..749b88000 100644
--- a/src/zenserver/hub/zenhubserver.cpp
+++ b/src/zenserver/hub/zenhubserver.cpp
@@ -142,6 +142,34 @@ ZenHubServerConfigurator::AddCliOptions(cxxopts::Options& Options)
 
 	Options.add_option("hub",
 					   "",
+					   "hub-instance-malloc",
+					   "Select memory allocator for provisioned instances (ansi|stomp|rpmalloc|mimalloc)",
+					   cxxopts::value<std::string>(m_ServerOptions.HubInstanceMalloc)->default_value(""),
+					   "<allocator>");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-instance-trace",
+					   "Trace channel specification for provisioned instances (e.g. default, cpu,log, memory)",
+					   cxxopts::value<std::string>(m_ServerOptions.HubInstanceTrace)->default_value(""),
+					   "<channels>");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-instance-tracehost",
+					   "Trace host for provisioned instances",
+					   cxxopts::value<std::string>(m_ServerOptions.HubInstanceTraceHost)->default_value(""),
+					   "<host>");
+
+	Options.add_option("hub",
+					   "",
+					   "hub-instance-tracefile",
+					   "Trace file path for provisioned instances",
+					   cxxopts::value<std::string>(m_ServerOptions.HubInstanceTraceFile)->default_value(""),
+					   "<path>");
+
+	Options.add_option("hub",
+					   "",
 					   "hub-instance-http-threads",
 					   "Number of http server connection threads for provisioned instances",
 					   cxxopts::value<unsigned int>(m_ServerOptions.HubInstanceHttpThreadCount),
@@ -312,6 +340,10 @@ ZenHubServerConfigurator::AddConfigOptions(LuaConfig::Options& Options)
 
 	Options.AddOption("hub.instance.baseportnumber"sv, m_ServerOptions.HubBasePortNumber, "hub-instance-base-port-number"sv);
 	Options.AddOption("hub.instance.http"sv, m_ServerOptions.HubInstanceHttpClass, "hub-instance-http"sv);
+	Options.AddOption("hub.instance.malloc"sv, m_ServerOptions.HubInstanceMalloc, "hub-instance-malloc"sv);
+	Options.AddOption("hub.instance.trace"sv, m_ServerOptions.HubInstanceTrace, "hub-instance-trace"sv);
+	Options.AddOption("hub.instance.tracehost"sv, m_ServerOptions.HubInstanceTraceHost, "hub-instance-tracehost"sv);
+	Options.AddOption("hub.instance.tracefile"sv, m_ServerOptions.HubInstanceTraceFile, "hub-instance-tracefile"sv);
 	Options.AddOption("hub.instance.httpthreads"sv, m_ServerOptions.HubInstanceHttpThreadCount, "hub-instance-http-threads"sv);
 	Options.AddOption("hub.instance.corelimit"sv, m_ServerOptions.HubInstanceCoreLimit, "hub-instance-corelimit"sv);
 	Options.AddOption("hub.instance.config"sv, m_ServerOptions.HubInstanceConfigPath, "hub-instance-config"sv);
@@ -612,6 +644,10 @@ ZenHubServer::InitializeServices(const ZenHubServerConfig& ServerConfig)
 		.InstanceLimit				  = ServerConfig.HubInstanceLimit,
 		.InstanceHttpThreadCount	  = ServerConfig.HubInstanceHttpThreadCount,
 		.InstanceCoreLimit			  = ServerConfig.HubInstanceCoreLimit,
+		.InstanceMalloc				  = ServerConfig.HubInstanceMalloc,
+		.InstanceTrace				  = ServerConfig.HubInstanceTrace,
+		.InstanceTraceHost			  = ServerConfig.HubInstanceTraceHost,
+		.InstanceTraceFile			  = ServerConfig.HubInstanceTraceFile,
 		.InstanceConfigPath			  = ServerConfig.HubInstanceConfigPath,
 		.HydrationTargetSpecification = ServerConfig.HydrationTargetSpecification,
 		.WatchDog =
diff --git a/src/zenserver/hub/zenhubserver.h b/src/zenserver/hub/zenhubserver.h
index 9860cd629..5e465bb14 100644
--- a/src/zenserver/hub/zenhubserver.h
+++ b/src/zenserver/hub/zenhubserver.h
@@ -48,6 +48,10 @@ struct ZenHubServerConfig : public ZenServerConfig
 	int			HubInstanceLimit	 = 1000;
 	bool		HubUseJobObject		 = true;
 	std::string HubInstanceHttpClass = "asio";
+	std::string HubInstanceMalloc;
+	std::string HubInstanceTrace;
+	std::string HubInstanceTraceHost;
+	std::string HubInstanceTraceFile;
 	uint32_t	HubInstanceHttpThreadCount		= 0;	 // Automatic
 	uint32_t	HubInstanceProvisionThreadCount = 0;	 // Synchronous provisioning
 	uint32_t	HubHydrationThreadCount			= 0;	 // Synchronous hydration/dehydration