fix fork() issues on linux and MacOS (#910)

- Improvement: Hub child process spawning on macOS now uses `posix_spawn` in line with Apple recommendations
- Bugfix: Hub child process spawning on Linux now uses `vfork` instead of `fork`, preventing ENOMEM failures on systems with strict memory overcommit (`vm.overcommit_memory=2`)
- Bugfix: Fixed process group management on POSIX; child processes were not placed into the correct process group, breaking group-wide signal delivery

6 files changed, 174 insertions, 22 deletions

diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h
index 5ae7fad68..8cbed781d 100644
--- a/src/zencore/include/zencore/process.h
+++ b/src/zencore/include/zencore/process.h
@@ -174,9 +174,11 @@ struct CreateProcOptions
 		// allocated and no conhost.exe is spawned. Stdout/stderr still work when redirected
 		// via pipes. Prefer this for headless worker processes.
 		Flag_NoConsole = 1 << 3,
-		// Create the child in a new process group (CREATE_NEW_PROCESS_GROUP on Windows).
-		// Allows sending CTRL_BREAK_EVENT to the child group without affecting the parent.
-		Flag_Windows_NewProcessGroup = 1 << 4,
+		// Spawn the child as a new process group leader (its pgid = its own pid).
+		// On Windows: CREATE_NEW_PROCESS_GROUP, enables CTRL_BREAK_EVENT targeting.
+		// On POSIX: child calls setpgid(0,0) / posix_spawn with POSIX_SPAWN_SETPGROUP+pgid=0.
+		// Mutually exclusive with ProcessGroupId > 0.
+		Flag_NewProcessGroup = 1 << 4,
 		// Allocate a hidden console for the child (CREATE_NO_WINDOW on Windows). Unlike
 		// Flag_NoConsole the child still gets a console (and a conhost.exe) but no visible
 		// window. Use this when the child needs a console for stdio but should not show a window.
@@ -197,9 +199,9 @@ struct CreateProcOptions
 #if ZEN_PLATFORM_WINDOWS
 	JobObject* AssignToJob = nullptr;  // When set, the process is created suspended, assigned to the job, then resumed
 #else
-	/// POSIX process group id. When > 0, the child is placed into this process
-	/// group via setpgid() before exec. Use the pid of the first child as the
-	/// pgid to create a group, then pass the same pgid for subsequent children.
+	/// When > 0, child joins this existing process group. Mutually exclusive with
+	/// Flag_NewProcessGroup; use that flag on the first spawn to create the group,
+	/// then pass the resulting pid here for subsequent spawns to join it.
 	int ProcessGroupId = 0;
 #endif
 };
diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp
index 9cbbfa56a..ee821944a 100644
--- a/src/zencore/process.cpp
+++ b/src/zencore/process.cpp
@@ -37,7 +37,9 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #endif
 
 #if ZEN_PLATFORM_MAC
+#	include <crt_externs.h>
 #	include <libproc.h>
+#	include <spawn.h>
 #	include <sys/types.h>
 #	include <sys/sysctl.h>
 #endif
@@ -135,8 +137,68 @@ IsZombieProcess(int pid, std::error_code& OutEc)
 	}
 	return false;
 }
+
+static char**
+GetEnviron()
+{
+	return *_NSGetEnviron();
+}
 #endif	// ZEN_PLATFORM_MAC
 
+#if ZEN_PLATFORM_LINUX
+static char**
+GetEnviron()
+{
+	return environ;
+}
+#endif	// ZEN_PLATFORM_LINUX
+
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+// Holds a null-terminated envp array built by merging the current process environment with
+// a set of overrides. When Overrides is empty, Data points directly to environ (no allocation).
+// Must outlive any posix_spawn / execve call that receives Data.
+struct EnvpHolder
+{
+	char** Data = GetEnviron();
+
+	explicit EnvpHolder(const std::vector<std::pair<std::string, std::string>>& Overrides)
+	{
+		if (Overrides.empty())
+		{
+			return;
+		}
+		std::map<std::string, std::string> EnvMap;
+		for (char** E = GetEnviron(); *E; ++E)
+		{
+			std::string_view Entry(*E);
+			const size_t	 EqPos = Entry.find('=');
+			if (EqPos != std::string_view::npos)
+			{
+				EnvMap[std::string(Entry.substr(0, EqPos))] = std::string(Entry.substr(EqPos + 1));
+			}
+		}
+		for (const auto& [Key, Value] : Overrides)
+		{
+			EnvMap[Key] = Value;
+		}
+		for (const auto& [Key, Value] : EnvMap)
+		{
+			m_Strings.push_back(Key + "=" + Value);
+		}
+		for (std::string& S : m_Strings)
+		{
+			m_Ptrs.push_back(S.data());
+		}
+		m_Ptrs.push_back(nullptr);
+		Data = m_Ptrs.data();
+	}
+
+private:
+	std::vector<std::string> m_Strings;
+	std::vector<char*>		 m_Ptrs;
+};
+#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+
 //////////////////////////////////////////////////////////////////////////
 // Pipe creation for child process stdout capture
 
@@ -691,6 +753,7 @@ BuildArgV(std::vector<char*>& Out, char* CommandLine)
 		++Cursor;
 	}
 }
+
 #endif	// !WINDOWS || TESTS
 
 #if ZEN_PLATFORM_WINDOWS
@@ -766,7 +829,7 @@ CreateProcNormal(const std::filesystem::path& Executable, std::string_view Comma
 	{
 		CreationFlags |= CREATE_NO_WINDOW;
 	}
-	if (Options.Flags & CreateProcOptions::Flag_Windows_NewProcessGroup)
+	if (Options.Flags & CreateProcOptions::Flag_NewProcessGroup)
 	{
 		CreationFlags |= CREATE_NEW_PROCESS_GROUP;
 	}
@@ -1070,23 +1133,30 @@ CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine
 	}
 
 	return CreateProcNormal(Executable, CommandLine, Options);
-#else
+#elif ZEN_PLATFORM_LINUX
+	// vfork uses CLONE_VM|CLONE_VFORK: the child shares the parent's address space and the
+	// parent is suspended until the child calls exec or _exit. This avoids page-table duplication
+	// and the ENOMEM that fork() produces on systems with strict overcommit (vm.overcommit_memory=2).
+	// All child-side setup uses only syscalls that do not modify user-space memory.
+	// Environment overrides are merged into envp before vfork so that setenv() is never called
+	// from the child (which would corrupt the shared address space).
 	std::vector<char*> ArgV;
 	std::string		   CommandLineZ(CommandLine);
 	BuildArgV(ArgV, CommandLineZ.data());
 	ArgV.push_back(nullptr);
 
-	int ChildPid = fork();
+	EnvpHolder Envp(Options.Environment);
+
+	int ChildPid = vfork();
 	if (ChildPid < 0)
 	{
-		ThrowLastError("Failed to fork a new child process");
+		ThrowLastError("Failed to vfork a new child process");
 	}
 	else if (ChildPid == 0)
 	{
 		if (Options.WorkingDirectory != nullptr)
 		{
-			int Result = chdir(Options.WorkingDirectory->c_str());
-			ZEN_UNUSED(Result);
+			chdir(Options.WorkingDirectory->c_str());
 		}
 
 		if (Options.StdoutPipe != nullptr && Options.StdoutPipe->WriteFd >= 0)
@@ -1118,23 +1188,99 @@ CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine
 			}
 		}
 
-		if (Options.ProcessGroupId > 0)
+		if (Options.Flags & CreateProcOptions::Flag_NewProcessGroup)
+		{
+			setpgid(0, 0);
+		}
+		else if (Options.ProcessGroupId > 0)
 		{
 			setpgid(0, Options.ProcessGroupId);
 		}
 
-		for (const auto& [Key, Value] : Options.Environment)
+		execve(Executable.c_str(), ArgV.data(), Envp.Data);
+		_exit(127);
+	}
+
+	return ChildPid;
+#else  // macOS
+	std::vector<char*> ArgV;
+	std::string		   CommandLineZ(CommandLine);
+	BuildArgV(ArgV, CommandLineZ.data());
+	ArgV.push_back(nullptr);
+
+	posix_spawn_file_actions_t FileActions;
+	posix_spawnattr_t		   Attr;
+
+	int Err = posix_spawn_file_actions_init(&FileActions);
+	if (Err != 0)
+	{
+		ThrowSystemError(Err, "posix_spawn_file_actions_init failed");
+	}
+	auto FileActionsGuard = MakeGuard([&] { posix_spawn_file_actions_destroy(&FileActions); });
+
+	Err = posix_spawnattr_init(&Attr);
+	if (Err != 0)
+	{
+		ThrowSystemError(Err, "posix_spawnattr_init failed");
+	}
+	auto AttrGuard = MakeGuard([&] { posix_spawnattr_destroy(&Attr); });
+
+	if (Options.WorkingDirectory != nullptr)
+	{
+		Err = posix_spawn_file_actions_addchdir_np(&FileActions, Options.WorkingDirectory->c_str());
+		if (Err != 0)
 		{
-			setenv(Key.c_str(), Value.c_str(), 1);
+			ThrowSystemError(Err, "posix_spawn_file_actions_addchdir_np failed");
 		}
+	}
+
+	if (Options.StdoutPipe != nullptr && Options.StdoutPipe->WriteFd >= 0)
+	{
+		const int StdoutWriteFd = Options.StdoutPipe->WriteFd;
+		ZEN_ASSERT(StdoutWriteFd > STDERR_FILENO);
+		posix_spawn_file_actions_adddup2(&FileActions, StdoutWriteFd, STDOUT_FILENO);
 
-		if (execv(Executable.c_str(), ArgV.data()) < 0)
+		if (Options.StderrPipe != nullptr && Options.StderrPipe->WriteFd >= 0)
 		{
-			ThrowLastError("Failed to exec() a new process image");
+			const int StderrWriteFd = Options.StderrPipe->WriteFd;
+			ZEN_ASSERT(StderrWriteFd > STDERR_FILENO && StderrWriteFd != StdoutWriteFd);
+			posix_spawn_file_actions_adddup2(&FileActions, StderrWriteFd, STDERR_FILENO);
+			posix_spawn_file_actions_addclose(&FileActions, StderrWriteFd);
 		}
+		else
+		{
+			posix_spawn_file_actions_adddup2(&FileActions, StdoutWriteFd, STDERR_FILENO);
+		}
+
+		posix_spawn_file_actions_addclose(&FileActions, StdoutWriteFd);
+	}
+	else if (!Options.StdoutFile.empty())
+	{
+		posix_spawn_file_actions_addopen(&FileActions, STDOUT_FILENO, Options.StdoutFile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
+		posix_spawn_file_actions_adddup2(&FileActions, STDOUT_FILENO, STDERR_FILENO);
 	}
 
-	return ChildPid;
+	if (Options.Flags & CreateProcOptions::Flag_NewProcessGroup)
+	{
+		posix_spawnattr_setflags(&Attr, POSIX_SPAWN_SETPGROUP);
+		posix_spawnattr_setpgroup(&Attr, 0);
+	}
+	else if (Options.ProcessGroupId > 0)
+	{
+		posix_spawnattr_setflags(&Attr, POSIX_SPAWN_SETPGROUP);
+		posix_spawnattr_setpgroup(&Attr, Options.ProcessGroupId);
+	}
+
+	EnvpHolder Envp(Options.Environment);
+
+	pid_t ChildPid = 0;
+	Err			   = posix_spawn(&ChildPid, Executable.c_str(), &FileActions, &Attr, ArgV.data(), Envp.Data);
+	if (Err != 0)
+	{
+		ThrowSystemError(Err, "Failed to posix_spawn a new child process");
+	}
+
+	return int(ChildPid);
 #endif
 }
 
diff --git a/src/zennomad/nomadprocess.cpp b/src/zennomad/nomadprocess.cpp
index 1ae968fb7..deecdef05 100644
--- a/src/zennomad/nomadprocess.cpp
+++ b/src/zennomad/nomadprocess.cpp
@@ -37,7 +37,7 @@ struct NomadProcess::Impl
 		}
 
 		CreateProcOptions Options;
-		Options.Flags |= CreateProcOptions::Flag_Windows_NewProcessGroup;
+		Options.Flags |= CreateProcOptions::Flag_NewProcessGroup;
 
 		CreateProcResult Result = CreateProc("nomad" ZEN_EXE_SUFFIX_LITERAL, "nomad" ZEN_EXE_SUFFIX_LITERAL " agent -dev", Options);
 
diff --git a/src/zenutil/cloud/minioprocess.cpp b/src/zenutil/cloud/minioprocess.cpp
index 457453bd8..e146f6677 100644
--- a/src/zenutil/cloud/minioprocess.cpp
+++ b/src/zenutil/cloud/minioprocess.cpp
@@ -45,7 +45,7 @@ struct MinioProcess::Impl
 		}
 
 		CreateProcOptions Options;
-		Options.Flags |= CreateProcOptions::Flag_Windows_NewProcessGroup;
+		Options.Flags |= CreateProcOptions::Flag_NewProcessGroup;
 		Options.Environment.emplace_back("MINIO_ROOT_USER", m_Options.RootUser);
 		Options.Environment.emplace_back("MINIO_ROOT_PASSWORD", m_Options.RootPassword);
 
diff --git a/src/zenutil/consul/consul.cpp b/src/zenutil/consul/consul.cpp
index ad1b92b38..c0cea20c0 100644
--- a/src/zenutil/consul/consul.cpp
+++ b/src/zenutil/consul/consul.cpp
@@ -31,7 +31,7 @@ struct ConsulProcess::Impl
 		}
 
 		CreateProcOptions Options;
-		Options.Flags |= CreateProcOptions::Flag_Windows_NewProcessGroup;
+		Options.Flags |= CreateProcOptions::Flag_NewProcessGroup;
 
 		const std::filesystem::path ConsulExe = GetRunningExecutablePath().parent_path() / ("consul" ZEN_EXE_SUFFIX_LITERAL);
 		CreateProcResult			Result	  = CreateProc(ConsulExe, "consul" ZEN_EXE_SUFFIX_LITERAL " agent -dev", Options);
diff --git a/src/zenutil/process/subprocessmanager.cpp b/src/zenutil/process/subprocessmanager.cpp
index b053ac6bd..e908dd63a 100644
--- a/src/zenutil/process/subprocessmanager.cpp
+++ b/src/zenutil/process/subprocessmanager.cpp
@@ -903,7 +903,11 @@ ProcessGroup::Impl::Spawn(const std::filesystem::path& Executable,
 		Options.AssignToJob = &m_JobObject;
 	}
 #else
-	if (m_Pgid > 0)
+	if (m_Pgid == 0)
+	{
+		Options.Flags |= CreateProcOptions::Flag_NewProcessGroup;
+	}
+	else
 	{
 		Options.ProcessGroupId = m_Pgid;
 	}