use posix_spawn() instead of fork to sidestep virtual memory issuede/switch-to-posix-spawn-on-linux

1 files changed, 112 insertions, 0 deletions

diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp
index 9cbbfa56a..af1eff19a 100644
--- a/src/zencore/process.cpp
+++ b/src/zencore/process.cpp
@@ -42,6 +42,10 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #	include <sys/sysctl.h>
 #endif
 
+#if ZEN_PLATFORM_LINUX
+#	include <spawn.h>
+#endif
+
 #include <fmt/format.h>
 
 ZEN_THIRD_PARTY_INCLUDES_END
@@ -1070,6 +1074,114 @@ CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine
 	}
 
 	return CreateProcNormal(Executable, CommandLine, Options);
+#elif ZEN_PLATFORM_LINUX
+	// posix_spawn uses CLONE_VM|CLONE_VFORK internally and does not copy the parent's
+	// virtual address space, preventing ENOMEM from fork() on systems where strict overcommit
+	// is disabled (vm.overcommit_memory=2) or a per-process RLIMIT_AS limit is enforced.
+	// Requires glibc >= 2.29 for posix_spawn_file_actions_addchdir_np.
+	std::vector<char*> ArgV;
+	std::string		   CommandLineZ(CommandLine);
+	BuildArgV(ArgV, CommandLineZ.data());
+	ArgV.push_back(nullptr);
+
+	posix_spawn_file_actions_t FileActions;
+	posix_spawnattr_t		   Attr;
+
+	int Err = posix_spawn_file_actions_init(&FileActions);
+	if (Err != 0)
+	{
+		ThrowSystemError(Err, "posix_spawn_file_actions_init failed");
+	}
+	auto FileActionsGuard = MakeGuard([&] { posix_spawn_file_actions_destroy(&FileActions); });
+
+	Err = posix_spawnattr_init(&Attr);
+	if (Err != 0)
+	{
+		ThrowSystemError(Err, "posix_spawnattr_init failed");
+	}
+	auto AttrGuard = MakeGuard([&] { posix_spawnattr_destroy(&Attr); });
+
+	if (Options.WorkingDirectory != nullptr)
+	{
+		Err = posix_spawn_file_actions_addchdir_np(&FileActions, Options.WorkingDirectory->c_str());
+		if (Err != 0)
+		{
+			ThrowSystemError(Err, "posix_spawn_file_actions_addchdir_np failed");
+		}
+	}
+
+	if (Options.StdoutPipe != nullptr && Options.StdoutPipe->WriteFd >= 0)
+	{
+		const int StdoutWriteFd = Options.StdoutPipe->WriteFd;
+		ZEN_ASSERT(StdoutWriteFd > STDERR_FILENO);
+		posix_spawn_file_actions_adddup2(&FileActions, StdoutWriteFd, STDOUT_FILENO);
+
+		if (Options.StderrPipe != nullptr && Options.StderrPipe->WriteFd >= 0)
+		{
+			const int StderrWriteFd = Options.StderrPipe->WriteFd;
+			ZEN_ASSERT(StderrWriteFd > STDERR_FILENO && StderrWriteFd != StdoutWriteFd);
+			posix_spawn_file_actions_adddup2(&FileActions, StderrWriteFd, STDERR_FILENO);
+			posix_spawn_file_actions_addclose(&FileActions, StderrWriteFd);
+		}
+		else
+		{
+			posix_spawn_file_actions_adddup2(&FileActions, StdoutWriteFd, STDERR_FILENO);
+		}
+
+		posix_spawn_file_actions_addclose(&FileActions, StdoutWriteFd);
+	}
+	else if (!Options.StdoutFile.empty())
+	{
+		posix_spawn_file_actions_addopen(&FileActions, STDOUT_FILENO, Options.StdoutFile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
+		posix_spawn_file_actions_adddup2(&FileActions, STDOUT_FILENO, STDERR_FILENO);
+	}
+
+	if (Options.ProcessGroupId > 0)
+	{
+		posix_spawnattr_setflags(&Attr, POSIX_SPAWN_SETPGROUP);
+		posix_spawnattr_setpgroup(&Attr, Options.ProcessGroupId);
+	}
+
+	// Build custom envp if overrides are specified; otherwise inherit parent environment unchanged.
+	char**					 Envp = environ;
+	std::vector<std::string> EnvStrings;
+	std::vector<char*>		 EnvPtrs;
+	if (!Options.Environment.empty())
+	{
+		std::map<std::string, std::string> EnvMap;
+		for (char** E = environ; *E; ++E)
+		{
+			std::string_view Entry(*E);
+			const size_t	 EqPos = Entry.find('=');
+			if (EqPos != std::string_view::npos)
+			{
+				EnvMap[std::string(Entry.substr(0, EqPos))] = std::string(Entry.substr(EqPos + 1));
+			}
+		}
+		for (const auto& [Key, Value] : Options.Environment)
+		{
+			EnvMap[Key] = Value;
+		}
+		for (const auto& [Key, Value] : EnvMap)
+		{
+			EnvStrings.push_back(Key + "=" + Value);
+		}
+		for (std::string& S : EnvStrings)
+		{
+			EnvPtrs.push_back(S.data());
+		}
+		EnvPtrs.push_back(nullptr);
+		Envp = EnvPtrs.data();
+	}
+
+	pid_t ChildPid = 0;
+	Err			   = posix_spawn(&ChildPid, Executable.c_str(), &FileActions, &Attr, ArgV.data(), Envp);
+	if (Err != 0)
+	{
+		ThrowSystemError(Err, "Failed to posix_spawn a new child process");
+	}
+
+	return int(ChildPid);
 #else
 	std::vector<char*> ArgV;
 	std::string		   CommandLineZ(CommandLine);