Merge branch 'main' of https://github.com/EpicGames/zen

10 files changed, 519 insertions, 151 deletions

diff --git a/src/zenutil/basicfile.cpp b/src/zenutil/basicfile.cpp
index 1dce71e60..819d0805d 100644
--- a/src/zenutil/basicfile.cpp
+++ b/src/zenutil/basicfile.cpp
@@ -76,16 +76,15 @@ BasicFile::Open(const std::filesystem::path& FileName, Mode InMode, std::error_c
 
 	const DWORD dwShareMode = FILE_SHARE_READ | (EnumHasAllFlags(InMode, Mode::kPreventWrite) ? 0 : FILE_SHARE_WRITE) |
 							  (EnumHasAllFlags(InMode, Mode::kPreventDelete) ? 0 : FILE_SHARE_DELETE);
-	const DWORD dwFlagsAndAttributes =
-		FILE_ATTRIBUTE_NORMAL | (EnumHasAllFlags(InMode, Mode::kDeleteOnClose) ? FILE_FLAG_DELETE_ON_CLOSE : 0);
-	const HANDLE hTemplateFile = nullptr;
-	const HANDLE FileHandle	   = CreateFile(FileName.c_str(),
-											dwDesiredAccess,
-											dwShareMode,
-											/* lpSecurityAttributes */ nullptr,
-											dwCreationDisposition,
-											dwFlagsAndAttributes,
-											hTemplateFile);
+	const DWORD	 dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL;
+	const HANDLE hTemplateFile		  = nullptr;
+	const HANDLE FileHandle			  = CreateFile(FileName.c_str(),
+										   dwDesiredAccess,
+										   dwShareMode,
+										   /* lpSecurityAttributes */ nullptr,
+										   dwCreationDisposition,
+										   dwFlagsAndAttributes,
+										   hTemplateFile);
 
 	if (FileHandle == INVALID_HANDLE_VALUE)
 	{
@@ -192,7 +191,8 @@ BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset)
 
 		if (!Success)
 		{
-			ThrowLastError(fmt::format("Failed to read from file '{}'", zen::PathFromHandle(m_FileHandle)));
+			std::error_code Dummy;
+			ThrowLastError(fmt::format("Failed to read from file '{}'", zen::PathFromHandle(m_FileHandle, Dummy)));
 		}
 
 		BytesToRead -= NumberOfBytesToRead;
@@ -325,7 +325,8 @@ BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset)
 
 	if (Ec)
 	{
-		throw std::system_error(Ec, fmt::format("Failed to write to file '{}'", zen::PathFromHandle(m_FileHandle)));
+		std::error_code Dummy;
+		throw std::system_error(Ec, fmt::format("Failed to write to file '{}'", zen::PathFromHandle(m_FileHandle, Dummy)));
 	}
 }
 
@@ -357,7 +358,8 @@ BasicFile::FileSize()
 		int Error = zen::GetLastError();
 		if (Error)
 		{
-			ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle)));
+			std::error_code Dummy;
+			ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy)));
 		}
 	}
 	return uint64_t(liFileSize.QuadPart);
@@ -367,7 +369,8 @@ BasicFile::FileSize()
 	struct stat Stat;
 	if (fstat(Fd, &Stat) == -1)
 	{
-		ThrowSystemError(GetLastError(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle)));
+		std::error_code Dummy;
+		ThrowSystemError(GetLastError(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy)));
 	}
 	return uint64_t(Stat.st_size);
 #endif
@@ -414,7 +417,9 @@ BasicFile::SetFileSize(uint64_t FileSize)
 		int Error = zen::GetLastError();
 		if (Error)
 		{
-			ThrowSystemError(Error, fmt::format("Failed to set file pointer to {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+			std::error_code Dummy;
+			ThrowSystemError(Error,
+							 fmt::format("Failed to set file pointer to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
 		}
 	}
 	OK = ::SetEndOfFile(m_FileHandle);
@@ -423,7 +428,9 @@ BasicFile::SetFileSize(uint64_t FileSize)
 		int Error = zen::GetLastError();
 		if (Error)
 		{
-			ThrowSystemError(Error, fmt::format("Failed to set end of file to {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+			std::error_code Dummy;
+			ThrowSystemError(Error,
+							 fmt::format("Failed to set end of file to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
 		}
 	}
 #elif ZEN_PLATFORM_MAC
@@ -433,7 +440,9 @@ BasicFile::SetFileSize(uint64_t FileSize)
 		int Error = zen::GetLastError();
 		if (Error)
 		{
-			ThrowSystemError(Error, fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+			std::error_code Dummy;
+			ThrowSystemError(Error,
+							 fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
 		}
 	}
 #else
@@ -443,7 +452,9 @@ BasicFile::SetFileSize(uint64_t FileSize)
 		int Error = zen::GetLastError();
 		if (Error)
 		{
-			ThrowSystemError(Error, fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+			std::error_code Dummy;
+			ThrowSystemError(Error,
+							 fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
 		}
 	}
 	if (FileSize > 0)
@@ -451,7 +462,9 @@ BasicFile::SetFileSize(uint64_t FileSize)
 		int Error = posix_fallocate64(Fd, 0, (off64_t)FileSize);
 		if (Error)
 		{
-			ThrowSystemError(Error, fmt::format("Failed to allocate space of {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+			std::error_code Dummy;
+			ThrowSystemError(Error,
+							 fmt::format("Failed to allocate space of {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
 		}
 	}
 #endif
@@ -588,6 +601,8 @@ LockFile::Update(CbObject Payload, std::error_code& Ec)
 	BasicFile::Write(Payload.GetBuffer(), 0, Ec);
 }
 
+//////////////////////////////////////////////////////////////////////////
+
 BasicFileBuffer::BasicFileBuffer(BasicFile& Base, uint64_t BufferSize)
 : m_Base(Base)
 , m_Buffer(nullptr)
@@ -662,6 +677,79 @@ BasicFileBuffer::MakeView(uint64_t Size, uint64_t FileOffset)
 	return MemoryView(m_Buffer + (FileOffset - m_BufferStart), Size);
 }
 
+//////////////////////////////////////////////////////////////////////////
+
+BasicFileWriter::BasicFileWriter(BasicFile& Base, uint64_t BufferSize)
+: m_Base(Base)
+, m_Buffer(nullptr)
+, m_BufferSize(BufferSize)
+, m_BufferStart(0)
+, m_BufferEnd(0)
+{
+	m_Buffer = (uint8_t*)Memory::Alloc(m_BufferSize);
+}
+
+BasicFileWriter::~BasicFileWriter()
+{
+	Flush();
+	Memory::Free(m_Buffer);
+}
+
+void
+BasicFileWriter::Write(void* Data, uint64_t Size, uint64_t FileOffset)
+{
+	if (m_Buffer == nullptr || (Size >= m_BufferSize))
+	{
+		m_Base.Write(Data, Size, FileOffset);
+		return;
+	}
+
+	// Note that this only supports buffering of sequential writes!
+
+	if (FileOffset != m_BufferEnd)
+	{
+		Flush();
+		m_BufferStart = m_BufferEnd = FileOffset;
+	}
+
+	while (Size)
+	{
+		const uint64_t RemainingBufferCapacity = m_BufferStart + m_BufferSize - m_BufferEnd;
+		const uint64_t BlockWriteBytes		   = Min(RemainingBufferCapacity, Size);
+		const uint64_t BufferWriteOffset	   = FileOffset - m_BufferStart;
+
+		ZEN_ASSERT_SLOW(BufferWriteOffset < m_BufferSize);
+		ZEN_ASSERT_SLOW((BufferWriteOffset + BlockWriteBytes) <= m_BufferSize);
+
+		memcpy(m_Buffer + BufferWriteOffset, Data, BlockWriteBytes);
+
+		Size -= BlockWriteBytes;
+		m_BufferEnd += BlockWriteBytes;
+		FileOffset += BlockWriteBytes;
+
+		if ((m_BufferEnd - m_BufferStart) == m_BufferSize)
+		{
+			Flush();
+		}
+	}
+}
+
+void
+BasicFileWriter::Flush()
+{
+	const uint64_t BufferedBytes = m_BufferEnd - m_BufferStart;
+
+	if (BufferedBytes == 0)
+		return;
+
+	const uint64_t WriteOffset = m_BufferStart;
+	m_BufferStart			   = m_BufferEnd;
+
+	m_Base.Write(m_Buffer, BufferedBytes, WriteOffset);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
 /*
 		___________              __
 		\__    ___/___   _______/  |_  ______
diff --git a/src/zenutil/include/zenutil/basicfile.h b/src/zenutil/include/zenutil/basicfile.h
index 7797258e8..f25d9f23c 100644
--- a/src/zenutil/include/zenutil/basicfile.h
+++ b/src/zenutil/include/zenutil/basicfile.h
@@ -44,7 +44,6 @@ public:
 		kModeMask	   = 0x0007,
 		kPreventDelete = 0x1000'0000,  // Do not open with delete sharing mode (prevent other processes from deleting file while open)
 		kPreventWrite  = 0x2000'0000,  // Do not open with write sharing mode (prevent other processes from writing to file while open)
-		kDeleteOnClose = 0x4000'0000,  // File should be deleted when the last handle is closed
 	};
 
 	void	 Open(const std::filesystem::path& FileName, Mode Mode);
@@ -138,6 +137,13 @@ public:
 	void	   Read(void* Data, uint64_t Size, uint64_t FileOffset);
 	MemoryView MakeView(uint64_t Size, uint64_t FileOffset);
 
+	template<typename T>
+	const T* MakeView(uint64_t FileOffset)
+	{
+		MemoryView View = MakeView(sizeof(T), FileOffset);
+		return reinterpret_cast<const T*>(View.GetData());
+	}
+
 private:
 	BasicFile&	   m_Base;
 	uint8_t*	   m_Buffer;
@@ -147,6 +153,29 @@ private:
 	uint64_t	   m_BufferEnd;
 };
 
+/** Adds a layer of buffered writing to a BasicFile
+
+This class is not intended for concurrent access, it is not thread safe.
+
+*/
+
+class BasicFileWriter
+{
+public:
+	BasicFileWriter(BasicFile& Base, uint64_t BufferSize);
+	~BasicFileWriter();
+
+	void Write(void* Data, uint64_t Size, uint64_t FileOffset);
+	void Flush();
+
+private:
+	BasicFile&	   m_Base;
+	uint8_t*	   m_Buffer;
+	const uint64_t m_BufferSize;
+	uint64_t	   m_BufferStart;
+	uint64_t	   m_BufferEnd;
+};
+
 ZENCORE_API void basicfile_forcelink();
 
 }  // namespace zen
diff --git a/src/zenutil/include/zenutil/logging/fullformatter.h b/src/zenutil/include/zenutil/logging/fullformatter.h
index 498ecd143..146fea7a0 100644
--- a/src/zenutil/include/zenutil/logging/fullformatter.h
+++ b/src/zenutil/include/zenutil/logging/fullformatter.h
@@ -16,136 +16,175 @@ namespace zen::logging {
 class full_formatter final : public spdlog::formatter
 {
 public:
-	full_formatter(std::string_view LogId, std::chrono::time_point<std::chrono::system_clock> Epoch) : m_Epoch(Epoch), m_LogId(LogId) {}
+	full_formatter(std::string_view LogId, std::chrono::time_point<std::chrono::system_clock> Epoch)
+	: m_Epoch(Epoch)
+	, m_LogId(LogId)
+	, m_LinePrefix(128, ' ')
+	, m_UseFullDate(false)
+	{
+	}
 
-	virtual std::unique_ptr<formatter> clone() const override { return std::make_unique<full_formatter>(m_LogId, m_Epoch); }
+	full_formatter(std::string_view LogId) : m_LogId(LogId), m_LinePrefix(128, ' '), m_UseFullDate(true) {}
 
-	static constexpr bool UseDate = false;
+	virtual std::unique_ptr<formatter> clone() const override { return std::make_unique<full_formatter>(m_LogId, m_Epoch); }
 
-	virtual void format(const spdlog::details::log_msg& msg, spdlog::memory_buf_t& dest) override
+	virtual void format(const spdlog::details::log_msg& msg, spdlog::memory_buf_t& OutBuffer) override
 	{
-		using namespace std::literals;
-
-		if constexpr (UseDate)
-		{
-			auto secs = std::chrono::duration_cast<std::chrono::seconds>(msg.time.time_since_epoch());
-			if (secs != m_LastLogSecs)
-			{
-				m_CachedTm	  = spdlog::details::os::localtime(spdlog::log_clock::to_time_t(msg.time));
-				m_LastLogSecs = secs;
-			}
-		}
+		// Note that the sink is responsible for ensuring there is only ever a
+		// single caller in here
 
-		const auto& tm_time = m_CachedTm;
+		using namespace std::literals;
 
-		// cache the date/time part for the next second.
-		auto duration = msg.time - m_Epoch;
-		auto secs	  = std::chrono::duration_cast<std::chrono::seconds>(duration);
+		std::chrono::seconds TimestampSeconds;
 
-		if (m_CacheTimestamp != secs || m_CachedDatetime.size() == 0)
+		if (m_UseFullDate)
 		{
-			m_CachedDatetime.clear();
-			m_CachedDatetime.push_back('[');
-
-			if constexpr (UseDate)
+			TimestampSeconds = std::chrono::duration_cast<std::chrono::seconds>(msg.time.time_since_epoch());
+			if (TimestampSeconds != m_LastLogSecs)
 			{
-				spdlog::details::fmt_helper::append_int(tm_time.tm_year + 1900, m_CachedDatetime);
-				m_CachedDatetime.push_back('-');
+				m_LastLogSecs = TimestampSeconds;
 
-				spdlog::details::fmt_helper::pad2(tm_time.tm_mon + 1, m_CachedDatetime);
+				m_CachedLocalTm = spdlog::details::os::localtime(spdlog::log_clock::to_time_t(msg.time));
+				m_CachedDatetime.clear();
+				m_CachedDatetime.push_back('[');
+				spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_year % 100, m_CachedDatetime);
 				m_CachedDatetime.push_back('-');
-
-				spdlog::details::fmt_helper::pad2(tm_time.tm_mday, m_CachedDatetime);
+				spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_mon + 1, m_CachedDatetime);
+				m_CachedDatetime.push_back('-');
+				spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_mday, m_CachedDatetime);
 				m_CachedDatetime.push_back(' ');
-
-				spdlog::details::fmt_helper::pad2(tm_time.tm_hour, m_CachedDatetime);
+				spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_hour, m_CachedDatetime);
 				m_CachedDatetime.push_back(':');
-
-				spdlog::details::fmt_helper::pad2(tm_time.tm_min, m_CachedDatetime);
+				spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_min, m_CachedDatetime);
 				m_CachedDatetime.push_back(':');
-
-				spdlog::details::fmt_helper::pad2(tm_time.tm_sec, m_CachedDatetime);
+				spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_sec, m_CachedDatetime);
+				m_CachedDatetime.push_back('.');
 			}
-			else
+		}
+		else
+		{
+			auto ElapsedTime = msg.time - m_Epoch;
+			TimestampSeconds = std::chrono::duration_cast<std::chrono::seconds>(ElapsedTime);
+
+			// cache the date/time part for the next second.
+
+			if (m_CacheTimestamp != TimestampSeconds || m_CachedDatetime.size() == 0)
 			{
-				int Count = int(secs.count());
+				m_CacheTimestamp = TimestampSeconds;
 
+				int		  Count	  = int(TimestampSeconds.count());
 				const int LogSecs = Count % 60;
 				Count /= 60;
-
 				const int LogMins = Count % 60;
 				Count /= 60;
-
 				const int LogHours = Count;
 
+				m_CachedDatetime.clear();
+				m_CachedDatetime.push_back('[');
 				spdlog::details::fmt_helper::pad2(LogHours, m_CachedDatetime);
 				m_CachedDatetime.push_back(':');
 				spdlog::details::fmt_helper::pad2(LogMins, m_CachedDatetime);
 				m_CachedDatetime.push_back(':');
 				spdlog::details::fmt_helper::pad2(LogSecs, m_CachedDatetime);
+				m_CachedDatetime.push_back('.');
 			}
-
-			m_CachedDatetime.push_back('.');
-
-			m_CacheTimestamp = secs;
 		}
 
-		dest.append(m_CachedDatetime.begin(), m_CachedDatetime.end());
+		OutBuffer.append(m_CachedDatetime.begin(), m_CachedDatetime.end());
 
 		auto millis = spdlog::details::fmt_helper::time_fraction<std::chrono::milliseconds>(msg.time);
-		spdlog::details::fmt_helper::pad3(static_cast<uint32_t>(millis.count()), dest);
-		dest.push_back(']');
-		dest.push_back(' ');
+		spdlog::details::fmt_helper::pad3(static_cast<uint32_t>(millis.count()), OutBuffer);
+		OutBuffer.push_back(']');
+		OutBuffer.push_back(' ');
 
 		if (!m_LogId.empty())
 		{
-			dest.push_back('[');
-			spdlog::details::fmt_helper::append_string_view(m_LogId, dest);
-			dest.push_back(']');
-			dest.push_back(' ');
+			OutBuffer.push_back('[');
+			spdlog::details::fmt_helper::append_string_view(m_LogId, OutBuffer);
+			OutBuffer.push_back(']');
+			OutBuffer.push_back(' ');
 		}
 
 		// append logger name if exists
 		if (msg.logger_name.size() > 0)
 		{
-			dest.push_back('[');
-			spdlog::details::fmt_helper::append_string_view(msg.logger_name, dest);
-			dest.push_back(']');
-			dest.push_back(' ');
+			OutBuffer.push_back('[');
+			spdlog::details::fmt_helper::append_string_view(msg.logger_name, OutBuffer);
+			OutBuffer.push_back(']');
+			OutBuffer.push_back(' ');
 		}
 
-		dest.push_back('[');
+		OutBuffer.push_back('[');
 		// wrap the level name with color
-		msg.color_range_start = dest.size();
-		spdlog::details::fmt_helper::append_string_view(spdlog::level::to_string_view(msg.level), dest);
-		msg.color_range_end = dest.size();
-		dest.push_back(']');
-		dest.push_back(' ');
+		msg.color_range_start = OutBuffer.size();
+		spdlog::details::fmt_helper::append_string_view(spdlog::level::to_string_view(msg.level), OutBuffer);
+		msg.color_range_end = OutBuffer.size();
+		OutBuffer.push_back(']');
+		OutBuffer.push_back(' ');
 
 		// add source location if present
 		if (!msg.source.empty())
 		{
-			dest.push_back('[');
+			OutBuffer.push_back('[');
 			const char* filename =
 				spdlog::details::short_filename_formatter<spdlog::details::null_scoped_padder>::basename(msg.source.filename);
-			spdlog::details::fmt_helper::append_string_view(filename, dest);
-			dest.push_back(':');
-			spdlog::details::fmt_helper::append_int(msg.source.line, dest);
-			dest.push_back(']');
-			dest.push_back(' ');
+			spdlog::details::fmt_helper::append_string_view(filename, OutBuffer);
+			OutBuffer.push_back(':');
+			spdlog::details::fmt_helper::append_int(msg.source.line, OutBuffer);
+			OutBuffer.push_back(']');
+			OutBuffer.push_back(' ');
 		}
 
-		spdlog::details::fmt_helper::append_string_view(msg.payload, dest);
-		spdlog::details::fmt_helper::append_string_view("\n"sv, dest);
+		// Handle newlines in single log call by prefixing each additional line to make
+		// subsequent lines align with the first line in the message
+
+		const size_t LinePrefixCount = Min<size_t>(OutBuffer.size(), m_LinePrefix.size());
+
+		auto ItLineBegin  = msg.payload.begin();
+		auto ItMessageEnd = msg.payload.end();
+		bool IsFirstline  = true;
+
+		{
+			auto ItLineEnd = ItLineBegin;
+
+			auto EmitLine = [&] {
+				if (IsFirstline)
+				{
+					IsFirstline = false;
+				}
+				else
+				{
+					spdlog::details::fmt_helper::append_string_view(std::string_view(m_LinePrefix.data(), LinePrefixCount), OutBuffer);
+				}
+				spdlog::details::fmt_helper::append_string_view(spdlog::string_view_t(&*ItLineBegin, ItLineEnd - ItLineBegin), OutBuffer);
+			};
+
+			while (ItLineEnd != ItMessageEnd)
+			{
+				if (*ItLineEnd++ == '\n')
+				{
+					EmitLine();
+					ItLineBegin = ItLineEnd;
+				}
+			}
+
+			if (ItLineBegin != ItMessageEnd)
+			{
+				EmitLine();
+				spdlog::details::fmt_helper::append_string_view("\n"sv, OutBuffer);
+			}
+		}
 	}
 
 private:
 	std::chrono::time_point<std::chrono::system_clock> m_Epoch;
-	std::tm											   m_CachedTm;
+	std::tm											   m_CachedLocalTm;
 	std::chrono::seconds							   m_LastLogSecs;
 	std::chrono::seconds							   m_CacheTimestamp{0};
 	spdlog::memory_buf_t							   m_CachedDatetime;
 	std::string										   m_LogId;
+	std::string										   m_LinePrefix;
+	bool											   m_UseFullDate = true;
 };
 
 }  // namespace zen::logging
diff --git a/src/zenutil/include/zenutil/workerpools.h b/src/zenutil/include/zenutil/workerpools.h
new file mode 100644
index 000000000..339120ece
--- /dev/null
+++ b/src/zenutil/include/zenutil/workerpools.h
@@ -0,0 +1,21 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/workthreadpool.h>
+
+namespace zen {
+
+// Worker pool with std::thread::hardware_concurrency() worker threads
+WorkerThreadPool& GetLargeWorkerPool();
+
+// Worker pool with std::thread::hardware_concurrency() / 4 worker threads
+WorkerThreadPool& GetSmallWorkerPool();
+
+// Special worker pool that does not use worker thread but issues all scheduled work on the calling thread
+// This is useful for debugging when multiple async thread can make stepping in debugger complicated
+WorkerThreadPool& GetSyncWorkerPool();
+
+void ShutdownWorkerPools();
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/zenserverprocess.h b/src/zenutil/include/zenutil/zenserverprocess.h
index 60adfba54..15138341c 100644
--- a/src/zenutil/include/zenutil/zenserverprocess.h
+++ b/src/zenutil/include/zenutil/zenserverprocess.h
@@ -4,6 +4,7 @@
 
 #include <zencore/enumflags.h>
 #include <zencore/logging.h>
+#include <zencore/process.h>
 #include <zencore/thread.h>
 #include <zencore/uid.h>
 
@@ -38,6 +39,7 @@ public:
 	inline bool				IsInitialized() const { return m_IsInitialized; }
 	inline bool				IsTestEnvironment() const { return m_IsTestInstance; }
 	inline std::string_view GetServerClass() const { return m_ServerClass; }
+	inline uint16_t			GetNewPortNumber() { return m_NextPortNumber.fetch_add(1); }
 
 private:
 	std::filesystem::path m_ProgramBaseDir;
@@ -45,6 +47,7 @@ private:
 	bool				  m_IsInitialized  = false;
 	bool				  m_IsTestInstance = false;
 	std::string			  m_ServerClass;
+	std::atomic_uint16_t  m_NextPortNumber{20000};
 };
 
 /** Zen Server Instance management
@@ -63,16 +66,29 @@ struct ZenServerInstance
 
 	void			   Shutdown();
 	void			   SignalShutdown();
-	void			   WaitUntilReady();
+	uint16_t		   WaitUntilReady();
 	[[nodiscard]] bool WaitUntilReady(int Timeout);
 	void			   EnableTermination() { m_Terminate = true; }
 	void			   DisableShutdownOnDestroy() { m_ShutdownOnDestroy = false; }
 	void			   Detach();
 	inline int		   GetPid() { return m_Process.Pid(); }
 	inline void		   SetOwnerPid(int Pid) { m_OwnerPid = Pid; }
+	bool			   IsRunning();
 
-	void		SetTestDir(std::filesystem::path TestDir);
-	inline void SpawnServer(int BasePort = 0, std::string_view AdditionalServerArgs = std::string_view())
+	void SetTestDir(std::filesystem::path TestDir);
+
+	inline void SpawnServer(std::string_view AdditionalServerArgs = std::string_view())
+	{
+		SpawnServer(m_Env.GetNewPortNumber(), AdditionalServerArgs, /* WaitTimeoutMs */ 0);
+	}
+
+	inline uint16_t SpawnServerAndWaitUntilReady(std::string_view AdditionalServerArgs = std::string_view())
+	{
+		SpawnServer(m_Env.GetNewPortNumber(), AdditionalServerArgs, /* WaitTimeoutMs */ 100'000);
+		return GetBasePort();
+	}
+
+	inline void SpawnServer(int BasePort, std::string_view AdditionalServerArgs = std::string_view())
 	{
 		SpawnServer(BasePort, AdditionalServerArgs, /* WaitTimeoutMs */ 0);
 	}
@@ -84,6 +100,7 @@ struct ZenServerInstance
 
 	void		AttachToRunningServer(int BasePort = 0);
 	std::string GetBaseUri() const;
+	uint16_t	GetBasePort() const { return m_BasePort; }
 
 private:
 	ZenServerEnvironment& m_Env;
@@ -93,11 +110,13 @@ private:
 	bool				  m_Terminate		  = false;
 	bool				  m_ShutdownOnDestroy = true;
 	std::filesystem::path m_TestDir;
-	int					  m_BasePort = 0;
+	uint16_t			  m_BasePort = 0;
 	std::optional<int>	  m_OwnerPid;
+	std::string			  m_Name;
 
 	void CreateShutdownEvent(int BasePort);
 	void SpawnServer(int BasePort, std::string_view AdditionalServerArgs, int WaitTimeoutMs);
+	void OnServerReady();
 };
 
 /** Shared system state
diff --git a/src/zenutil/include/zenutil/zenutil.h b/src/zenutil/include/zenutil/zenutil.h
index 14d21ea0d..662743de8 100644
--- a/src/zenutil/include/zenutil/zenutil.h
+++ b/src/zenutil/include/zenutil/zenutil.h
@@ -1,3 +1,9 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
 #pragma once
+
+namespace zen {
+
+void zenutil_forcelinktests();
+
+}
diff --git a/src/zenutil/logging.cpp b/src/zenutil/logging.cpp
index 512c7901c..fedfdc7e8 100644
--- a/src/zenutil/logging.cpp
+++ b/src/zenutil/logging.cpp
@@ -12,6 +12,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 #include <zencore/compactbinary.h>
 #include <zencore/filesystem.h>
+#include <zencore/logging.h>
 #include <zencore/string.h>
 #include <zenutil/logging/fullformatter.h>
 #include <zenutil/logging/jsonformatter.h>
@@ -152,24 +153,25 @@ BeginInitializeLogging(const LoggingOptions& LogOptions)
 void
 FinishInitializeLogging(const LoggingOptions& LogOptions)
 {
-	spdlog::level::level_enum LogLevel = spdlog::level::info;
+	logging::level::LogLevel LogLevel = logging::level::Info;
 
 	if (LogOptions.IsDebug)
 	{
-		LogLevel = spdlog::level::debug;
+		LogLevel = logging::level::Debug;
 	}
 
 	if (LogOptions.IsTest)
 	{
-		LogLevel = spdlog::level::trace;
+		LogLevel = logging::level::Trace;
 	}
 
 	// Configure all registered loggers according to settings
 
-	spdlog::set_level(LogLevel);
+	logging::RefreshLogLevels(LogLevel);
 	spdlog::flush_on(spdlog::level::err);
 	spdlog::flush_every(std::chrono::seconds{2});
-	spdlog::set_formatter(std::make_unique<logging::full_formatter>(LogOptions.LogId, std::chrono::system_clock::now()));
+	spdlog::set_formatter(
+		std::make_unique<logging::full_formatter>(LogOptions.LogId, std::chrono::system_clock::now()));	 // default to duration prefix
 
 	if (g_FileSink)
 	{
@@ -179,7 +181,7 @@ FinishInitializeLogging(const LoggingOptions& LogOptions)
 		}
 		else
 		{
-			g_FileSink->set_pattern("[%C-%m-%d.%e %T] [%n] [%l] %v");
+			g_FileSink->set_formatter(std::make_unique<logging::full_formatter>(LogOptions.LogId));	 // this will have a date prefix
 		}
 	}
 
diff --git a/src/zenutil/workerpools.cpp b/src/zenutil/workerpools.cpp
new file mode 100644
index 000000000..3ae302064
--- /dev/null
+++ b/src/zenutil/workerpools.cpp
@@ -0,0 +1,95 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zenutil/workerpools.h"
+
+#include <zencore/intmath.h>
+#include <zencore/thread.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+namespace {
+	const int LargeWorkerThreadPoolTreadCount = gsl::narrow<int>(std::thread::hardware_concurrency());
+	const int SmallWorkerThreadPoolTreadCount = gsl::narrow<int>(Max((std::thread::hardware_concurrency() / 4u), 1u));
+
+	static bool IsShutDown = false;
+
+	RwLock PoolLock;
+
+	std::unique_ptr<WorkerThreadPool> LargeWorkerPool;
+	std::unique_ptr<WorkerThreadPool> SmallWorkerPool;
+	std::unique_ptr<WorkerThreadPool> SyncWorkerPool;
+}  // namespace
+
+WorkerThreadPool&
+GetLargeWorkerPool()
+{
+	{
+		RwLock::SharedLockScope _(PoolLock);
+		if (LargeWorkerPool)
+		{
+			return *LargeWorkerPool;
+		}
+	}
+	RwLock::ExclusiveLockScope _(PoolLock);
+	ZEN_ASSERT(!IsShutDown);
+	if (LargeWorkerPool)
+	{
+		return *LargeWorkerPool;
+	}
+	LargeWorkerPool.reset(new WorkerThreadPool(LargeWorkerThreadPoolTreadCount, "LargeThreadPool"));
+	return *LargeWorkerPool;
+}
+
+WorkerThreadPool&
+GetSmallWorkerPool()
+{
+	{
+		RwLock::SharedLockScope _(PoolLock);
+		if (SmallWorkerPool)
+		{
+			return *SmallWorkerPool;
+		}
+	}
+	RwLock::ExclusiveLockScope _(PoolLock);
+	ZEN_ASSERT(!IsShutDown);
+	if (SmallWorkerPool)
+	{
+		return *SmallWorkerPool;
+	}
+	SmallWorkerPool.reset(new WorkerThreadPool(SmallWorkerThreadPoolTreadCount, "SmallThreadPool"));
+	return *SmallWorkerPool;
+}
+
+WorkerThreadPool&
+GetSyncWorkerPool()
+{
+	{
+		RwLock::SharedLockScope _(PoolLock);
+		if (SyncWorkerPool)
+		{
+			return *SyncWorkerPool;
+		}
+	}
+	RwLock::ExclusiveLockScope _(PoolLock);
+	ZEN_ASSERT(!IsShutDown);
+	if (SyncWorkerPool)
+	{
+		return *SyncWorkerPool;
+	}
+	SyncWorkerPool.reset(new WorkerThreadPool(0, "SyncThreadPool"));
+	return *SyncWorkerPool;
+}
+
+void
+ShutdownWorkerPools()
+{
+	RwLock::ExclusiveLockScope _(PoolLock);
+	IsShutDown = true;
+	LargeWorkerPool.reset();
+	SmallWorkerPool.reset();
+	SyncWorkerPool.reset();
+}
+}  // namespace zen
diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp
index 83c6668ba..909692fbc 100644
--- a/src/zenutil/zenserverprocess.cpp
+++ b/src/zenutil/zenserverprocess.cpp
@@ -12,6 +12,8 @@
 
 #include <atomic>
 
+#include <gsl/gsl-lite.hpp>
+
 #if ZEN_PLATFORM_WINDOWS
 #	include <zencore/windows.h>
 #else
@@ -468,7 +470,14 @@ ZenServerInstance::ZenServerInstance(ZenServerEnvironment& TestEnvironment) : m_
 
 ZenServerInstance::~ZenServerInstance()
 {
-	Shutdown();
+	try
+	{
+		Shutdown();
+	}
+	catch (const std::exception& Err)
+	{
+		ZEN_ERROR("Shutting down zenserver instance failed, reason: '{}'", Err.what());
+	}
 }
 
 void
@@ -480,19 +489,33 @@ ZenServerInstance::SignalShutdown()
 void
 ZenServerInstance::Shutdown()
 {
-	if (m_Process.IsValid() && m_ShutdownOnDestroy)
+	if (m_Process.IsValid())
 	{
-		if (m_Terminate)
+		if (m_ShutdownOnDestroy)
 		{
-			ZEN_INFO("Terminating zenserver process");
-			m_Process.Terminate(111);
-			m_Process.Reset();
+			if (m_Terminate)
+			{
+				ZEN_INFO("Terminating zenserver process {}", m_Name);
+				m_Process.Terminate(111);
+				m_Process.Reset();
+				ZEN_DEBUG("zenserver process {} ({}) terminated", m_Name, m_Process.Pid());
+			}
+			else
+			{
+				ZEN_DEBUG("Requesting zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
+				SignalShutdown();
+				ZEN_DEBUG("Waiting for zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
+				while (!m_Process.Wait(5000))
+				{
+					ZEN_WARN("Waiting for zenserver process {} ({}) timed out", m_Name, m_Process.Pid());
+				}
+				m_Process.Reset();
+			}
+			ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
 		}
 		else
 		{
-			SignalShutdown();
-			m_Process.Wait();
-			m_Process.Reset();
+			ZEN_DEBUG("Detached from zenserver process {} ({})", m_Name, m_Process.Pid());
 		}
 	}
 }
@@ -509,10 +532,9 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 	ChildEventName << "Zen_Child_" << ChildId;
 	NamedEvent ChildEvent{ChildEventName};
 
-	CreateShutdownEvent(BasePort);
-
 	ExtendableStringBuilder<32> LogId;
 	LogId << "Zen" << ChildId;
+	m_Name = LogId.ToString();
 
 	ExtendableStringBuilder<512> CommandLine;
 	CommandLine << "zenserver" ZEN_EXE_SUFFIX_LITERAL;	// see CreateProc() call for actual binary path
@@ -526,7 +548,8 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 			m_OwnerPid = MyPid;
 		}
 
-		CommandLine << " --test --log-id " << LogId;
+		CommandLine << " --test --log-id " << m_Name;
+		CommandLine << " --no-sentry";
 	}
 
 	if (m_OwnerPid.has_value())
@@ -544,7 +567,7 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 	if (BasePort)
 	{
 		CommandLine << " --port " << BasePort;
-		m_BasePort = BasePort;
+		m_BasePort = gsl::narrow_cast<uint16_t>(BasePort);
 	}
 
 	if (!m_TestDir.empty())
@@ -604,41 +627,8 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 	{
 		if (!WaitUntilReady(WaitTimeoutMs))
 		{
-			throw std::runtime_error(fmt::format("server start timeout after {}", NiceTimeSpanMs(WaitTimeoutMs)));
+			throw std::runtime_error(fmt::format("server start of {} timeout after {}", m_Name, NiceTimeSpanMs(WaitTimeoutMs)));
 		}
-
-		// Determine effective base port
-
-		ZenServerState State;
-		if (!State.InitializeReadOnly())
-		{
-			// TODO: return success/error code instead?
-			throw std::runtime_error("no zen state found");
-		}
-
-		const ZenServerState::ZenServerEntry* Entry = nullptr;
-
-		if (BasePort)
-		{
-			Entry = State.Lookup(BasePort);
-		}
-		else
-		{
-			State.Snapshot([&](const ZenServerState::ZenServerEntry& InEntry) {
-				if (InEntry.Pid == static_cast<uint32_t>(GetProcessId(ChildPid)))
-				{
-					Entry = &InEntry;
-				}
-			});
-		}
-
-		if (!Entry)
-		{
-			// TODO: return success/error code instead?
-			throw std::runtime_error("no server entry found");
-		}
-
-		m_BasePort = Entry->EffectiveListenPort;
 	}
 }
 
@@ -694,23 +684,73 @@ ZenServerInstance::Detach()
 	}
 }
 
-void
+uint16_t
 ZenServerInstance::WaitUntilReady()
 {
 	while (m_ReadyEvent.Wait(100) == false)
 	{
 		if (!m_Process.IsRunning() || !m_Process.IsValid())
 		{
-			ZEN_INFO("Wait abandoned by invalid process (running={})", m_Process.IsRunning());
-			return;
+			ZEN_WARN("Wait abandoned by invalid process (running={})", m_Process.IsRunning());
+
+			return 0;
 		}
 	}
+
+	OnServerReady();
+
+	return m_BasePort;
 }
 
 bool
 ZenServerInstance::WaitUntilReady(int Timeout)
 {
-	return m_ReadyEvent.Wait(Timeout);
+	if (m_ReadyEvent.Wait(Timeout))
+	{
+		OnServerReady();
+
+		return true;
+	}
+
+	return false;
+}
+
+void
+ZenServerInstance::OnServerReady()
+{
+	// Determine effective base port
+
+	ZenServerState State;
+	if (!State.InitializeReadOnly())
+	{
+		// TODO: return success/error code instead?
+		throw std::runtime_error("no zen state found");
+	}
+
+	const ZenServerState::ZenServerEntry* Entry = nullptr;
+
+	if (m_BasePort)
+	{
+		Entry = State.Lookup(m_BasePort);
+	}
+	else
+	{
+		State.Snapshot([&](const ZenServerState::ZenServerEntry& InEntry) {
+			if (InEntry.Pid == (uint32_t)m_Process.Pid())
+			{
+				Entry = &InEntry;
+			}
+		});
+	}
+
+	if (!Entry)
+	{
+		// TODO: return success/error code instead?
+		throw std::runtime_error("no server entry found");
+	}
+
+	m_BasePort = Entry->EffectiveListenPort;
+	CreateShutdownEvent(m_BasePort);
 }
 
 std::string
@@ -728,4 +768,14 @@ ZenServerInstance::SetTestDir(std::filesystem::path TestDir)
 	m_TestDir = TestDir;
 }
 
+bool
+ZenServerInstance::IsRunning()
+{
+	if (!m_Process.IsValid())
+	{
+		return false;
+	}
+	return m_Process.IsRunning();
+}
+
 }  // namespace zen
diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp
new file mode 100644
index 000000000..df075ea3f
--- /dev/null
+++ b/src/zenutil/zenutil.cpp
@@ -0,0 +1,19 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zenutil/zenutil.h"
+
+#if ZEN_WITH_TESTS
+
+#	include <zenutil/basicfile.h>
+
+namespace zen {
+
+void
+zenutil_forcelinktests()
+{
+	basicfile_forcelink();
+}
+
+}  // namespace zen
+
+#endif