added `--verbose` option to zenserver-test and `xmake test` (#798)

* when `--verbose` is specified to zenserver-test, all child process output (typically, zenserver instances) is piped through to stdout. you can also pass `--verbose` to `xmake test` to accomplish the same thing.
* this PR also consolidates all test runner `main` function logic (such as from zencore-test, zenhttp-test etc) into central implementation in zencore for consistency and ease of maintenance
* also added extended utf8-tests including a fix to `Utf8ToWide()`

4 files changed, 186 insertions, 11 deletions

diff --git a/src/zencore/include/zencore/testing.h b/src/zencore/include/zencore/testing.h
index a00ee3166..43bdbbffe 100644
--- a/src/zencore/include/zencore/testing.h
+++ b/src/zencore/include/zencore/testing.h
@@ -59,6 +59,8 @@ private:
 			return Runner.Run();                 \
 		}()
 
+int RunTestMain(int argc, char* argv[], const char* traceName, void (*forceLink)());
+
 }  // namespace zen::testing
 #endif
 
diff --git a/src/zencore/include/zencore/testutils.h b/src/zencore/include/zencore/testutils.h
index e2a4f8346..2a789d18f 100644
--- a/src/zencore/include/zencore/testutils.h
+++ b/src/zencore/include/zencore/testutils.h
@@ -59,6 +59,33 @@ struct TrueType
 	static const bool Enabled = true;
 };
 
+namespace utf8test {
+
+	// 2-byte UTF-8 (Latin extended)
+	static constexpr const char	   kLatin[]	 = u8"café_résumé";
+	static constexpr const wchar_t kLatinW[] = L"café_résumé";
+
+	// 2-byte UTF-8 (Cyrillic)
+	static constexpr const char	   kCyrillic[]	= u8"данные";
+	static constexpr const wchar_t kCyrillicW[] = L"данные";
+
+	// 3-byte UTF-8 (CJK)
+	static constexpr const char	   kCJK[]  = u8"日本語";
+	static constexpr const wchar_t kCJKW[] = L"日本語";
+
+	// Mixed scripts
+	static constexpr const char	   kMixed[]	 = u8"zen_éд日";
+	static constexpr const wchar_t kMixedW[] = L"zen_éд日";
+
+	// 4-byte UTF-8 (supplementary plane) — string tests only, NOT filesystem
+	static constexpr const char	   kEmoji[]	 = u8"📦";
+	static constexpr const wchar_t kEmojiW[] = L"📦";
+
+	// BMP-only test strings suitable for filesystem use
+	static constexpr const char* kFilenameSafe[] = {kLatin, kCyrillic, kCJK, kMixed};
+
+}  // namespace utf8test
+
 }  // namespace zen
 
 #endif	// ZEN_WITH_TESTS
diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp
index a9aed6309..ab1c7de58 100644
--- a/src/zencore/string.cpp
+++ b/src/zencore/string.cpp
@@ -4,6 +4,7 @@
 #include <zencore/memoryview.h>
 #include <zencore/string.h>
 #include <zencore/testing.h>
+#include <zencore/testutils.h>
 
 #include <inttypes.h>
 #include <math.h>
@@ -184,7 +185,21 @@ Utf8ToWide(const std::u8string_view& Str8, WideStringBuilderBase& OutString)
 
 			if (!ByteCount)
 			{
+#if ZEN_SIZEOF_WCHAR_T == 2
+				if (CurrentOutChar > 0xFFFF)
+				{
+					// Supplementary plane: emit a UTF-16 surrogate pair
+					uint32_t Adjusted = uint32_t(CurrentOutChar - 0x10000);
+					OutString.Append(wchar_t(0xD800 + (Adjusted >> 10)));
+					OutString.Append(wchar_t(0xDC00 + (Adjusted & 0x3FF)));
+				}
+				else
+				{
+					OutString.Append(wchar_t(CurrentOutChar));
+				}
+#else
 				OutString.Append(wchar_t(CurrentOutChar));
+#endif
 				CurrentOutChar = 0;
 			}
 		}
@@ -967,33 +982,131 @@ TEST_CASE("ExtendableWideStringBuilder")
 
 TEST_CASE("utf8")
 {
+	using namespace utf8test;
+
 	SUBCASE("utf8towide")
 	{
-		// TODO: add more extensive testing here - this covers a very small space
-
 		WideStringBuilder<32> wout;
 		Utf8ToWide(u8"abcdefghi", wout);
 		CHECK(StringEquals(L"abcdefghi", wout.c_str()));
 
 		wout.Reset();
+		Utf8ToWide(u8"abc\xC3\xA4\xC3\xB6\xC3\xBC", wout);
+		CHECK(StringEquals(L"abc\u00E4\u00F6\u00FC", wout.c_str()));
+
+		wout.Reset();
+		Utf8ToWide(std::string_view(kLatin), wout);
+		CHECK(StringEquals(kLatinW, wout.c_str()));
+
+		wout.Reset();
+		Utf8ToWide(std::string_view(kCyrillic), wout);
+		CHECK(StringEquals(kCyrillicW, wout.c_str()));
+
+		wout.Reset();
+		Utf8ToWide(std::string_view(kCJK), wout);
+		CHECK(StringEquals(kCJKW, wout.c_str()));
+
+		wout.Reset();
+		Utf8ToWide(std::string_view(kMixed), wout);
+		CHECK(StringEquals(kMixedW, wout.c_str()));
 
-		Utf8ToWide(u8"abc���", wout);
-		CHECK(StringEquals(L"abc���", wout.c_str()));
+		wout.Reset();
+		Utf8ToWide(std::string_view(kEmoji), wout);
+		CHECK(StringEquals(kEmojiW, wout.c_str()));
 	}
 
 	SUBCASE("widetoutf8")
 	{
-		// TODO: add more extensive testing here - this covers a very small space
-
-		StringBuilder<32> out;
+		StringBuilder<64> out;
 
 		WideToUtf8(L"abcdefghi", out);
 		CHECK(StringEquals("abcdefghi", out.c_str()));
 
 		out.Reset();
+		WideToUtf8(kLatinW, out);
+		CHECK(StringEquals(kLatin, out.c_str()));
 
-		WideToUtf8(L"abc���", out);
-		CHECK(StringEquals(u8"abc���", out.c_str()));
+		out.Reset();
+		WideToUtf8(kCyrillicW, out);
+		CHECK(StringEquals(kCyrillic, out.c_str()));
+
+		out.Reset();
+		WideToUtf8(kCJKW, out);
+		CHECK(StringEquals(kCJK, out.c_str()));
+
+		out.Reset();
+		WideToUtf8(kMixedW, out);
+		CHECK(StringEquals(kMixed, out.c_str()));
+
+		out.Reset();
+		WideToUtf8(kEmojiW, out);
+		CHECK(StringEquals(kEmoji, out.c_str()));
+	}
+
+	SUBCASE("roundtrip")
+	{
+		// UTF-8 -> Wide -> UTF-8 identity
+		const char* Utf8Strings[] = {kLatin, kCyrillic, kCJK, kMixed, kEmoji};
+		for (const char* Utf8Str : Utf8Strings)
+		{
+			ExtendableWideStringBuilder<64> Wide;
+			Utf8ToWide(std::string_view(Utf8Str), Wide);
+
+			ExtendableStringBuilder<64> Back;
+			WideToUtf8(std::wstring_view(Wide.c_str()), Back);
+			CHECK(StringEquals(Utf8Str, Back.c_str()));
+		}
+
+		// Wide -> UTF-8 -> Wide identity
+		const wchar_t* WideStrings[] = {kLatinW, kCyrillicW, kCJKW, kMixedW, kEmojiW};
+		for (const wchar_t* WideStr : WideStrings)
+		{
+			ExtendableStringBuilder<64> Utf8;
+			WideToUtf8(std::wstring_view(WideStr), Utf8);
+
+			ExtendableWideStringBuilder<64> Back;
+			Utf8ToWide(std::string_view(Utf8.c_str()), Back);
+			CHECK(StringEquals(WideStr, Back.c_str()));
+		}
+
+		// Empty string round-trip
+		{
+			ExtendableWideStringBuilder<8> Wide;
+			Utf8ToWide(std::string_view(""), Wide);
+			CHECK(Wide.Size() == 0);
+
+			ExtendableStringBuilder<8> Narrow;
+			WideToUtf8(std::wstring_view(L""), Narrow);
+			CHECK(Narrow.Size() == 0);
+		}
+	}
+
+	SUBCASE("IsValidUtf8")
+	{
+		// Valid inputs
+		CHECK(IsValidUtf8(""));
+		CHECK(IsValidUtf8("hello world"));
+		CHECK(IsValidUtf8(kLatin));
+		CHECK(IsValidUtf8(kCyrillic));
+		CHECK(IsValidUtf8(kCJK));
+		CHECK(IsValidUtf8(kMixed));
+		CHECK(IsValidUtf8(kEmoji));
+
+		// Invalid: truncated 2-byte sequence
+		CHECK(!IsValidUtf8(std::string_view("\xC3", 1)));
+
+		// Invalid: truncated 3-byte sequence
+		CHECK(!IsValidUtf8(std::string_view("\xE6\x97", 2)));
+
+		// Invalid: truncated 4-byte sequence
+		CHECK(!IsValidUtf8(std::string_view("\xF0\x9F\x93", 3)));
+
+		// Invalid: bad start byte
+		CHECK(!IsValidUtf8(std::string_view("\xFF", 1)));
+		CHECK(!IsValidUtf8(std::string_view("\xFE", 1)));
+
+		// Invalid: overlong encoding of '/' (U+002F)
+		CHECK(!IsValidUtf8(std::string_view("\xC0\xAF", 2)));
 	}
 }
 
diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp
index ef8fb0480..6000bd95c 100644
--- a/src/zencore/testing.cpp
+++ b/src/zencore/testing.cpp
@@ -1,18 +1,23 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
+#define ZEN_TEST_WITH_RUNNER 1
+
 #include "zencore/testing.h"
+
+#include "zencore/filesystem.h"
 #include "zencore/logging.h"
+#include "zencore/process.h"
+#include "zencore/trace.h"
 
 #if ZEN_WITH_TESTS
 
 #	include <chrono>
+#	include <clocale>
 #	include <cstdlib>
 #	include <cstdio>
 #	include <string>
 #	include <vector>
 
-#	include <doctest/doctest.h>
-
 namespace zen::testing {
 
 using namespace std::literals;
@@ -149,6 +154,34 @@ TestRunner::Run()
 	return m_Impl->Session.run();
 }
 
+int
+RunTestMain(int argc, char* argv[], [[maybe_unused]] const char* traceName, void (*forceLink)())
+{
+#	if ZEN_PLATFORM_WINDOWS
+	setlocale(LC_ALL, "en_us.UTF8");
+#	endif
+
+	forceLink();
+
+#	if ZEN_PLATFORM_LINUX
+	zen::IgnoreChildSignals();
+#	endif
+
+#	if ZEN_WITH_TRACE
+	zen::TraceInit(traceName);
+	zen::TraceOptions TraceCommandlineOptions;
+	if (GetTraceOptionsFromCommandline(TraceCommandlineOptions))
+	{
+		TraceConfigure(TraceCommandlineOptions);
+	}
+#	endif
+
+	zen::logging::InitializeLogging();
+	zen::MaximizeOpenFileCount();
+
+	return ZEN_RUN_TESTS(argc, argv);
+}
+
 }  // namespace zen::testing
 
 #endif	// ZEN_WITH_TESTS