use current locale when converting wide char command line arguments to string (#712)

* set utf8 locale, only set LC_CTYPE as we don't want to affect how decimal point or sorting orders are handled
* set language/region explicitly so we can use LC_ALL

12 files changed, 50 insertions, 1 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5bf06fb88..b30a77cf6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,6 +21,7 @@
 - Improvement: Excluded folder names are now matched by folder name in subfolders in addition to root level folders
 - Bugfix: Upstream propagation of Put operations would not retain the overwrite cache policy if it was used
 - Bugfix: `zen oplog-download` failed to initialize build part id causing it to fail to download the build part
+- Bugfix: Windows: Set up UTF8 as current locale to properly handle non-ascii characters
 
 ## 5.7.15
 - Feature: `zen oplog-export`, `zen oplog-import` and `zen oplog-download` now has options to boost workers
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index 4d4966222..c03ae476f 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -296,11 +296,15 @@ ZenCmdBase::LogExecutableVersionAndPid()
 int
 main(int argc, char** argv)
 {
+#if ZEN_PLATFORM_WINDOWS
+	setlocale(LC_ALL, "en_us.UTF8");
+#endif	// ZEN_PLATFORM_WINDOWS
+
 	zen::SetCurrentThreadName("main");
 
 	std::vector<std::string> Args;
 #if ZEN_PLATFORM_WINDOWS
-	LPWSTR		RawCommandLine = GetCommandLine();
+	LPWSTR		RawCommandLine = GetCommandLineW();
 	std::string CommandLine	   = zen::WideToUtf8(RawCommandLine);
 	Args					   = zen::ParseCommandLine(CommandLine);
 #else
diff --git a/src/zencore-test/zencore-test.cpp b/src/zencore-test/zencore-test.cpp
index 327550b32..68fc940ee 100644
--- a/src/zencore-test/zencore-test.cpp
+++ b/src/zencore-test/zencore-test.cpp
@@ -19,6 +19,10 @@
 int
 main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[])
 {
+#if ZEN_PLATFORM_WINDOWS
+	setlocale(LC_ALL, "en_us.UTF8");
+#endif	// ZEN_PLATFORM_WINDOWS
+
 #if ZEN_WITH_TESTS
 	zen::zencore_forcelinktests();
 
diff --git a/src/zencore/commandline.cpp b/src/zencore/commandline.cpp
index 78260aeef..426cf23d6 100644
--- a/src/zencore/commandline.cpp
+++ b/src/zencore/commandline.cpp
@@ -22,6 +22,10 @@ void
 IterateCommandlineArgs(std::function<void(const std::string_view& Arg)>& ProcessArg)
 {
 #if ZEN_PLATFORM_WINDOWS
+	// It might seem odd to do this here in addition to at start of main functions but the InitGMalloc() function is called before main (via
+	// static data) so we need to make sure we set the locale before parsing the command line
+	setlocale(LC_ALL, "en_us.UTF8");
+
 	int			  ArgC	  = 0;
 	const LPWSTR  CmdLine = ::GetCommandLineW();
 	const LPWSTR* ArgV	  = ::CommandLineToArgvW(CmdLine, &ArgC);
diff --git a/src/zenhttp-test/zenhttp-test.cpp b/src/zenhttp-test/zenhttp-test.cpp
index d18b2167e..c18759beb 100644
--- a/src/zenhttp-test/zenhttp-test.cpp
+++ b/src/zenhttp-test/zenhttp-test.cpp
@@ -15,6 +15,10 @@
 int
 main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[])
 {
+#if ZEN_PLATFORM_WINDOWS
+	setlocale(LC_ALL, "en_us.UTF8");
+#endif	// ZEN_PLATFORM_WINDOWS
+
 #if ZEN_WITH_TESTS
 	zen::zenhttp_forcelinktests();
 
diff --git a/src/zennet-test/zennet-test.cpp b/src/zennet-test/zennet-test.cpp
index 5e4d29220..bc3b8e8e9 100644
--- a/src/zennet-test/zennet-test.cpp
+++ b/src/zennet-test/zennet-test.cpp
@@ -16,6 +16,10 @@
 int
 main([[maybe_unused]] int argc, [[maybe_unused]] char** argv)
 {
+#if ZEN_PLATFORM_WINDOWS
+	setlocale(LC_ALL, "en_us.UTF8");
+#endif	// ZEN_PLATFORM_WINDOWS
+
 #if ZEN_WITH_TESTS
 	zen::zennet_forcelinktests();
 
diff --git a/src/zenremotestore-test/zenremotestore-test.cpp b/src/zenremotestore-test/zenremotestore-test.cpp
index a49c6d273..5db185041 100644
--- a/src/zenremotestore-test/zenremotestore-test.cpp
+++ b/src/zenremotestore-test/zenremotestore-test.cpp
@@ -17,6 +17,10 @@
 int
 main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[])
 {
+#if ZEN_PLATFORM_WINDOWS
+	setlocale(LC_ALL, "en_us.UTF8");
+#endif	// ZEN_PLATFORM_WINDOWS
+
 #if ZEN_WITH_TESTS
 	zen::zenremotestore_forcelinktests();
 
diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp
index 42296cbe1..418fc7978 100644
--- a/src/zenserver-test/zenserver-test.cpp
+++ b/src/zenserver-test/zenserver-test.cpp
@@ -61,6 +61,10 @@ zen::ZenServerEnvironment TestEnv;
 int
 main(int argc, char** argv)
 {
+#	if ZEN_PLATFORM_WINDOWS
+	setlocale(LC_ALL, "en_us.UTF8");
+#	endif	// ZEN_PLATFORM_WINDOWS
+
 	using namespace std::literals;
 	using namespace zen;
 
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index 34848c831..da256d06d 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -206,6 +206,10 @@ AppMain(int argc, char* argv[])
 int
 test_main(int argc, char** argv)
 {
+#	if ZEN_PLATFORM_WINDOWS
+	setlocale(LC_ALL, "en_us.UTF8");
+#	endif	// ZEN_PLATFORM_WINDOWS
+
 	zen::logging::InitializeLogging();
 	zen::logging::SetLogLevel(zen::logging::level::Debug);
 
@@ -218,6 +222,10 @@ test_main(int argc, char** argv)
 int
 main(int argc, char* argv[])
 {
+#if ZEN_PLATFORM_WINDOWS
+	setlocale(LC_ALL, "en_us.UTF8");
+#endif	// ZEN_PLATFORM_WINDOWS
+
 	using namespace zen;
 
 	if (argc >= 2)
diff --git a/src/zenstore-test/zenstore-test.cpp b/src/zenstore-test/zenstore-test.cpp
index 6df7162fd..c055dbb64 100644
--- a/src/zenstore-test/zenstore-test.cpp
+++ b/src/zenstore-test/zenstore-test.cpp
@@ -16,6 +16,10 @@
 int
 main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[])
 {
+#if ZEN_PLATFORM_WINDOWS
+	setlocale(LC_ALL, "en_us.UTF8");
+#endif	// ZEN_PLATFORM_WINDOWS
+
 #if ZEN_WITH_TESTS
 	zen::zenstore_forcelinktests();
 
diff --git a/src/zentelemetry-test/zentelemetry-test.cpp b/src/zentelemetry-test/zentelemetry-test.cpp
index c8b067226..83fd549db 100644
--- a/src/zentelemetry-test/zentelemetry-test.cpp
+++ b/src/zentelemetry-test/zentelemetry-test.cpp
@@ -16,6 +16,10 @@
 int
 main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[])
 {
+#if ZEN_PLATFORM_WINDOWS
+	setlocale(LC_ALL, "en_us.UTF8");
+#endif	// ZEN_PLATFORM_WINDOWS
+
 #if ZEN_WITH_TESTS
 	zen::zentelemetry_forcelinktests();
 
diff --git a/src/zenutil-test/zenutil-test.cpp b/src/zenutil-test/zenutil-test.cpp
index 3e3a11a01..f5cfd5a72 100644
--- a/src/zenutil-test/zenutil-test.cpp
+++ b/src/zenutil-test/zenutil-test.cpp
@@ -16,6 +16,10 @@
 int
 main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[])
 {
+#if ZEN_PLATFORM_WINDOWS
+	setlocale(LC_ALL, "en_us.UTF8");
+#endif	// ZEN_PLATFORM_WINDOWS
+
 #if ZEN_WITH_TESTS
 	zen::zenutil_forcelinktests();