Merge branch 'main' into ddcref

8 files changed, 156 insertions, 36 deletions

diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp
index cb628af4b..ffc1da10c 100644
--- a/zencore/compactbinary.cpp
+++ b/zencore/compactbinary.cpp
@@ -60,22 +60,27 @@ GetPlatformToDateTimeBiasInSeconds()
 	return uint64_t(double(PlatformEpochYear - DateTimeEpochYear) * 365.2425) * 86400;
 }
 
-DateTime
-DateTime::Now()
+uint64_t
+DateTime::NowTicks()
 {
-	static const uint64_t EpochBias		   = GetPlatformToDateTimeBiasInSeconds();
-	static const uint64_t SecsTo100nsTicks = int64_t(10e9 / 100);
+	static constexpr uint64_t EpochBias = GetPlatformToDateTimeBiasInSeconds();
 
 #if ZEN_PLATFORM_WINDOWS
 	FILETIME SysTime;
-	GetSystemTimeAsFileTime(&SysTime);
-	return DateTime{(EpochBias * SecsTo100nsTicks) + (uint64_t(SysTime.dwHighDateTime) << 32) | SysTime.dwLowDateTime};
+	GetSystemTimePreciseAsFileTime(&SysTime);
+	return (EpochBias * TimeSpan::TicksPerSecond) + ((uint64_t(SysTime.dwHighDateTime) << 32) | SysTime.dwLowDateTime);
 #else
 	int64_t		   SecondsSinceUnixEpoch = time(nullptr);
-	return DateTime{(EpochBias + SecondsSinceUnixEpoch) * SecsTo100nsTicks};
+	return (EpochBias + SecondsSinceUnixEpoch) * TimeSpan::TicksPerSecond;
 #endif
 }
 
+DateTime
+DateTime::Now()
+{
+	return DateTime{NowTicks()};
+}
+
 void
 DateTime::Set(int Year, int Month, int Day, int Hour, int Minute, int Second, int MilliSecond)
 {
@@ -1392,7 +1397,7 @@ LoadCompactBinaryObject(IoBuffer&& Payload)
 }
 
 CbObject
-LoadCompactBinaryObject(IoBuffer& Payload)
+LoadCompactBinaryObject(const IoBuffer& Payload)
 {
 	return CbObject{SharedBuffer(Payload)};
 }
@@ -1404,7 +1409,7 @@ LoadCompactBinaryObject(CompressedBuffer&& Payload)
 }
 
 CbObject
-LoadCompactBinaryObject(CompressedBuffer& Payload)
+LoadCompactBinaryObject(const CompressedBuffer& Payload)
 {
 	return CbObject{SharedBuffer(Payload.DecompressToComposite().Flatten())};
 }
diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
index 041abaf1d..437741161 100644
--- a/zencore/filesystem.cpp
+++ b/zencore/filesystem.cpp
@@ -23,6 +23,7 @@
 #if ZEN_PLATFORM_LINUX
 #	include <dirent.h>
 #	include <fcntl.h>
+#	include <sys/resource.h>
 #	include <sys/stat.h>
 #	include <unistd.h>
 #endif
@@ -31,6 +32,7 @@
 #	include <dirent.h>
 #	include <fcntl.h>
 #	include <libproc.h>
+#	include <sys/resource.h>
 #	include <sys/stat.h>
 #	include <sys/syslimits.h>
 #	include <unistd.h>
@@ -921,6 +923,10 @@ PathFromHandle(void* NativeHandle)
 	}
 
 	const DWORD RequiredLengthIncludingNul = GetFinalPathNameByHandleW(NativeHandle, nullptr, 0, FILE_NAME_OPENED);
+	if (RequiredLengthIncludingNul == 0)
+	{
+		ThrowLastError(fmt::format("failed to get path from file handle {}", NativeHandle));
+	}
 
 	std::wstring FullPath;
 	FullPath.resize(RequiredLengthIncludingNul - 1);
@@ -982,6 +988,40 @@ GetRunningExecutablePath()
 #endif	// ZEN_PLATFORM_WINDOWS
 }
 
+void
+MaximizeOpenFileCount()
+{
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	struct rlimit Limit;
+	int			  Error = getrlimit(RLIMIT_NOFILE, &Limit);
+	if (Error)
+	{
+		ZEN_WARN("failed getting rlimit RLIMIT_NOFILE, reason '{}'", zen::MakeErrorCode(Error).message());
+	}
+	else
+	{
+		struct rlimit NewLimit = Limit;
+		NewLimit.rlim_cur	   = NewLimit.rlim_max;
+		ZEN_INFO("changing RLIMIT_NOFILE from rlim_cur = {}, rlim_max {} to rlim_cur = {}, rlim_max {}",
+				 Limit.rlim_cur,
+				 Limit.rlim_max,
+				 NewLimit.rlim_cur,
+				 NewLimit.rlim_max);
+
+		Error = setrlimit(RLIMIT_NOFILE, &NewLimit);
+		if (Error != 0)
+		{
+			ZEN_WARN("failed to set RLIMIT_NOFILE limits from rlim_cur = {}, rlim_max {} to rlim_cur = {}, rlim_max {}, reason '{}'",
+					 Limit.rlim_cur,
+					 Limit.rlim_max,
+					 NewLimit.rlim_cur,
+					 NewLimit.rlim_max,
+					 zen::MakeErrorCode(Error).message());
+		}
+	}
+#endif
+}
+
 //////////////////////////////////////////////////////////////////////////
 //
 // Testing related code follows...
diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h
index 8985144dd..19f1597dc 100644
--- a/zencore/include/zencore/compactbinary.h
+++ b/zencore/include/zencore/compactbinary.h
@@ -43,6 +43,7 @@ public:
 
 	inline uint64_t GetTicks() const { return Ticks; }
 
+	static uint64_t NowTicks();
 	static DateTime Now();
 
 	int	 GetYear() const;
@@ -1400,9 +1401,9 @@ CbField::AsBinary(const SharedBuffer& Default) &&
 ZENCORE_API CbField LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator);
 
 ZENCORE_API CbObject LoadCompactBinaryObject(IoBuffer&& Payload);
-ZENCORE_API CbObject LoadCompactBinaryObject(IoBuffer& Payload);
-ZENCORE_API CbObject LoadCompactBinaryObject(CompressedBuffer& Payload);
+ZENCORE_API CbObject LoadCompactBinaryObject(const IoBuffer& Payload);
 ZENCORE_API CbObject LoadCompactBinaryObject(CompressedBuffer&& Payload);
+ZENCORE_API CbObject LoadCompactBinaryObject(const CompressedBuffer& Payload);
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h
index d1a5f3e0c..a6e76eaa0 100644
--- a/zencore/include/zencore/filesystem.h
+++ b/zencore/include/zencore/filesystem.h
@@ -34,6 +34,10 @@ ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle);
 
 ZENCORE_API std::filesystem::path GetRunningExecutablePath();
 
+/** Set the max open file handle count to max allowed for the current process on Linux and MacOS
+ */
+ZENCORE_API void MaximizeOpenFileCount();
+
 struct FileContents
 {
 	std::vector<IoBuffer> Data;
diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h
index 4c378730f..027730063 100644
--- a/zencore/include/zencore/string.h
+++ b/zencore/include/zencore/string.h
@@ -9,6 +9,7 @@
 #include <string.h>
 #include <charconv>
 #include <codecvt>
+#include <concepts>
 #include <optional>
 #include <span>
 #include <string_view>
@@ -488,6 +489,26 @@ std::string WideToUtf8(const wchar_t* wstr);
 void		WideToUtf8(const std::wstring_view& wstr, StringBuilderBase& out);
 std::string WideToUtf8(const std::wstring_view Wstr);
 
+inline uint8_t
+Char2Nibble(char c)
+{
+	if (c >= '0' && c <= '9')
+	{
+		return uint8_t(c - '0');
+	}
+	if (c >= 'a' && c <= 'f')
+	{
+		return uint8_t(c - 'a' + 10);
+	}
+	if (c >= 'A' && c <= 'F')
+	{
+		return uint8_t(c - 'A' + 10);
+	}
+	return uint8_t(0xff);
+};
+
+static constexpr const char HexChars[] = "0123456789abcdef";
+
 /// <summary>
 /// Parse hex string into a byte buffer
 /// </summary>
@@ -501,38 +522,56 @@ ParseHexBytes(const char* InputString, size_t CharacterCount, uint8_t* OutPtr)
 {
 	ZEN_ASSERT((CharacterCount & 1) == 0);
 
-	auto char2nibble = [](char c) {
-		uint8_t c8 = uint8_t(c - '0');
+	uint8_t allBits = 0;
 
-		if (c8 < 10)
-			return c8;
+	while (CharacterCount)
+	{
+		uint8_t n0 = Char2Nibble(InputString[0]);
+		uint8_t n1 = Char2Nibble(InputString[1]);
 
-		c8 -= 'A' - '0' - 10;
+		allBits |= n0 | n1;
 
-		if (c8 < 16)
-			return c8;
+		*OutPtr = (n0 << 4) | n1;
 
-		c8 -= 'a' - 'A';
+		OutPtr += 1;
+		InputString += 2;
+		CharacterCount -= 2;
+	}
 
-		if (c8 < 16)
-			return c8;
+	return (allBits & 0x80) == 0;
+}
 
-		return uint8_t(0xff);
-	};
+inline void
+ToHexBytes(const uint8_t* InputData, size_t ByteCount, char* OutString)
+{
+	while (ByteCount--)
+	{
+		uint8_t byte = *InputData++;
+
+		*OutString++ = HexChars[byte >> 4];
+		*OutString++ = HexChars[byte & 15];
+	}
+}
+
+inline bool
+ParseHexNumber(const char* InputString, size_t CharacterCount, uint8_t* OutPtr)
+{
+	ZEN_ASSERT((CharacterCount & 1) == 0);
 
 	uint8_t allBits = 0;
 
+	InputString += CharacterCount;
 	while (CharacterCount)
 	{
-		uint8_t n0 = char2nibble(InputString[0]);
-		uint8_t n1 = char2nibble(InputString[1]);
+		InputString -= 2;
+		uint8_t n0 = Char2Nibble(InputString[0]);
+		uint8_t n1 = Char2Nibble(InputString[1]);
 
 		allBits |= n0 | n1;
 
 		*OutPtr = (n0 << 4) | n1;
 
 		OutPtr += 1;
-		InputString += 2;
 		CharacterCount -= 2;
 	}
 
@@ -540,19 +579,43 @@ ParseHexBytes(const char* InputString, size_t CharacterCount, uint8_t* OutPtr)
 }
 
 inline void
-ToHexBytes(const uint8_t* InputData, size_t ByteCount, char* OutString)
+ToHexNumber(const uint8_t* InputData, size_t ByteCount, char* OutString)
 {
-	const char hexchars[] = "0123456789abcdef";
-
+	InputData += ByteCount;
 	while (ByteCount--)
 	{
-		uint8_t byte = *InputData++;
+		uint8_t byte = *(--InputData);
 
-		*OutString++ = hexchars[byte >> 4];
-		*OutString++ = hexchars[byte & 15];
+		*OutString++ = HexChars[byte >> 4];
+		*OutString++ = HexChars[byte & 15];
 	}
 }
 
+/// <summary>
+/// Generates a hex number from a pointer to an integer type, this formats the number in the correct order for a hexadecimal number
+/// </summary>
+/// <param name="Value">Integer value type</param>
+/// <param name="outString">Output buffer where resulting string is written</param>
+void
+ToHexNumber(UnsignedIntegral auto Value, char* OutString)
+{
+	ToHexNumber((const uint8_t*)&Value, sizeof(Value), OutString);
+	OutString[sizeof(Value) * 2] = 0;
+}
+
+/// <summary>
+/// Parse hex number string into a value, this formats the number in the correct order for a hexadecimal number
+/// </summary>
+/// <param name="string">Input string</param>
+/// <param name="characterCount">Number of characters in string</param>
+/// <param name="OutValue">Pointer to output value</param>
+/// <returns>true if the input consisted of all valid hexadecimal characters</returns>
+bool
+ParseHexNumber(const std::string HexString, UnsignedIntegral auto& OutValue)
+{
+	return ParseHexNumber(HexString.c_str(), sizeof(OutValue) * 2, (uint8_t*)&OutValue);
+}
+
 //////////////////////////////////////////////////////////////////////////
 // Format numbers for humans
 //
diff --git a/zencore/include/zencore/workthreadpool.h b/zencore/include/zencore/workthreadpool.h
index 834339d50..0ddc65298 100644
--- a/zencore/include/zencore/workthreadpool.h
+++ b/zencore/include/zencore/workthreadpool.h
@@ -36,9 +36,11 @@ public:
 	void ScheduleWork(Ref<IWork> Work);
 	void ScheduleWork(std::function<void()>&& Work);
 
-	void WorkerThreadFunction();
+	[[nodiscard]] size_t PendingWork() const;
 
 private:
+	void WorkerThreadFunction();
+
 	std::vector<std::thread>  m_WorkerThreads;
 	BlockingQueue<Ref<IWork>> m_WorkQueue;
 };
diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp
index e2aaa3169..8a3ab8427 100644
--- a/zencore/iobuffer.cpp
+++ b/zencore/iobuffer.cpp
@@ -186,7 +186,7 @@ IoBufferExtendedCore::IoBufferExtendedCore(const IoBufferExtendedCore* Outer, ui
 , m_FileHandle(Outer->m_FileHandle)
 , m_FileOffset(Outer->m_FileOffset + Offset)
 {
-	m_Flags.fetch_or(kIsOwnedByThis | kIsExtended, std::memory_order_relaxed);
+	m_Flags.fetch_or(kIsExtended, std::memory_order_relaxed);
 }
 
 IoBufferExtendedCore::~IoBufferExtendedCore()
@@ -217,10 +217,9 @@ IoBufferExtendedCore::~IoBufferExtendedCore()
 		int	 Fd		 = int(uintptr_t(m_FileHandle));
 		bool Success = (close(Fd) == 0);
 #endif
-
 		if (!Success)
 		{
-			ZEN_WARN("Error reported on file handle close!");
+			ZEN_WARN("Error reported on file handle close, reason '{}'", GetLastErrorAsString());
 		}
 	}
 
diff --git a/zencore/workthreadpool.cpp b/zencore/workthreadpool.cpp
index 3fd1d11a6..4291d1581 100644
--- a/zencore/workthreadpool.cpp
+++ b/zencore/workthreadpool.cpp
@@ -48,6 +48,12 @@ WorkerThreadPool::ScheduleWork(std::function<void()>&& Work)
 	m_WorkQueue.Enqueue(new detail::LambdaWork(Work));
 }
 
+[[nodiscard]] size_t
+WorkerThreadPool::PendingWork() const
+{
+	return m_WorkQueue.Size();
+}
+
 void
 WorkerThreadPool::WorkerThreadFunction()
 {