From 74a5e2fb8dec43682e81a98c9677aef849ca7cc1 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 16 Feb 2026 15:37:13 +0100 Subject: added ResetConsoleLog (#758) also made sure log initialization calls it to ensure the console output format is retained even if the console logger was set up before logging is initialized --- src/zencore/include/zencore/logging.h | 1 + src/zencore/logging.cpp | 8 ++++++++ 2 files changed, 9 insertions(+) (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/logging.h b/src/zencore/include/zencore/logging.h index afbbbd3ee..74a44d028 100644 --- a/src/zencore/include/zencore/logging.h +++ b/src/zencore/include/zencore/logging.h @@ -31,6 +31,7 @@ void FlushLogging(); LoggerRef Default(); void SetDefault(std::string_view NewDefaultLoggerId); LoggerRef ConsoleLog(); +void ResetConsoleLog(); void SuppressConsoleLog(); LoggerRef ErrorLog(); void SetErrorLog(std::string_view LoggerId); diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp index a6697c443..77e05a909 100644 --- a/src/zencore/logging.cpp +++ b/src/zencore/logging.cpp @@ -404,6 +404,14 @@ ConsoleLog() return *ConLogger; } +void +ResetConsoleLog() +{ + LoggerRef ConLog = ConsoleLog(); + + ConLog.SpdLogger->set_pattern("%v"); +} + void InitializeLogging() { -- cgit v1.2.3 From 2159b2ce105935ce4d52a726094f9bbb91537d0c Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 17 Feb 2026 13:56:33 +0100 Subject: misc fixes brought over from sb/proto (#759) * `RwLock::WithSharedLock` and `RwLock::WithExclusiveLock` can now return a value (which is returned by the passed function) * Comma-separated logger specification now correctly deals with commas * `GetSystemMetrics` properly accounts for cores * cpr response formatter passes arguments in the right order * `HttpServerRequest::SetLogRequest` can be used to selectively log HTTP requests --- src/zencore/include/zencore/thread.h | 8 ++++---- src/zencore/logging.cpp | 2 +- src/zencore/system.cpp | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h index de8f9399c..a1c68b0b2 100644 --- a/src/zencore/include/zencore/thread.h +++ b/src/zencore/include/zencore/thread.h @@ -61,10 +61,10 @@ public: RwLock* m_Lock; }; - inline void WithSharedLock(auto&& Fun) + inline auto WithSharedLock(auto&& Fun) { SharedLockScope $(*this); - Fun(); + return Fun(); } struct ExclusiveLockScope @@ -85,10 +85,10 @@ public: RwLock* m_Lock; }; - inline void WithExclusiveLock(auto&& Fun) + inline auto WithExclusiveLock(auto&& Fun) { ExclusiveLockScope $(*this); - Fun(); + return Fun(); } private: diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp index 77e05a909..e79c4b41c 100644 --- a/src/zencore/logging.cpp +++ b/src/zencore/logging.cpp @@ -251,7 +251,7 @@ RefreshLogLevels(level::LogLevel* DefaultLevel) if (auto CommaPos = Spec.find_first_of(','); CommaPos != std::string_view::npos) { - LoggerName = Spec.substr(CommaPos + 1); + LoggerName = Spec.substr(0, CommaPos); Spec.remove_prefix(CommaPos + 1); } else diff --git a/src/zencore/system.cpp b/src/zencore/system.cpp index b9ac3bdee..e92691781 100644 --- a/src/zencore/system.cpp +++ b/src/zencore/system.cpp @@ -66,15 +66,15 @@ GetSystemMetrics() // Determine physical core count DWORD BufferSize = 0; - BOOL Result = GetLogicalProcessorInformation(nullptr, &BufferSize); + BOOL Result = GetLogicalProcessorInformationEx(RelationAll, nullptr, &BufferSize); if (int32_t Error = GetLastError(); Error != ERROR_INSUFFICIENT_BUFFER) { ThrowSystemError(Error, "Failed to get buffer size for logical processor information"); } - PSYSTEM_LOGICAL_PROCESSOR_INFORMATION Buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)Memory::Alloc(BufferSize); + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX Buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)Memory::Alloc(BufferSize); - Result = GetLogicalProcessorInformation(Buffer, &BufferSize); + Result = GetLogicalProcessorInformationEx(RelationAll, Buffer, &BufferSize); if (!Result) { Memory::Free(Buffer); @@ -84,9 +84,9 @@ GetSystemMetrics() DWORD ProcessorPkgCount = 0; DWORD ProcessorCoreCount = 0; DWORD ByteOffset = 0; - while (ByteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= BufferSize) + while (ByteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) <= BufferSize) { - const SYSTEM_LOGICAL_PROCESSOR_INFORMATION& Slpi = Buffer[ByteOffset / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION)]; + const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX& Slpi = Buffer[ByteOffset / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)]; if (Slpi.Relationship == RelationProcessorCore) { ProcessorCoreCount++; @@ -95,7 +95,7 @@ GetSystemMetrics() { ProcessorPkgCount++; } - ByteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); + ByteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX); } Metrics.CoreCount = ProcessorCoreCount; -- cgit v1.2.3 From 5e1e23e209eec75a396c18f8eee3d93a9e196bfc Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 17 Feb 2026 14:00:53 +0100 Subject: add http server root password protection (#757) - Feature: Added `--security-config-path` option to zenserver to configure security settings - Expects a path to a .json file - Default is an empty path resulting in no extra security settings and legacy behavior - Current support is a top level filter of incoming http requests restricted to the `password` type - `password` type will check the `Authorization` header and match it to the selected authorization strategy - Currently the security settings is very basic and configured to a fixed username+password at startup { "http" { "root": { "filter": { "type": "password", "config": { "password": { "username": "", "password": "" }, "protect-machine-local-requests": false, "unprotected-uris": [ "/health/", "/health/info", "/health/version" ] } } } } } --- src/zencore/include/zencore/string.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h index cbff6454f..5a12ba5d2 100644 --- a/src/zencore/include/zencore/string.h +++ b/src/zencore/include/zencore/string.h @@ -796,6 +796,22 @@ HashStringDjb2(const std::string_view& InString) return HashValue; } +constexpr uint32_t +HashStringDjb2(const std::span InStrings) +{ + uint32_t HashValue = 5381; + + for (const std::string_view& String : InStrings) + { + for (int CurChar : String) + { + HashValue = HashValue * 33 + CurChar; + } + } + + return HashValue; +} + constexpr uint32_t HashStringAsLowerDjb2(const std::string_view& InString) { -- cgit v1.2.3 From ae9c30841074da9226a76c1eb2fb3a3e29086bf6 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 18 Feb 2026 09:40:35 +0100 Subject: add selective request logging support to http.sys (#762) * implemented selective request logging for http.sys for consistency with asio * fixed traversal of GetLogicalProcessorInformationEx to account for variable-sized records * also adds CPU usage metrics --- src/zencore/include/zencore/system.h | 1 + src/zencore/system.cpp | 169 +++++++++++++++++++++++++++-------- 2 files changed, 134 insertions(+), 36 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/system.h b/src/zencore/include/zencore/system.h index aec2e0ce4..bf3c15d3d 100644 --- a/src/zencore/include/zencore/system.h +++ b/src/zencore/include/zencore/system.h @@ -25,6 +25,7 @@ struct SystemMetrics uint64_t AvailVirtualMemoryMiB = 0; uint64_t PageFileMiB = 0; uint64_t AvailPageFileMiB = 0; + float CpuUsagePercent = 0.0f; }; SystemMetrics GetSystemMetrics(); diff --git a/src/zencore/system.cpp b/src/zencore/system.cpp index e92691781..267c87e12 100644 --- a/src/zencore/system.cpp +++ b/src/zencore/system.cpp @@ -13,6 +13,8 @@ ZEN_THIRD_PARTY_INCLUDES_START # include # include +# include +# pragma comment(lib, "pdh.lib") ZEN_THIRD_PARTY_INCLUDES_END #elif ZEN_PLATFORM_LINUX # include @@ -65,55 +67,98 @@ GetSystemMetrics() // Determine physical core count - DWORD BufferSize = 0; - BOOL Result = GetLogicalProcessorInformationEx(RelationAll, nullptr, &BufferSize); - if (int32_t Error = GetLastError(); Error != ERROR_INSUFFICIENT_BUFFER) { - ThrowSystemError(Error, "Failed to get buffer size for logical processor information"); - } - - PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX Buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)Memory::Alloc(BufferSize); + DWORD BufferSize = 0; + BOOL Result = GetLogicalProcessorInformationEx(RelationAll, nullptr, &BufferSize); + if (int32_t Error = GetLastError(); Error != ERROR_INSUFFICIENT_BUFFER) + { + ThrowSystemError(Error, "Failed to get buffer size for logical processor information"); + } - Result = GetLogicalProcessorInformationEx(RelationAll, Buffer, &BufferSize); - if (!Result) - { - Memory::Free(Buffer); - throw std::runtime_error("Failed to get logical processor information"); - } + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX Buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)Memory::Alloc(BufferSize); - DWORD ProcessorPkgCount = 0; - DWORD ProcessorCoreCount = 0; - DWORD ByteOffset = 0; - while (ByteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) <= BufferSize) - { - const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX& Slpi = Buffer[ByteOffset / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)]; - if (Slpi.Relationship == RelationProcessorCore) + Result = GetLogicalProcessorInformationEx(RelationAll, Buffer, &BufferSize); + if (!Result) { - ProcessorCoreCount++; + Memory::Free(Buffer); + throw std::runtime_error("Failed to get logical processor information"); } - else if (Slpi.Relationship == RelationProcessorPackage) + + DWORD ProcessorPkgCount = 0; + DWORD ProcessorCoreCount = 0; + DWORD LogicalProcessorCount = 0; + + BYTE* Ptr = reinterpret_cast(Buffer); + BYTE* const End = Ptr + BufferSize; + while (Ptr < End) { - ProcessorPkgCount++; + const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX& Slpi = *reinterpret_cast(Ptr); + if (Slpi.Relationship == RelationProcessorCore) + { + ++ProcessorCoreCount; + + // Count logical processors (threads) across all processor groups for this core. + // Each core entry lists one GROUP_AFFINITY per group it spans; each set bit + // in the Mask represents one logical processor (HyperThreading sibling). + for (WORD g = 0; g < Slpi.Processor.GroupCount; ++g) + { + LogicalProcessorCount += static_cast(__popcnt64(Slpi.Processor.GroupMask[g].Mask)); + } + } + else if (Slpi.Relationship == RelationProcessorPackage) + { + ++ProcessorPkgCount; + } + Ptr += Slpi.Size; } - ByteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX); - } - Metrics.CoreCount = ProcessorCoreCount; - Metrics.CpuCount = ProcessorPkgCount; + Metrics.CoreCount = ProcessorCoreCount; + Metrics.CpuCount = ProcessorPkgCount; + Metrics.LogicalProcessorCount = LogicalProcessorCount; - Memory::Free(Buffer); + Memory::Free(Buffer); + } // Query memory status - MEMORYSTATUSEX MemStatus{.dwLength = sizeof(MEMORYSTATUSEX)}; - GlobalMemoryStatusEx(&MemStatus); + { + MEMORYSTATUSEX MemStatus{.dwLength = sizeof(MEMORYSTATUSEX)}; + GlobalMemoryStatusEx(&MemStatus); + + Metrics.SystemMemoryMiB = MemStatus.ullTotalPhys / 1024 / 1024; + Metrics.AvailSystemMemoryMiB = MemStatus.ullAvailPhys / 1024 / 1024; + Metrics.VirtualMemoryMiB = MemStatus.ullTotalVirtual / 1024 / 1024; + Metrics.AvailVirtualMemoryMiB = MemStatus.ullAvailVirtual / 1024 / 1024; + Metrics.PageFileMiB = MemStatus.ullTotalPageFile / 1024 / 1024; + Metrics.AvailPageFileMiB = MemStatus.ullAvailPageFile / 1024 / 1024; + } + + // Query CPU usage using PDH + // + // TODO: This should be changed to not require a Sleep, perhaps by using some + // background metrics gathering mechanism. + + { + PDH_HQUERY QueryHandle = nullptr; + PDH_HCOUNTER CounterHandle = nullptr; - Metrics.SystemMemoryMiB = MemStatus.ullTotalPhys / 1024 / 1024; - Metrics.AvailSystemMemoryMiB = MemStatus.ullAvailPhys / 1024 / 1024; - Metrics.VirtualMemoryMiB = MemStatus.ullTotalVirtual / 1024 / 1024; - Metrics.AvailVirtualMemoryMiB = MemStatus.ullAvailVirtual / 1024 / 1024; - Metrics.PageFileMiB = MemStatus.ullTotalPageFile / 1024 / 1024; - Metrics.AvailPageFileMiB = MemStatus.ullAvailPageFile / 1024 / 1024; + if (PdhOpenQueryW(nullptr, 0, &QueryHandle) == ERROR_SUCCESS) + { + if (PdhAddEnglishCounterW(QueryHandle, L"\\Processor(_Total)\\% Processor Time", 0, &CounterHandle) == ERROR_SUCCESS) + { + PdhCollectQueryData(QueryHandle); + Sleep(100); + PdhCollectQueryData(QueryHandle); + + PDH_FMT_COUNTERVALUE CounterValue; + if (PdhGetFormattedCounterValue(CounterHandle, PDH_FMT_DOUBLE, nullptr, &CounterValue) == ERROR_SUCCESS) + { + Metrics.CpuUsagePercent = static_cast(CounterValue.doubleValue); + } + } + PdhCloseQuery(QueryHandle); + } + } return Metrics; } @@ -190,6 +235,39 @@ GetSystemMetrics() } } + // Query CPU usage + Metrics.CpuUsagePercent = 0.0f; + if (FILE* Stat = fopen("/proc/stat", "r")) + { + char Line[256]; + unsigned long User, Nice, System, Idle, IoWait, Irq, SoftIrq; + static unsigned long PrevUser = 0, PrevNice = 0, PrevSystem = 0, PrevIdle = 0, PrevIoWait = 0, PrevIrq = 0, PrevSoftIrq = 0; + + if (fgets(Line, sizeof(Line), Stat)) + { + if (sscanf(Line, "cpu %lu %lu %lu %lu %lu %lu %lu", &User, &Nice, &System, &Idle, &IoWait, &Irq, &SoftIrq) == 7) + { + unsigned long TotalDelta = (User + Nice + System + Idle + IoWait + Irq + SoftIrq) - + (PrevUser + PrevNice + PrevSystem + PrevIdle + PrevIoWait + PrevIrq + PrevSoftIrq); + unsigned long IdleDelta = Idle - PrevIdle; + + if (TotalDelta > 0) + { + Metrics.CpuUsagePercent = 100.0f * (TotalDelta - IdleDelta) / TotalDelta; + } + + PrevUser = User; + PrevNice = Nice; + PrevSystem = System; + PrevIdle = Idle; + PrevIoWait = IoWait; + PrevIrq = Irq; + PrevSoftIrq = SoftIrq; + } + } + fclose(Stat); + } + // Get memory information long Pages = sysconf(_SC_PHYS_PAGES); long PageSize = sysconf(_SC_PAGE_SIZE); @@ -270,6 +348,25 @@ GetSystemMetrics() sysctlbyname("hw.packages", &Packages, &Size, nullptr, 0); Metrics.CpuCount = Packages > 0 ? Packages : 1; + // Query CPU usage using host_statistics64 + Metrics.CpuUsagePercent = 0.0f; + host_cpu_load_info_data_t CpuLoad; + mach_msg_type_number_t CpuCount = sizeof(CpuLoad) / sizeof(natural_t); + if (host_statistics(mach_host_self(), HOST_CPU_LOAD_INFO, (host_info_t)&CpuLoad, &CpuCount) == KERN_SUCCESS) + { + unsigned long TotalTicks = 0; + for (int i = 0; i < CPU_STATE_MAX; ++i) + { + TotalTicks += CpuLoad.cpu_ticks[i]; + } + + if (TotalTicks > 0) + { + unsigned long IdleTicks = CpuLoad.cpu_ticks[CPU_STATE_IDLE]; + Metrics.CpuUsagePercent = 100.0f * (TotalTicks - IdleTicks) / TotalTicks; + } + } + // Get memory information uint64_t MemSize = 0; Size = sizeof(MemSize); -- cgit v1.2.3 From 5789739f042791d38f0d7f4219e5b0b823d8b3d6 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 20 Feb 2026 09:07:00 +0100 Subject: fix MakeSafeAbsolutePathInPlace mis-spelling (#765) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (was MakeSafeAbsolutePathÍnPlace - note accent) Also fixed misleading comments on multiple functions in filesystem.h --- src/zencore/filesystem.cpp | 4 ++-- src/zencore/include/zencore/filesystem.h | 38 ++++++++++++++++---------------- 2 files changed, 21 insertions(+), 21 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 92a065707..1a4ee4b9b 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -3069,7 +3069,7 @@ SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) } void -MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path) +MakeSafeAbsolutePathInPlace(std::filesystem::path& Path) { if (!Path.empty()) { @@ -3091,7 +3091,7 @@ std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path) { std::filesystem::path Tmp(Path); - MakeSafeAbsolutePathÍnPlace(Tmp); + MakeSafeAbsolutePathInPlace(Tmp); return Tmp; } diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index f28863679..16e2b59f8 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -64,80 +64,80 @@ std::filesystem::path PathFromHandle(void* NativeHandle, std::error_code& Ec); */ std::filesystem::path CanonicalPath(std::filesystem::path InPath, std::error_code& Ec); -/** Query file size +/** Check if a path exists and is a regular file (throws) */ bool IsFile(const std::filesystem::path& Path); -/** Query file size +/** Check if a path exists and is a regular file (does not throw) */ bool IsFile(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size +/** Check if a path exists and is a directory (throws) */ bool IsDir(const std::filesystem::path& Path); -/** Query file size +/** Check if a path exists and is a directory (does not throw) */ bool IsDir(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size +/** Delete file at path, if it exists (throws) */ bool RemoveFile(const std::filesystem::path& Path); -/** Query file size +/** Delete file at path, if it exists (does not throw) */ bool RemoveFile(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size +/** Delete directory at path, if it exists (throws) */ bool RemoveDir(const std::filesystem::path& Path); -/** Query file size +/** Delete directory at path, if it exists (does not throw) */ bool RemoveDir(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size +/** Query file size (throws) */ uint64_t FileSizeFromPath(const std::filesystem::path& Path); -/** Query file size +/** Query file size (does not throw) */ uint64_t FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size from native file handle +/** Query file size from native file handle (throws) */ uint64_t FileSizeFromHandle(void* NativeHandle); -/** Query file size from native file handle +/** Query file size from native file handle (does not throw) */ uint64_t FileSizeFromHandle(void* NativeHandle, std::error_code& Ec); /** Get a native time tick of last modification time */ -uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec); +uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); /** Get a native time tick of last modification time */ -uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); +uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec); bool TryGetFileProperties(const std::filesystem::path& Path, uint64_t& OutSize, uint64_t& OutModificationTick, uint32_t& OutNativeModeOrAttributes); -/** Move a file, if the files are not on the same drive the function will fail +/** Move/rename a file, if the files are not on the same drive the function will fail (throws) */ void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); -/** Move a file, if the files are not on the same drive the function will fail +/** Move/rename a file, if the files are not on the same drive the function will fail */ void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); -/** Move a directory, if the files are not on the same drive the function will fail +/** Move/rename a directory, if the files are not on the same drive the function will fail (throws) */ void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); -/** Move a directory, if the files are not on the same drive the function will fail +/** Move/rename a directory, if the files are not on the same drive the function will fail */ void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); @@ -421,7 +421,7 @@ uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); -void MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path); +void MakeSafeAbsolutePathInPlace(std::filesystem::path& Path); [[nodiscard]] std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path); class SharedMemory -- cgit v1.2.3 From 73f3eb4feedf3bca0ddc832a89a05f09813c6858 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 23 Feb 2026 11:08:24 +0100 Subject: implemented base64 decoding (#777) Co-authored-by: Stefan Boberg --- src/zencore/base64.cpp | 192 ++++++++++++++++++++++++++++++++++- src/zencore/include/zencore/base64.h | 4 + 2 files changed, 194 insertions(+), 2 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/base64.cpp b/src/zencore/base64.cpp index 1f56ee6c3..fdf5f2d66 100644 --- a/src/zencore/base64.cpp +++ b/src/zencore/base64.cpp @@ -1,6 +1,10 @@ // Copyright Epic Games, Inc. All Rights Reserved. #include +#include +#include + +#include namespace zen { @@ -11,7 +15,6 @@ static const uint8_t EncodingAlphabet[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; /** The table used to convert an ascii character into a 6 bit value */ -#if 0 static const uint8_t DecodingAlphabet[256] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x00-0x0f 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x10-0x1f @@ -30,7 +33,6 @@ static const uint8_t DecodingAlphabet[256] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xe0-0xef 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // 0xf0-0xff }; -#endif // 0 template uint32_t @@ -104,4 +106,190 @@ Base64::Encode(const uint8_t* Source, uint32_t Length, CharType* Dest) template uint32_t Base64::Encode(const uint8_t* Source, uint32_t Length, char* Dest); template uint32_t Base64::Encode(const uint8_t* Source, uint32_t Length, wchar_t* Dest); +template +bool +Base64::Decode(const CharType* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength) +{ + // Length must be a multiple of 4 + if (Length % 4 != 0) + { + OutLength = 0; + return false; + } + + uint8_t* DecodedBytes = Dest; + + // Process 4 encoded characters at a time, producing 3 decoded bytes + while (Length > 0) + { + // Count padding characters at the end + uint32_t PadCount = 0; + if (Source[3] == '=') + { + PadCount++; + if (Source[2] == '=') + { + PadCount++; + } + } + + // Look up each character in the decoding table + uint8_t A = DecodingAlphabet[static_cast(Source[0])]; + uint8_t B = DecodingAlphabet[static_cast(Source[1])]; + uint8_t C = (PadCount >= 2) ? 0 : DecodingAlphabet[static_cast(Source[2])]; + uint8_t D = (PadCount >= 1) ? 0 : DecodingAlphabet[static_cast(Source[3])]; + + // Check for invalid characters (0xFF means not in the base64 alphabet) + if (A == 0xFF || B == 0xFF || C == 0xFF || D == 0xFF) + { + OutLength = 0; + return false; + } + + // Reconstruct the 24-bit value from 4 6-bit chunks + uint32_t ByteTriplet = (A << 18) | (B << 12) | (C << 6) | D; + + // Extract the 3 bytes + *DecodedBytes++ = static_cast(ByteTriplet >> 16); + if (PadCount < 2) + { + *DecodedBytes++ = static_cast((ByteTriplet >> 8) & 0xFF); + } + if (PadCount < 1) + { + *DecodedBytes++ = static_cast(ByteTriplet & 0xFF); + } + + Source += 4; + Length -= 4; + } + + OutLength = uint32_t(DecodedBytes - Dest); + return true; +} + +template bool Base64::Decode(const char* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength); +template bool Base64::Decode(const wchar_t* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength); + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +#if ZEN_WITH_TESTS + +using namespace std::string_literals; + +TEST_CASE("Base64") +{ + auto EncodeString = [](std::string_view Input) -> std::string { + std::string Result; + Result.resize(Base64::GetEncodedDataSize(uint32_t(Input.size()))); + Base64::Encode(reinterpret_cast(Input.data()), uint32_t(Input.size()), Result.data()); + return Result; + }; + + auto DecodeString = [](std::string_view Input) -> std::string { + std::string Result; + Result.resize(Base64::GetMaxDecodedDataSize(uint32_t(Input.size()))); + uint32_t DecodedLength = 0; + bool Success = Base64::Decode(Input.data(), uint32_t(Input.size()), reinterpret_cast(Result.data()), DecodedLength); + CHECK(Success); + Result.resize(DecodedLength); + return Result; + }; + + SUBCASE("Encode") + { + CHECK(EncodeString("") == ""s); + CHECK(EncodeString("f") == "Zg=="s); + CHECK(EncodeString("fo") == "Zm8="s); + CHECK(EncodeString("foo") == "Zm9v"s); + CHECK(EncodeString("foob") == "Zm9vYg=="s); + CHECK(EncodeString("fooba") == "Zm9vYmE="s); + CHECK(EncodeString("foobar") == "Zm9vYmFy"s); + } + + SUBCASE("Decode") + { + CHECK(DecodeString("") == ""s); + CHECK(DecodeString("Zg==") == "f"s); + CHECK(DecodeString("Zm8=") == "fo"s); + CHECK(DecodeString("Zm9v") == "foo"s); + CHECK(DecodeString("Zm9vYg==") == "foob"s); + CHECK(DecodeString("Zm9vYmE=") == "fooba"s); + CHECK(DecodeString("Zm9vYmFy") == "foobar"s); + } + + SUBCASE("RoundTrip") + { + auto RoundTrip = [&](const std::string& Input) { + std::string Encoded = EncodeString(Input); + std::string Decoded = DecodeString(Encoded); + CHECK(Decoded == Input); + }; + + RoundTrip("Hello, World!"); + RoundTrip("Base64 encoding test with various lengths"); + RoundTrip("A"); + RoundTrip("AB"); + RoundTrip("ABC"); + RoundTrip("ABCD"); + RoundTrip("\x00\x01\x02\xff\xfe\xfd"s); + } + + SUBCASE("BinaryRoundTrip") + { + // Test with all byte values 0-255 + uint8_t AllBytes[256]; + for (int i = 0; i < 256; ++i) + { + AllBytes[i] = static_cast(i); + } + + char Encoded[Base64::GetEncodedDataSize(256) + 1]; + Base64::Encode(AllBytes, 256, Encoded); + + uint8_t Decoded[256]; + uint32_t DecodedLength = 0; + bool Success = Base64::Decode(Encoded, uint32_t(strlen(Encoded)), Decoded, DecodedLength); + CHECK(Success); + CHECK(DecodedLength == 256); + CHECK(memcmp(AllBytes, Decoded, 256) == 0); + } + + SUBCASE("DecodeInvalidInput") + { + uint8_t Dest[64]; + uint32_t OutLength = 0; + + // Length not a multiple of 4 + CHECK_FALSE(Base64::Decode("abc", 3u, Dest, OutLength)); + + // Invalid character + CHECK_FALSE(Base64::Decode("ab!d", 4u, Dest, OutLength)); + } + + SUBCASE("EncodedDataSize") + { + CHECK(Base64::GetEncodedDataSize(0) == 0); + CHECK(Base64::GetEncodedDataSize(1) == 4); + CHECK(Base64::GetEncodedDataSize(2) == 4); + CHECK(Base64::GetEncodedDataSize(3) == 4); + CHECK(Base64::GetEncodedDataSize(4) == 8); + CHECK(Base64::GetEncodedDataSize(5) == 8); + CHECK(Base64::GetEncodedDataSize(6) == 8); + } + + SUBCASE("MaxDecodedDataSize") + { + CHECK(Base64::GetMaxDecodedDataSize(0) == 0); + CHECK(Base64::GetMaxDecodedDataSize(4) == 3); + CHECK(Base64::GetMaxDecodedDataSize(8) == 6); + CHECK(Base64::GetMaxDecodedDataSize(12) == 9); + } +} + +#endif + } // namespace zen diff --git a/src/zencore/include/zencore/base64.h b/src/zencore/include/zencore/base64.h index 4d78b085f..08d9f3043 100644 --- a/src/zencore/include/zencore/base64.h +++ b/src/zencore/include/zencore/base64.h @@ -11,7 +11,11 @@ struct Base64 template static uint32_t Encode(const uint8_t* Source, uint32_t Length, CharType* Dest); + template + static bool Decode(const CharType* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength); + static inline constexpr int32_t GetEncodedDataSize(uint32_t Size) { return ((Size + 2) / 3) * 4; } + static inline constexpr int32_t GetMaxDecodedDataSize(uint32_t Length) { return (Length / 4) * 3; } }; } // namespace zen -- cgit v1.2.3 From 9aac0fd369b87e965fb34b5168646387de7ea1cd Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 23 Feb 2026 11:19:52 +0100 Subject: implement yaml generation (#774) this implements a yaml generation strategy similar to the JSON generation where we just build a string instead of building a ryml tree. This also removes the dependency on ryml for reduced binary/build times. --- src/zencore/compactbinaryyaml.cpp | 427 +++++++++++++++++++++++++++----------- src/zencore/xmake.lua | 2 - 2 files changed, 302 insertions(+), 127 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/compactbinaryyaml.cpp b/src/zencore/compactbinaryyaml.cpp index 5122e952a..b308af418 100644 --- a/src/zencore/compactbinaryyaml.cpp +++ b/src/zencore/compactbinaryyaml.cpp @@ -14,11 +14,6 @@ #include #include -ZEN_THIRD_PARTY_INCLUDES_START -#include -#include -ZEN_THIRD_PARTY_INCLUDES_END - namespace zen { ////////////////////////////////////////////////////////////////////////// @@ -26,193 +21,349 @@ namespace zen { class CbYamlWriter { public: - explicit CbYamlWriter(StringBuilderBase& InBuilder) : m_StrBuilder(InBuilder) { m_NodeStack.push_back(m_Tree.rootref()); } + explicit CbYamlWriter(StringBuilderBase& InBuilder) : m_Builder(InBuilder) {} void WriteField(CbFieldView Field) { - ryml::NodeRef Node; + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); - if (m_IsFirst) + switch (Type) { - Node = Top(); + case CbFieldType::Object: + case CbFieldType::UniformObject: + WriteMapEntries(Field, 0); + break; + case CbFieldType::Array: + case CbFieldType::UniformArray: + WriteSeqEntries(Field, 0); + break; + default: + WriteScalarValue(Field); + m_Builder << '\n'; + break; + } + } + + void WriteMapEntry(CbFieldView Field, int32_t Indent) + { + WriteIndent(Indent); + WriteMapEntryContent(Field, Indent); + } + + void WriteSeqEntry(CbFieldView Field, int32_t Indent) + { + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); - m_IsFirst = false; + if (Type == CbFieldType::Object || Type == CbFieldType::UniformObject) + { + bool First = true; + for (CbFieldView MapChild : Field) + { + if (First) + { + WriteIndent(Indent); + m_Builder << "- "; + First = false; + } + else + { + WriteIndent(Indent + 1); + } + WriteMapEntryContent(MapChild, Indent + 1); + } + } + else if (Type == CbFieldType::Array || Type == CbFieldType::UniformArray) + { + WriteIndent(Indent); + m_Builder << "-\n"; + WriteSeqEntries(Field, Indent + 1); } else { - Node = Top().append_child(); + WriteIndent(Indent); + m_Builder << "- "; + WriteScalarValue(Field); + m_Builder << '\n'; } + } - if (std::u8string_view Name = Field.GetU8Name(); !Name.empty()) +private: + void WriteMapEntries(CbFieldView MapField, int32_t Indent) + { + for (CbFieldView Child : MapField) { - Node.set_key_serialized(ryml::csubstr((const char*)Name.data(), Name.size())); + WriteIndent(Indent); + WriteMapEntryContent(Child, Indent); } + } + + void WriteMapEntryContent(CbFieldView Field, int32_t Indent) + { + std::u8string_view Name = Field.GetU8Name(); + m_Builder << std::string_view(reinterpret_cast(Name.data()), Name.size()); - switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); + + if (IsContainer(Type)) { - case CbFieldType::Null: - Node.set_val("null"); - break; - case CbFieldType::Object: - case CbFieldType::UniformObject: - Node |= ryml::MAP; - m_NodeStack.push_back(Node); - for (CbFieldView It : Field) + m_Builder << ":\n"; + WriteFieldValue(Field, Indent + 1); + } + else + { + m_Builder << ": "; + WriteScalarValue(Field); + m_Builder << '\n'; + } + } + + void WriteSeqEntries(CbFieldView SeqField, int32_t Indent) + { + for (CbFieldView Child : SeqField) + { + CbValue Accessor = Child.GetValue(); + CbFieldType Type = Accessor.GetType(); + + if (Type == CbFieldType::Object || Type == CbFieldType::UniformObject) + { + bool First = true; + for (CbFieldView MapChild : Child) { - WriteField(It); + if (First) + { + WriteIndent(Indent); + m_Builder << "- "; + First = false; + } + else + { + WriteIndent(Indent + 1); + } + WriteMapEntryContent(MapChild, Indent + 1); } - m_NodeStack.pop_back(); + } + else if (Type == CbFieldType::Array || Type == CbFieldType::UniformArray) + { + WriteIndent(Indent); + m_Builder << "-\n"; + WriteSeqEntries(Child, Indent + 1); + } + else + { + WriteIndent(Indent); + m_Builder << "- "; + WriteScalarValue(Child); + m_Builder << '\n'; + } + } + } + + void WriteFieldValue(CbFieldView Field, int32_t Indent) + { + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); + + switch (Type) + { + case CbFieldType::Object: + case CbFieldType::UniformObject: + WriteMapEntries(Field, Indent); break; case CbFieldType::Array: case CbFieldType::UniformArray: - Node |= ryml::SEQ; - m_NodeStack.push_back(Node); - for (CbFieldView It : Field) - { - WriteField(It); - } - m_NodeStack.pop_back(); + WriteSeqEntries(Field, Indent); break; - case CbFieldType::Binary: - { - ExtendableStringBuilder<256> Builder; - const MemoryView Value = Accessor.AsBinary(); - ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); - - Node.set_key_serialized(Builder.c_str()); - } + case CbFieldType::CustomById: + WriteCustomById(Field.GetValue().AsCustomById(), Indent); break; - case CbFieldType::String: - { - const std::u8string_view U8String = Accessor.AsU8String(); - Node.set_val(ryml::csubstr((const char*)U8String.data(), U8String.size())); - } + case CbFieldType::CustomByName: + WriteCustomByName(Field.GetValue().AsCustomByName(), Indent); + break; + default: + WriteScalarValue(Field); + m_Builder << '\n'; + break; + } + } + + void WriteScalarValue(CbFieldView Field) + { + CbValue Accessor = Field.GetValue(); + switch (Accessor.GetType()) + { + case CbFieldType::Null: + m_Builder << "null"; + break; + case CbFieldType::BoolFalse: + m_Builder << "false"; + break; + case CbFieldType::BoolTrue: + m_Builder << "true"; break; case CbFieldType::IntegerPositive: - Node << Accessor.AsIntegerPositive(); + m_Builder << Accessor.AsIntegerPositive(); break; case CbFieldType::IntegerNegative: - Node << Accessor.AsIntegerNegative(); + m_Builder << Accessor.AsIntegerNegative(); break; case CbFieldType::Float32: if (const float Value = Accessor.AsFloat32(); std::isfinite(Value)) - { - Node << Value; - } + m_Builder.Append(fmt::format("{}", Value)); else - { - Node << "null"; - } + m_Builder << "null"; break; case CbFieldType::Float64: if (const double Value = Accessor.AsFloat64(); std::isfinite(Value)) - { - Node << Value; - } + m_Builder.Append(fmt::format("{}", Value)); else + m_Builder << "null"; + break; + case CbFieldType::String: { - Node << "null"; + const std::u8string_view U8String = Accessor.AsU8String(); + WriteString(std::string_view(reinterpret_cast(U8String.data()), U8String.size())); } break; - case CbFieldType::BoolFalse: - Node << "false"; - break; - case CbFieldType::BoolTrue: - Node << "true"; + case CbFieldType::Hash: + WriteString(Accessor.AsHash().ToHexString()); break; case CbFieldType::ObjectAttachment: case CbFieldType::BinaryAttachment: - Node << Accessor.AsAttachment().ToHexString(); - break; - case CbFieldType::Hash: - Node << Accessor.AsHash().ToHexString(); + WriteString(Accessor.AsAttachment().ToHexString()); break; case CbFieldType::Uuid: - Node << fmt::format("{}", Accessor.AsUuid()); + WriteString(fmt::format("{}", Accessor.AsUuid())); break; case CbFieldType::DateTime: - Node << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); + WriteString(DateTime(Accessor.AsDateTimeTicks()).ToIso8601()); break; case CbFieldType::TimeSpan: if (const TimeSpan Span(Accessor.AsTimeSpanTicks()); Span.GetDays() == 0) - { - Node << Span.ToString("%h:%m:%s.%n"); - } + WriteString(Span.ToString("%h:%m:%s.%n")); else - { - Node << Span.ToString("%d.%h:%m:%s.%n"); - } + WriteString(Span.ToString("%d.%h:%m:%s.%n")); break; case CbFieldType::ObjectId: - Node << fmt::format("{}", Accessor.AsObjectId()); + WriteString(fmt::format("{}", Accessor.AsObjectId())); break; - case CbFieldType::CustomById: - { - CbCustomById Custom = Accessor.AsCustomById(); + case CbFieldType::Binary: + WriteBase64(Accessor.AsBinary()); + break; + default: + ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType())); + break; + } + } - Node |= ryml::MAP; + void WriteCustomById(CbCustomById Custom, int32_t Indent) + { + WriteIndent(Indent); + m_Builder << "Id: "; + m_Builder.Append(fmt::format("{}", Custom.Id)); + m_Builder << '\n'; + + WriteIndent(Indent); + m_Builder << "Data: "; + WriteBase64(Custom.Data); + m_Builder << '\n'; + } - ryml::NodeRef IdNode = Node.append_child(); - IdNode.set_key("Id"); - IdNode.set_val_serialized(fmt::format("{}", Custom.Id)); + void WriteCustomByName(CbCustomByName Custom, int32_t Indent) + { + WriteIndent(Indent); + m_Builder << "Name: "; + WriteString(std::string_view(reinterpret_cast(Custom.Name.data()), Custom.Name.size())); + m_Builder << '\n'; + + WriteIndent(Indent); + m_Builder << "Data: "; + WriteBase64(Custom.Data); + m_Builder << '\n'; + } - ryml::NodeRef DataNode = Node.append_child(); - DataNode.set_key("Data"); + void WriteBase64(MemoryView Value) + { + ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); + ExtendableStringBuilder<256> Buf; + const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); + const size_t EncodedIndex = Buf.AddUninitialized(size_t(EncodedSize)); + Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Buf.Data() + EncodedIndex); + WriteString(Buf.ToView()); + } - ExtendableStringBuilder<256> Builder; - const MemoryView& Value = Custom.Data; - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + void WriteString(std::string_view Str) + { + if (NeedsQuoting(Str)) + { + m_Builder << '\''; + for (char C : Str) + { + if (C == '\'') + m_Builder << "''"; + else + m_Builder << C; + } + m_Builder << '\''; + } + else + { + m_Builder << Str; + } + } - DataNode.set_val_serialized(Builder.c_str()); - } - break; - case CbFieldType::CustomByName: - { - CbCustomByName Custom = Accessor.AsCustomByName(); + void WriteIndent(int32_t Indent) + { + for (int32_t I = 0; I < Indent; ++I) + m_Builder << " "; + } - Node |= ryml::MAP; + static bool NeedsQuoting(std::string_view Str) + { + if (Str.empty()) + return false; - ryml::NodeRef NameNode = Node.append_child(); - NameNode.set_key("Name"); - std::string_view Name = std::string_view((const char*)Custom.Name.data(), Custom.Name.size()); - NameNode.set_val_serialized(std::string(Name)); + char First = Str[0]; + if (First == ' ' || First == '\n' || First == '\t' || First == '\r' || First == '*' || First == '&' || First == '%' || + First == '@' || First == '`') + return true; - ryml::NodeRef DataNode = Node.append_child(); - DataNode.set_key("Data"); + if (Str.size() >= 2 && Str[0] == '<' && Str[1] == '<') + return true; - ExtendableStringBuilder<256> Builder; - const MemoryView& Value = Custom.Data; - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + char Last = Str.back(); + if (Last == ' ' || Last == '\n' || Last == '\t' || Last == '\r') + return true; - DataNode.set_val_serialized(Builder.c_str()); - } - break; - default: - ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType())); - break; + for (char C : Str) + { + if (C == '#' || C == ':' || C == '-' || C == '?' || C == ',' || C == '\n' || C == '{' || C == '}' || C == '[' || C == ']' || + C == '\'' || C == '"') + return true; } - if (m_NodeStack.size() == 1) + return false; + } + + static bool IsContainer(CbFieldType Type) + { + switch (Type) { - std::string Yaml = ryml::emitrs_yaml(m_Tree); - m_StrBuilder << Yaml; + case CbFieldType::Object: + case CbFieldType::UniformObject: + case CbFieldType::Array: + case CbFieldType::UniformArray: + case CbFieldType::CustomById: + case CbFieldType::CustomByName: + return true; + default: + return false; } } -private: - StringBuilderBase& m_StrBuilder; - bool m_IsFirst = true; - - ryml::Tree m_Tree; - std::vector m_NodeStack; - ryml::NodeRef& Top() { return m_NodeStack.back(); } + StringBuilderBase& m_Builder; }; void @@ -229,6 +380,32 @@ CompactBinaryToYaml(const CbArrayView& Array, StringBuilderBase& Builder) Writer.WriteField(Array.AsFieldView()); } +void +CompactBinaryToYaml(MemoryView Data, StringBuilderBase& InBuilder) +{ + std::vector Fields = ReadCompactBinaryStream(Data); + if (Fields.empty()) + return; + + CbYamlWriter Writer(InBuilder); + if (Fields.size() == 1) + { + Writer.WriteField(Fields[0]); + return; + } + + if (Fields[0].HasName()) + { + for (const CbFieldView& Field : Fields) + Writer.WriteMapEntry(Field, 0); + } + else + { + for (const CbFieldView& Field : Fields) + Writer.WriteSeqEntry(Field, 0); + } +} + #if ZEN_WITH_TESTS void cbyaml_forcelink() diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index a3fd4dacb..9a67175a0 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -33,8 +33,6 @@ target('zencore') add_deps("timesinceprocessstart") add_deps("doctest") add_deps("fmt") - add_deps("ryml") - add_packages("json11") if is_plat("linux", "macosx") then -- cgit v1.2.3 From 5c5e12d1f02bb7cc1f42750e47a2735dc933c194 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 24 Feb 2026 14:56:57 +0100 Subject: Various bug fixes (#778) zencore fixes: - filesystem.cpp: ReadFile error reporting logic - compactbinaryvalue.h: CbValue::As*String error reporting logic zenhttp fixes: - httpasio BindAcceptor would `return 0;` in a function returning `std::string` (UB) - httpsys async workpool initialization race zenstore fixes: - cas.cpp: GetFileCasResults Results param passed by value instead of reference (large chunk results were silently lost) - structuredcachestore.cpp: MissCount unconditionally incremented (counted hits as misses) - cacherpc.cpp: Wrong boolean in Incomplete response array (all entries marked incomplete) - cachedisklayer.cpp: sizeof(sizeof(...)) in two validation checks computed sizeof(size_t) instead of struct size - buildstore.cpp: Wrong hash tracked in GC key list (BlobHash pushed twice instead of MetadataHash) - buildstore.cpp: Removed duplicate m_LastAccessTimeUpdateCount increment in PutBlob zenserver fixes: - httpbuildstore.cpp: Reversed subtraction in HTTP range calculation (unsigned underflow) - hubservice.cpp: Deadlock in Provision() calling Wake() while holding m_Lock (extracted WakeLocked helper) - zipfs.cpp: Data race in GetFile() lazy initialization (added RwLock with shared/exclusive paths) --- src/zencore/filesystem.cpp | 16 +++------------- src/zencore/include/zencore/compactbinaryvalue.h | 24 ++++++++++++++++-------- src/zencore/memtrack/callstacktrace.cpp | 8 ++++---- src/zencore/string.cpp | 4 ++++ 4 files changed, 27 insertions(+), 25 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 1a4ee4b9b..553897407 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -1326,11 +1326,6 @@ ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uin { BytesRead = size_t(dwNumberOfBytesRead); } - else if ((BytesRead != NumberOfBytesToRead)) - { - Ec = MakeErrorCode(ERROR_HANDLE_EOF); - return; - } else { Ec = MakeErrorCodeFromLastError(); @@ -1344,20 +1339,15 @@ ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uin { BytesRead = size_t(ReadResult); } - else if ((BytesRead != NumberOfBytesToRead)) - { - Ec = MakeErrorCode(EIO); - return; - } else { Ec = MakeErrorCodeFromLastError(); return; } #endif - Size -= NumberOfBytesToRead; - FileOffset += NumberOfBytesToRead; - Data = reinterpret_cast(Data) + NumberOfBytesToRead; + Size -= BytesRead; + FileOffset += BytesRead; + Data = reinterpret_cast(Data) + BytesRead; } } diff --git a/src/zencore/include/zencore/compactbinaryvalue.h b/src/zencore/include/zencore/compactbinaryvalue.h index aa2d2821d..4ce8009b8 100644 --- a/src/zencore/include/zencore/compactbinaryvalue.h +++ b/src/zencore/include/zencore/compactbinaryvalue.h @@ -128,17 +128,21 @@ CbValue::AsString(CbFieldError* OutError, std::string_view Default) const uint32_t ValueSizeByteCount; const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount); - if (OutError) + if (ValueSize >= (uint64_t(1) << 31)) { - if (ValueSize >= (uint64_t(1) << 31)) + if (OutError) { *OutError = CbFieldError::RangeError; - return Default; } + return Default; + } + + if (OutError) + { *OutError = CbFieldError::None; } - return std::string_view(Chars + ValueSizeByteCount, int32_t(ValueSize)); + return std::string_view(Chars + ValueSizeByteCount, size_t(ValueSize)); } inline std::u8string_view @@ -148,17 +152,21 @@ CbValue::AsU8String(CbFieldError* OutError, std::u8string_view Default) const uint32_t ValueSizeByteCount; const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount); - if (OutError) + if (ValueSize >= (uint64_t(1) << 31)) { - if (ValueSize >= (uint64_t(1) << 31)) + if (OutError) { *OutError = CbFieldError::RangeError; - return Default; } + return Default; + } + + if (OutError) + { *OutError = CbFieldError::None; } - return std::u8string_view(Chars + ValueSizeByteCount, int32_t(ValueSize)); + return std::u8string_view(Chars + ValueSizeByteCount, size_t(ValueSize)); } inline uint64_t diff --git a/src/zencore/memtrack/callstacktrace.cpp b/src/zencore/memtrack/callstacktrace.cpp index a5b7fede6..4a7068568 100644 --- a/src/zencore/memtrack/callstacktrace.cpp +++ b/src/zencore/memtrack/callstacktrace.cpp @@ -169,13 +169,13 @@ private: std::atomic_uint64_t Key; std::atomic_uint32_t Value; - inline uint64 GetKey() const { return Key.load(std::memory_order_relaxed); } + inline uint64 GetKey() const { return Key.load(std::memory_order_acquire); } inline uint32_t GetValue() const { return Value.load(std::memory_order_relaxed); } - inline bool IsEmpty() const { return Key.load(std::memory_order_relaxed) == 0; } + inline bool IsEmpty() const { return Key.load(std::memory_order_acquire) == 0; } inline void SetKeyValue(uint64_t InKey, uint32_t InValue) { - Value.store(InValue, std::memory_order_release); - Key.store(InKey, std::memory_order_relaxed); + Value.store(InValue, std::memory_order_relaxed); + Key.store(InKey, std::memory_order_release); } static inline uint32_t KeyHash(uint64_t Key) { return static_cast(Key); } static inline void ClearEntries(FEncounteredCallstackSetEntry* Entries, int32_t EntryCount) diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp index 0ee863b74..a9aed6309 100644 --- a/src/zencore/string.cpp +++ b/src/zencore/string.cpp @@ -24,6 +24,10 @@ utf16to8_impl(u16bit_iterator StartIt, u16bit_iterator EndIt, ::zen::StringBuild // Take care of surrogate pairs first if (utf8::internal::is_lead_surrogate(cp)) { + if (StartIt == EndIt) + { + break; + } uint32_t trail_surrogate = utf8::internal::mask16(*StartIt++); cp = (cp << 10) + trail_surrogate + utf8::internal::SURROGATE_OFFSET; } -- cgit v1.2.3 From 3cfc1b18f6b86b9830730f0055b8e3b955b77c95 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 24 Feb 2026 15:36:59 +0100 Subject: Add `zen ui` command (#779) Allows user to automate launching of zenserver dashboard, including when multiple instances are running. If multiple instances are running you can open all dashboards with `--all`, and also using the in-terminal chooser which also allows you to open a specific instance. Also includes a fix to `zen exec` when using offset/stride/limit --- src/zencore/include/zencore/process.h | 1 + src/zencore/process.cpp | 226 ++++++++++++++++++++++++++++++++++ 2 files changed, 227 insertions(+) (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index e3b7a70d7..c51163a68 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -105,6 +105,7 @@ int GetCurrentProcessId(); int GetProcessId(CreateProcResult ProcId); std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc); +std::string GetProcessCommandLine(int Pid, std::error_code& OutEc); std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle, bool IncludeSelf = true); /** Wait for all threads in the current process to exit (except the calling thread) diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 56849a10d..4a2668912 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -1001,6 +1001,232 @@ GetProcessExecutablePath(int Pid, std::error_code& OutEc) #endif // ZEN_PLATFORM_LINUX } +std::string +GetProcessCommandLine(int Pid, std::error_code& OutEc) +{ +#if ZEN_PLATFORM_WINDOWS + HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, static_cast(Pid)); + if (!hProcess) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + auto _ = MakeGuard([hProcess] { CloseHandle(hProcess); }); + + // NtQueryInformationProcess is an undocumented NT API; load it dynamically. + // Info class 60 = ProcessCommandLine, available since Windows 8.1. + using PFN_NtQIP = LONG(WINAPI*)(HANDLE, UINT, PVOID, ULONG, PULONG); + static const PFN_NtQIP s_NtQIP = + reinterpret_cast(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "NtQueryInformationProcess")); + if (!s_NtQIP) + { + return {}; + } + + constexpr UINT ProcessCommandLineClass = 60; + constexpr LONG StatusInfoLengthMismatch = static_cast(0xC0000004L); + + ULONG ReturnLength = 0; + LONG Status = s_NtQIP(hProcess, ProcessCommandLineClass, nullptr, 0, &ReturnLength); + if (Status != StatusInfoLengthMismatch || ReturnLength == 0) + { + return {}; + } + + std::vector Buf(ReturnLength); + Status = s_NtQIP(hProcess, ProcessCommandLineClass, Buf.data(), ReturnLength, &ReturnLength); + if (Status < 0) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + + // Output: UNICODE_STRING header immediately followed by the UTF-16 string data. + // The UNICODE_STRING.Buffer field points into our Buf. + struct LocalUnicodeString + { + USHORT Length; + USHORT MaximumLength; + WCHAR* Buffer; + }; + if (ReturnLength < sizeof(LocalUnicodeString)) + { + return {}; + } + const auto* Us = reinterpret_cast(Buf.data()); + if (Us->Length == 0 || Us->Buffer == nullptr) + { + return {}; + } + + // Skip argv[0]: may be a quoted path ("C:\...\exe.exe") or a bare path + const WCHAR* p = Us->Buffer; + const WCHAR* End = Us->Buffer + Us->Length / sizeof(WCHAR); + if (p < End && *p == L'"') + { + ++p; + while (p < End && *p != L'"') + { + ++p; + } + if (p < End) + { + ++p; // skip closing quote + } + } + else + { + while (p < End && *p != L' ') + { + ++p; + } + } + while (p < End && *p == L' ') + { + ++p; + } + if (p >= End) + { + return {}; + } + + int Utf8Size = WideCharToMultiByte(CP_UTF8, 0, p, static_cast(End - p), nullptr, 0, nullptr, nullptr); + if (Utf8Size <= 0) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + std::string Result(Utf8Size, '\0'); + WideCharToMultiByte(CP_UTF8, 0, p, static_cast(End - p), Result.data(), Utf8Size, nullptr, nullptr); + return Result; + +#elif ZEN_PLATFORM_LINUX + std::string CmdlinePath = fmt::format("/proc/{}/cmdline", Pid); + FILE* F = fopen(CmdlinePath.c_str(), "rb"); + if (!F) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + auto FGuard = MakeGuard([F] { fclose(F); }); + + // /proc/{pid}/cmdline contains null-separated argv entries; read it all + std::string Raw; + char Chunk[4096]; + size_t BytesRead; + while ((BytesRead = fread(Chunk, 1, sizeof(Chunk), F)) > 0) + { + Raw.append(Chunk, BytesRead); + } + if (Raw.empty()) + { + return {}; + } + + // Skip argv[0] (first null-terminated entry) + const char* p = Raw.data(); + const char* End = Raw.data() + Raw.size(); + while (p < End && *p != '\0') + { + ++p; + } + if (p < End) + { + ++p; // skip null terminator of argv[0] + } + + // Build result: remaining entries joined by spaces (inter-arg nulls → spaces) + std::string Result; + Result.reserve(static_cast(End - p)); + for (const char* q = p; q < End; ++q) + { + Result += (*q == '\0') ? ' ' : *q; + } + while (!Result.empty() && Result.back() == ' ') + { + Result.pop_back(); + } + return Result; + +#elif ZEN_PLATFORM_MAC + int Mib[3] = {CTL_KERN, KERN_PROCARGS2, Pid}; + size_t BufSize = 0; + if (sysctl(Mib, 3, nullptr, &BufSize, nullptr, 0) != 0 || BufSize == 0) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + + std::vector Buf(BufSize); + if (sysctl(Mib, 3, Buf.data(), &BufSize, nullptr, 0) != 0) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + + // Layout: [int argc][exec_path\0][null padding][argv[0]\0][argv[1]\0]...[envp\0]... + if (BufSize < sizeof(int)) + { + return {}; + } + int Argc = 0; + memcpy(&Argc, Buf.data(), sizeof(int)); + if (Argc <= 1) + { + return {}; + } + + const char* p = Buf.data() + sizeof(int); + const char* End = Buf.data() + BufSize; + + // Skip exec_path and any trailing null padding that follows it + while (p < End && *p != '\0') + { + ++p; + } + while (p < End && *p == '\0') + { + ++p; + } + + // Skip argv[0] + while (p < End && *p != '\0') + { + ++p; + } + if (p < End) + { + ++p; + } + + // Collect argv[1..Argc-1] + std::string Result; + for (int i = 1; i < Argc && p < End; ++i) + { + if (i > 1) + { + Result += ' '; + } + const char* ArgStart = p; + while (p < End && *p != '\0') + { + ++p; + } + Result.append(ArgStart, p); + if (p < End) + { + ++p; + } + } + return Result; + +#else + ZEN_UNUSED(Pid); + ZEN_UNUSED(OutEc); + return {}; +#endif +} + std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle, bool IncludeSelf) { -- cgit v1.2.3 From fb19c8a86e89762ea89df3b361494a055680b432 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 24 Feb 2026 22:24:11 +0100 Subject: Fix zencore bugs and propagate content type through IoBufferBuilder (#783) - Add missing includes in hashutils.h (``, ``) - Add `ZenContentType` parameter to all `IoBufferBuilder` factory methods so content type is set at buffer creation time - Fix null dereference in `SharedBuffer::GetFileReference()` when buffer is null - Fix out-of-bounds read in trace command-line argument parsing when arg length exactly matches option length - Add unit tests for 32-bit `CountLeadingZeros` --- src/zencore/include/zencore/hashutils.h | 3 +++ src/zencore/include/zencore/iobuffer.h | 37 ++++++++++++++++++++++-------- src/zencore/include/zencore/sharedbuffer.h | 13 ++++++----- src/zencore/intmath.cpp | 6 +++++ src/zencore/iobuffer.cpp | 20 +++++++++------- src/zencore/trace.cpp | 13 ++++++++--- 6 files changed, 65 insertions(+), 27 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/hashutils.h b/src/zencore/include/zencore/hashutils.h index 4e877e219..6b9902b3a 100644 --- a/src/zencore/include/zencore/hashutils.h +++ b/src/zencore/include/zencore/hashutils.h @@ -2,6 +2,9 @@ #pragma once +#include +#include + namespace zen { template diff --git a/src/zencore/include/zencore/iobuffer.h b/src/zencore/include/zencore/iobuffer.h index 182768ff6..82c201edd 100644 --- a/src/zencore/include/zencore/iobuffer.h +++ b/src/zencore/include/zencore/iobuffer.h @@ -426,22 +426,39 @@ private: class IoBufferBuilder { public: - static IoBuffer MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); - static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName); - static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); - /** Make sure buffer data is memory resident, but avoid memory mapping data from files - */ - static IoBuffer ReadFromFileMaybe(const IoBuffer& InBuffer); - inline static IoBuffer MakeFromMemory(MemoryView Memory) { return IoBuffer(IoBuffer::Wrap, Memory.GetData(), Memory.GetSize()); } - inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) + static IoBuffer MakeFromFile(const std::filesystem::path& FileName, + uint64_t Offset = 0, + uint64_t Size = ~0ull, + ZenContentType ContentType = ZenContentType::kBinary); + static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName, ZenContentType ContentType = ZenContentType::kBinary); + static IoBuffer MakeFromFileHandle(void* FileHandle, + uint64_t Offset = 0, + uint64_t Size = ~0ull, + ZenContentType ContentType = ZenContentType::kBinary); + inline static IoBuffer MakeFromMemory(MemoryView Memory, ZenContentType ContentType = ZenContentType::kBinary) + { + IoBuffer NewBuffer(IoBuffer::Wrap, Memory.GetData(), Memory.GetSize()); + NewBuffer.SetContentType(ContentType); + return NewBuffer; + } + inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz, ZenContentType ContentType = ZenContentType::kBinary) { if (Sz) { - return IoBuffer(IoBuffer::Clone, Ptr, Sz); + IoBuffer NewBuffer(IoBuffer::Clone, Ptr, Sz); + NewBuffer.SetContentType(ContentType); + return NewBuffer; } return {}; } - inline static IoBuffer MakeCloneFromMemory(MemoryView Memory) { return MakeCloneFromMemory(Memory.GetData(), Memory.GetSize()); } + inline static IoBuffer MakeCloneFromMemory(MemoryView Memory, ZenContentType ContentType = ZenContentType::kBinary) + { + return MakeCloneFromMemory(Memory.GetData(), Memory.GetSize(), ContentType); + } + + /** Make sure buffer data is memory resident, but avoid memory mapping data from files + */ + static IoBuffer ReadFromFileMaybe(const IoBuffer& InBuffer); }; void iobuffer_forcelink(); diff --git a/src/zencore/include/zencore/sharedbuffer.h b/src/zencore/include/zencore/sharedbuffer.h index c57e9f568..3d4c19282 100644 --- a/src/zencore/include/zencore/sharedbuffer.h +++ b/src/zencore/include/zencore/sharedbuffer.h @@ -116,14 +116,15 @@ public: inline void Reset() { m_Buffer = nullptr; } inline bool GetFileReference(IoBufferFileReference& OutRef) const { - if (const IoBufferExtendedCore* Core = m_Buffer->ExtendedCore()) + if (!IsNull()) { - return Core->GetFileReference(OutRef); - } - else - { - return false; + if (const IoBufferExtendedCore* Core = m_Buffer->ExtendedCore()) + { + return Core->GetFileReference(OutRef); + } } + + return false; } [[nodiscard]] MemoryView GetView() const diff --git a/src/zencore/intmath.cpp b/src/zencore/intmath.cpp index 5a686dc8e..32f82b486 100644 --- a/src/zencore/intmath.cpp +++ b/src/zencore/intmath.cpp @@ -43,6 +43,12 @@ TEST_CASE("intmath") CHECK(FloorLog2_64(0x0000'0001'0000'0000ull) == 32); CHECK(FloorLog2_64(0x8000'0000'0000'0000ull) == 63); + CHECK(CountLeadingZeros(0x8000'0000u) == 0); + CHECK(CountLeadingZeros(0x0000'0000u) == 32); + CHECK(CountLeadingZeros(0x0000'0001u) == 31); + CHECK(CountLeadingZeros(0x0000'8000u) == 16); + CHECK(CountLeadingZeros(0x0001'0000u) == 15); + CHECK(CountLeadingZeros64(0x8000'0000'0000'0000ull) == 0); CHECK(CountLeadingZeros64(0x0000'0000'0000'0000ull) == 64); CHECK(CountLeadingZeros64(0x0000'0000'0000'0001ull) == 63); diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp index be9b39e7a..1c31d6620 100644 --- a/src/zencore/iobuffer.cpp +++ b/src/zencore/iobuffer.cpp @@ -592,15 +592,17 @@ IoBufferBuilder::ReadFromFileMaybe(const IoBuffer& InBuffer) } IoBuffer -IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t Size) +IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t Size, ZenContentType ContentType) { ZEN_TRACE_CPU("IoBufferBuilder::MakeFromFileHandle"); - return IoBuffer(IoBuffer::BorrowedFile, FileHandle, Offset, Size); + IoBuffer Buffer(IoBuffer::BorrowedFile, FileHandle, Offset, Size); + Buffer.SetContentType(ContentType); + return Buffer; } IoBuffer -IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size) +IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size, ZenContentType ContentType) { ZEN_TRACE_CPU("IoBufferBuilder::MakeFromFile"); @@ -632,8 +634,6 @@ IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Of FileSize = Stat.st_size; #endif // ZEN_PLATFORM_WINDOWS - // TODO: should validate that offset is in range - if (Size == ~0ull) { Size = FileSize - Offset; @@ -652,7 +652,9 @@ IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Of #if ZEN_PLATFORM_WINDOWS void* Fd = DataFile.Detach(); #endif - return IoBuffer(IoBuffer::File, (void*)uintptr_t(Fd), Offset, Size, Offset == 0 && Size == FileSize); + IoBuffer NewBuffer(IoBuffer::File, (void*)uintptr_t(Fd), Offset, Size, Offset == 0 && Size == FileSize); + NewBuffer.SetContentType(ContentType); + return NewBuffer; } #if !ZEN_PLATFORM_WINDOWS @@ -664,7 +666,7 @@ IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Of } IoBuffer -IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName) +IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName, ZenContentType ContentType) { ZEN_TRACE_CPU("IoBufferBuilder::MakeFromTemporaryFile"); @@ -703,7 +705,9 @@ IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName) Handle = (void*)uintptr_t(Fd); #endif // ZEN_PLATFORM_WINDOWS - return IoBuffer(IoBuffer::File, Handle, 0, FileSize, /*IsWholeFile*/ true); + IoBuffer NewBuffer(IoBuffer::File, Handle, 0, FileSize, /*IsWholeFile*/ true); + NewBuffer.SetContentType(ContentType); + return NewBuffer; } ////////////////////////////////////////////////////////////////////////// diff --git a/src/zencore/trace.cpp b/src/zencore/trace.cpp index 87035554f..a026974c0 100644 --- a/src/zencore/trace.cpp +++ b/src/zencore/trace.cpp @@ -165,10 +165,17 @@ GetTraceOptionsFromCommandline(TraceOptions& OutOptions) auto MatchesArg = [](std::string_view Option, std::string_view Arg) -> std::optional { if (Arg.starts_with(Option)) { - std::string_view::value_type DelimChar = Arg[Option.length()]; - if (DelimChar == ' ' || DelimChar == '=') + if (Arg.length() > Option.length()) { - return Arg.substr(Option.size() + 1); + std::string_view::value_type DelimChar = Arg[Option.length()]; + if (DelimChar == ' ' || DelimChar == '=') + { + return Arg.substr(Option.size() + 1); + } + } + else + { + return ""sv; } } return {}; -- cgit v1.2.3 From 4d5caf7d011bf73c7b90ff1d8c1cfdad817fa2f5 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Fri, 27 Feb 2026 11:47:00 +0100 Subject: Ported "lane trace" feature from UE (by way of IAX) (#771) * Ported "lane trace" feature from UE (by way of IAX) --- src/zencore/include/zencore/trace.h | 1 + 1 file changed, 1 insertion(+) (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/trace.h b/src/zencore/include/zencore/trace.h index 99a565151..d17e018ea 100644 --- a/src/zencore/include/zencore/trace.h +++ b/src/zencore/include/zencore/trace.h @@ -13,6 +13,7 @@ ZEN_THIRD_PARTY_INCLUDES_START # define TRACE_IMPLEMENT 0 #endif #include +#include #undef TRACE_IMPLEMENT ZEN_THIRD_PARTY_INCLUDES_END -- cgit v1.2.3 From 65eefdfe5a216b546f0d3d8fdfc5e9e58916e5f8 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 27 Feb 2026 17:12:00 +0100 Subject: add sentry-sdk logger (#793) eliminates spurious sentry log output during startup as the new channel defaults to WARN The level can be overridden via `--log-debug=sentry-sdk` or `--log-info=sentry-sdk` --- src/zencore/sentryintegration.cpp | 62 ++++++++++++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 8 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index 00e67dc85..636e182b4 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -145,6 +145,8 @@ SentryAssertImpl::OnAssert(const char* Filename, namespace zen { # if ZEN_USE_SENTRY +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogSentry, "sentry-sdk"); + static void SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[maybe_unused]] void* Userdata) { @@ -163,26 +165,61 @@ SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[may MessagePtr = LogMessage.c_str(); } + // SentryLogFunction can be called before the logging system is initialized + // (during sentry_init which runs before InitializeLogging). Fall back to + // console logging when the category logger is not yet available. + // + // Since we want to default to WARN level but this runs before logging has + // been configured, we ignore the callbacks for DEBUG/INFO explicitly here + // which means users don't see every possible log message if they're trying + // to configure the levels using --log-debug=sentry-sdk + if (!TheDefaultLogger) + { + switch (Level) + { + case SENTRY_LEVEL_DEBUG: + // ZEN_CONSOLE_DEBUG("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_INFO: + // ZEN_CONSOLE_INFO("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_WARNING: + ZEN_CONSOLE_WARN("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_ERROR: + ZEN_CONSOLE_ERROR("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_FATAL: + ZEN_CONSOLE_CRITICAL("sentry: {}", MessagePtr); + break; + } + return; + } + switch (Level) { case SENTRY_LEVEL_DEBUG: - ZEN_CONSOLE_DEBUG("sentry: {}", MessagePtr); + ZEN_LOG_DEBUG(LogSentry, "sentry: {}", MessagePtr); break; case SENTRY_LEVEL_INFO: - ZEN_CONSOLE_INFO("sentry: {}", MessagePtr); + ZEN_LOG_INFO(LogSentry, "sentry: {}", MessagePtr); break; case SENTRY_LEVEL_WARNING: - ZEN_CONSOLE_WARN("sentry: {}", MessagePtr); + ZEN_LOG_WARN(LogSentry, "sentry: {}", MessagePtr); break; case SENTRY_LEVEL_ERROR: - ZEN_CONSOLE_ERROR("sentry: {}", MessagePtr); + ZEN_LOG_ERROR(LogSentry, "sentry: {}", MessagePtr); break; case SENTRY_LEVEL_FATAL: - ZEN_CONSOLE_CRITICAL("sentry: {}", MessagePtr); + ZEN_LOG_CRITICAL(LogSentry, "sentry: {}", MessagePtr); break; } } @@ -310,22 +347,31 @@ SentryIntegration::Initialize(const Config& Conf, const std::string& CommandLine void SentryIntegration::LogStartupInformation() { + // Initialize the sentry-sdk log category at Warn level to reduce startup noise. + // The level can be overridden via --log-debug=sentry-sdk or --log-info=sentry-sdk + LogSentry.Logger().SetLogLevel(logging::level::Warn); + if (m_IsInitialized) { if (m_SentryErrorCode == 0) { if (m_AllowPII) { - ZEN_INFO("sentry initialized, username: '{}', hostname: '{}', id: '{}'", m_SentryUserName, m_SentryHostName, m_SentryId); + ZEN_LOG_INFO(LogSentry, + "sentry initialized, username: '{}', hostname: '{}', id: '{}'", + m_SentryUserName, + m_SentryHostName, + m_SentryId); } else { - ZEN_INFO("sentry initialized with anonymous reports"); + ZEN_LOG_INFO(LogSentry, "sentry initialized with anonymous reports"); } } else { - ZEN_WARN( + ZEN_LOG_WARN( + LogSentry, "sentry_init returned failure! (error code: {}) note that sentry expects crashpad_handler to exist alongside the running " "executable", m_SentryErrorCode); -- cgit v1.2.3 From 0a41fd42aa43080fbc991e7d976dde70aeaec594 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 27 Feb 2026 17:13:40 +0100 Subject: add full WebSocket (RFC 6455) client/server support for zenhttp (#792) * This branch adds full WebSocket (RFC 6455) support to the HTTP server layer, covering both transport backends, a client, and tests. - **`websocket.h`** -- Core interfaces: `WebSocketOpcode`, `WebSocketMessage`, `WebSocketConnection` (ref-counted), and `IWebSocketHandler`. Services opt in to WebSocket support by implementing `IWebSocketHandler` alongside their existing `HttpService`. - **`httpwsclient.h`** -- `HttpWsClient`: an ASIO-backed `ws://` client with both standalone (own thread) and shared `io_context` modes. Supports connect timeout and optional auth token injection via `IWsClientHandler` callbacks. - **`wsasio.cpp/h`** -- `WsAsioConnection`: WebSocket over ASIO TCP. Takes over the socket after the HTTP 101 handshake and runs an async read/write loop with a queued write path (guarded by `RwLock`). - **`wshttpsys.cpp/h`** -- `WsHttpSysConnection`: WebSocket over http.sys opaque-mode connections (Windows only). Uses `HttpReceiveRequestEntityBody` / `HttpSendResponseEntityBody` via IOCP, sharing the same threadpool as normal http.sys traffic. Self-ref lifetime management ensures graceful drain of outstanding async ops. - **`httpsys_iocontext.h`** -- Tagged `OVERLAPPED` wrapper (`HttpSysIoContext`) used to distinguish normal HTTP transactions from WebSocket read/write completions in the single IOCP callback. - **`wsframecodec.cpp/h`** -- `WsFrameCodec`: static helpers for parsing (unmasked and masked) and building (unmasked server frames and masked client frames) RFC 6455 frames across all three payload length encodings (7-bit, 16-bit, 64-bit). Also computes `Sec-WebSocket-Accept` keys. - **`clients/httpwsclient.cpp`** -- `HttpWsClient::Impl`: ASIO-based client that performs the HTTP upgrade handshake, then hands off to the frame codec for the read loop. Manages its own `io_context` thread or plugs into an external one. - **`httpasio.cpp`** -- ASIO server now detects `Upgrade: websocket` requests, checks the matching `HttpService` for `IWebSocketHandler` via `dynamic_cast`, performs the RFC 6455 handshake (101 response), and spins up a `WsAsioConnection`. - **`httpsys.cpp`** -- Same upgrade detection and handshake logic for the http.sys backend, using `WsHttpSysConnection` and `HTTP_SEND_RESPONSE_FLAG_OPAQUE`. - **`httpparser.cpp/h`** -- Extended to surface the `Upgrade` / `Connection` / `Sec-WebSocket-Key` headers needed by the handshake. - **`httpcommon.h`** -- Minor additions (probably new header constants or response codes for the WS upgrade). - **`httpserver.h`** -- Small interface changes to support WebSocket registration. - **`zenhttp.cpp` / `xmake.lua`** -- New source files wired in; build config updated. - **Unit tests** (`websocket.framecodec`): round-trip encode/decode for text, binary, close frames; all three payload sizes; masked and unmasked variants; RFC 6455 `Sec-WebSocket-Accept` test vector. - **Integration tests** (`websocket.integration`): full ASIO server tests covering handshake (101), normal HTTP coexistence, echo, server-push broadcast, client close handshake, ping/pong auto-response, sequential messages, and rejection of upgrades on non-WS services. - **Client tests** (`websocket.client`): `HttpWsClient` connect+echo+close, connection failure (bad port -> close code 1006), and server-initiated close. * changed HttpRequestParser::ParseCurrentHeader to use switch instead of if/else chain * remove spurious printf --------- Co-authored-by: Stefan Boberg --- src/zencore/filesystem.cpp | 1 - 1 file changed, 1 deletion(-) (limited to 'src/zencore') diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 553897407..03398860b 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -3533,7 +3533,6 @@ TEST_CASE("PathBuilder") Path.Reset(); Path.Append(fspath(L"/\u0119oo/")); Path /= L"bar"; - printf("%ls\n", Path.ToPath().c_str()); CHECK(Path.ToView() == L"/\u0119oo/bar"); CHECK(Path.ToPath() == L"\\\u0119oo\\bar"); # endif -- cgit v1.2.3 From 1ed3139e577f6c8aa6d07f7e76afa3a80d9d4852 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 27 Feb 2026 19:36:22 +0100 Subject: Add test summary table and failure reporting to xmake test (#794) - Add a summary table printed after all test suites complete, showing per-suite test case counts, assertion counts, timings and pass/fail status. - Add failure reporting: individual failing test cases are listed at the end with their file path and line number for easy navigation. - Made zenserver instances spawned by a hub not create new console windows for a better background testing experience - The TestListener in testing.cpp now writes a machine-readable summary file (via `ZEN_TEST_SUMMARY_FILE` env var) containing aggregate counts and per-test-case failure details. This runs as a doctest listener alongside any active reporter, so it works with both console and JUnit modes. - Tests now run in a deterministic order defined by a single ordered list that also serves as the test name/target mapping, replacing the previous unordered table + separate order list. - The `--run` option now accepts comma-separated values (e.g. `--run=core,http,util`) and validates each name, reporting unknown test names early. - Fix platform detection in `xmake test`: the config command now passes `-p` explicitly, fixing "mingw" misdetection when running from Git Bash on Windows. - Add missing "util" entry to the help text for `--run`. --- src/zencore/testing.cpp | 56 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 4 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp index 936424e0f..ef8fb0480 100644 --- a/src/zencore/testing.cpp +++ b/src/zencore/testing.cpp @@ -5,6 +5,12 @@ #if ZEN_WITH_TESTS +# include +# include +# include +# include +# include + # include namespace zen::testing { @@ -21,9 +27,35 @@ struct TestListener : public doctest::IReporter void report_query(const doctest::QueryData& /*in*/) override {} - void test_run_start() override {} + void test_run_start() override { RunStart = std::chrono::steady_clock::now(); } + + void test_run_end(const doctest::TestRunStats& in) override + { + auto elapsed = std::chrono::steady_clock::now() - RunStart; + double elapsedSeconds = std::chrono::duration_cast(elapsed).count() / 1000.0; - void test_run_end(const doctest::TestRunStats& /*in*/) override {} + // Write machine-readable summary to file if requested (used by xmake test summary table) + const char* summaryFile = std::getenv("ZEN_TEST_SUMMARY_FILE"); + if (summaryFile && summaryFile[0] != '\0') + { + if (FILE* f = std::fopen(summaryFile, "w")) + { + std::fprintf(f, + "cases_total=%u\ncases_passed=%u\nassertions_total=%d\nassertions_passed=%d\n" + "elapsed_seconds=%.3f\n", + in.numTestCasesPassingFilters, + in.numTestCasesPassingFilters - in.numTestCasesFailed, + in.numAsserts, + in.numAsserts - in.numAssertsFailed, + elapsedSeconds); + for (const auto& failure : FailedTests) + { + std::fprintf(f, "failed=%s|%s|%u\n", failure.Name.c_str(), failure.File.c_str(), failure.Line); + } + std::fclose(f); + } + } + } void test_case_start(const doctest::TestCaseData& in) override { @@ -37,7 +69,14 @@ struct TestListener : public doctest::IReporter ZEN_CONSOLE("{}-------------------------------------------------------------------------------{}", ColorYellow, ColorNone); } - void test_case_end(const doctest::CurrentTestCaseStats& /*in*/) override { Current = nullptr; } + void test_case_end(const doctest::CurrentTestCaseStats& in) override + { + if (!in.testCaseSuccess && Current) + { + FailedTests.push_back({Current->m_name, Current->m_file.c_str(), Current->m_line}); + } + Current = nullptr; + } void test_case_exception(const doctest::TestCaseException& /*in*/) override {} @@ -57,7 +96,16 @@ struct TestListener : public doctest::IReporter void test_case_skipped(const doctest::TestCaseData& /*in*/) override {} - const doctest::TestCaseData* Current = nullptr; + const doctest::TestCaseData* Current = nullptr; + std::chrono::steady_clock::time_point RunStart = {}; + + struct FailedTestInfo + { + std::string Name; + std::string File; + unsigned Line; + }; + std::vector FailedTests; }; struct TestRunner::Impl -- cgit v1.2.3 From f796ee9e650d5f73844f862ed51a6de6bb33c219 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sat, 28 Feb 2026 15:36:50 +0100 Subject: subprocess tracking using Jobs on Windows/hub (#796) This change introduces job object support on Windows to be able to more accurately track and limit resource usage on storage instances created by the hub service. It also ensures that all child instances can be torn down reliably on exit. Also made it so hub tests no longer pop up console windows while running. --- src/zencore/include/zencore/process.h | 33 +++++++++++++ src/zencore/process.cpp | 89 +++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index c51163a68..809312c7b 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -9,6 +9,10 @@ namespace zen { +#if ZEN_PLATFORM_WINDOWS +class JobObject; +#endif + /** Basic process abstraction */ class ProcessHandle @@ -46,6 +50,7 @@ private: /** Basic process creation */ + struct CreateProcOptions { enum @@ -63,6 +68,9 @@ struct CreateProcOptions const std::filesystem::path* WorkingDirectory = nullptr; uint32_t Flags = 0; std::filesystem::path StdoutFile; +#if ZEN_PLATFORM_WINDOWS + JobObject* AssignToJob = nullptr; // When set, the process is created suspended, assigned to the job, then resumed +#endif }; #if ZEN_PLATFORM_WINDOWS @@ -99,6 +107,31 @@ private: std::vector m_ProcessHandles; }; +#if ZEN_PLATFORM_WINDOWS +/** Windows Job Object wrapper + * + * When configured with JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, the OS will + * terminate all assigned child processes when the job handle is closed + * (including abnormal termination of the owning process). This provides + * an OS-level guarantee against orphaned child processes. + */ +class JobObject +{ +public: + JobObject(); + ~JobObject(); + JobObject(const JobObject&) = delete; + JobObject& operator=(const JobObject&) = delete; + + void Initialize(); + bool AssignProcess(void* ProcessHandle); + [[nodiscard]] bool IsValid() const; + +private: + void* m_JobHandle = nullptr; +}; +#endif // ZEN_PLATFORM_WINDOWS + bool IsProcessRunning(int pid); bool IsProcessRunning(int pid, std::error_code& OutEc); int GetCurrentProcessId(); diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 4a2668912..226a94050 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -490,6 +490,8 @@ CreateProcNormal(const std::filesystem::path& Executable, std::string_view Comma LPSECURITY_ATTRIBUTES ProcessAttributes = nullptr; LPSECURITY_ATTRIBUTES ThreadAttributes = nullptr; + const bool AssignToJob = Options.AssignToJob && Options.AssignToJob->IsValid(); + DWORD CreationFlags = 0; if (Options.Flags & CreateProcOptions::Flag_NewConsole) { @@ -503,6 +505,10 @@ CreateProcNormal(const std::filesystem::path& Executable, std::string_view Comma { CreationFlags |= CREATE_NEW_PROCESS_GROUP; } + if (AssignToJob) + { + CreationFlags |= CREATE_SUSPENDED; + } const wchar_t* WorkingDir = nullptr; if (Options.WorkingDirectory != nullptr) @@ -571,6 +577,15 @@ CreateProcNormal(const std::filesystem::path& Executable, std::string_view Comma return nullptr; } + if (AssignToJob) + { + if (!Options.AssignToJob->AssignProcess(ProcessInfo.hProcess)) + { + ZEN_WARN("Failed to assign newly created process to job object"); + } + ResumeThread(ProcessInfo.hThread); + } + CloseHandle(ProcessInfo.hThread); return ProcessInfo.hProcess; } @@ -644,6 +659,8 @@ CreateProcUnelevated(const std::filesystem::path& Executable, std::string_view C }; PROCESS_INFORMATION ProcessInfo = {}; + const bool AssignToJob = Options.AssignToJob && Options.AssignToJob->IsValid(); + if (Options.Flags & CreateProcOptions::Flag_NewConsole) { CreateProcFlags |= CREATE_NEW_CONSOLE; @@ -652,6 +669,10 @@ CreateProcUnelevated(const std::filesystem::path& Executable, std::string_view C { CreateProcFlags |= CREATE_NO_WINDOW; } + if (AssignToJob) + { + CreateProcFlags |= CREATE_SUSPENDED; + } ExtendableWideStringBuilder<256> CommandLineZ; CommandLineZ << CommandLine; @@ -679,6 +700,15 @@ CreateProcUnelevated(const std::filesystem::path& Executable, std::string_view C return nullptr; } + if (AssignToJob) + { + if (!Options.AssignToJob->AssignProcess(ProcessInfo.hProcess)) + { + ZEN_WARN("Failed to assign newly created process to job object"); + } + ResumeThread(ProcessInfo.hThread); + } + CloseHandle(ProcessInfo.hThread); return ProcessInfo.hProcess; } @@ -845,6 +875,65 @@ ProcessMonitor::IsActive() const ////////////////////////////////////////////////////////////////////////// +#if ZEN_PLATFORM_WINDOWS +JobObject::JobObject() = default; + +JobObject::~JobObject() +{ + if (m_JobHandle) + { + CloseHandle(m_JobHandle); + m_JobHandle = nullptr; + } +} + +void +JobObject::Initialize() +{ + ZEN_ASSERT(m_JobHandle == nullptr, "JobObject already initialized"); + + m_JobHandle = CreateJobObjectW(nullptr, nullptr); + if (!m_JobHandle) + { + ZEN_WARN("Failed to create job object: {}", zen::GetLastError()); + return; + } + + JOBOBJECT_EXTENDED_LIMIT_INFORMATION LimitInfo = {}; + LimitInfo.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + + if (!SetInformationJobObject(m_JobHandle, JobObjectExtendedLimitInformation, &LimitInfo, sizeof(LimitInfo))) + { + ZEN_WARN("Failed to set job object limits: {}", zen::GetLastError()); + CloseHandle(m_JobHandle); + m_JobHandle = nullptr; + } +} + +bool +JobObject::AssignProcess(void* ProcessHandle) +{ + ZEN_ASSERT(m_JobHandle != nullptr, "JobObject not initialized"); + ZEN_ASSERT(ProcessHandle != nullptr, "ProcessHandle is null"); + + if (!AssignProcessToJobObject(m_JobHandle, ProcessHandle)) + { + ZEN_WARN("Failed to assign process to job object: {}", zen::GetLastError()); + return false; + } + + return true; +} + +bool +JobObject::IsValid() const +{ + return m_JobHandle != nullptr; +} +#endif // ZEN_PLATFORM_WINDOWS + +////////////////////////////////////////////////////////////////////////// + bool IsProcessRunning(int pid, std::error_code& OutEc) { -- cgit v1.2.3 From 4d01aaee0a45f4c9f96e8a4925eff696be98de8d Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 1 Mar 2026 12:40:20 +0100 Subject: added `--verbose` option to zenserver-test and `xmake test` (#798) * when `--verbose` is specified to zenserver-test, all child process output (typically, zenserver instances) is piped through to stdout. you can also pass `--verbose` to `xmake test` to accomplish the same thing. * this PR also consolidates all test runner `main` function logic (such as from zencore-test, zenhttp-test etc) into central implementation in zencore for consistency and ease of maintenance * also added extended utf8-tests including a fix to `Utf8ToWide()` --- src/zencore/include/zencore/testing.h | 2 + src/zencore/include/zencore/testutils.h | 27 +++++++ src/zencore/string.cpp | 131 +++++++++++++++++++++++++++++--- src/zencore/testing.cpp | 37 ++++++++- 4 files changed, 186 insertions(+), 11 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/testing.h b/src/zencore/include/zencore/testing.h index a00ee3166..43bdbbffe 100644 --- a/src/zencore/include/zencore/testing.h +++ b/src/zencore/include/zencore/testing.h @@ -59,6 +59,8 @@ private: return Runner.Run(); \ }() +int RunTestMain(int argc, char* argv[], const char* traceName, void (*forceLink)()); + } // namespace zen::testing #endif diff --git a/src/zencore/include/zencore/testutils.h b/src/zencore/include/zencore/testutils.h index e2a4f8346..2a789d18f 100644 --- a/src/zencore/include/zencore/testutils.h +++ b/src/zencore/include/zencore/testutils.h @@ -59,6 +59,33 @@ struct TrueType static const bool Enabled = true; }; +namespace utf8test { + + // 2-byte UTF-8 (Latin extended) + static constexpr const char kLatin[] = u8"café_résumé"; + static constexpr const wchar_t kLatinW[] = L"café_résumé"; + + // 2-byte UTF-8 (Cyrillic) + static constexpr const char kCyrillic[] = u8"данные"; + static constexpr const wchar_t kCyrillicW[] = L"данные"; + + // 3-byte UTF-8 (CJK) + static constexpr const char kCJK[] = u8"日本語"; + static constexpr const wchar_t kCJKW[] = L"日本語"; + + // Mixed scripts + static constexpr const char kMixed[] = u8"zen_éд日"; + static constexpr const wchar_t kMixedW[] = L"zen_éд日"; + + // 4-byte UTF-8 (supplementary plane) — string tests only, NOT filesystem + static constexpr const char kEmoji[] = u8"📦"; + static constexpr const wchar_t kEmojiW[] = L"📦"; + + // BMP-only test strings suitable for filesystem use + static constexpr const char* kFilenameSafe[] = {kLatin, kCyrillic, kCJK, kMixed}; + +} // namespace utf8test + } // namespace zen #endif // ZEN_WITH_TESTS diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp index a9aed6309..ab1c7de58 100644 --- a/src/zencore/string.cpp +++ b/src/zencore/string.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -184,7 +185,21 @@ Utf8ToWide(const std::u8string_view& Str8, WideStringBuilderBase& OutString) if (!ByteCount) { +#if ZEN_SIZEOF_WCHAR_T == 2 + if (CurrentOutChar > 0xFFFF) + { + // Supplementary plane: emit a UTF-16 surrogate pair + uint32_t Adjusted = uint32_t(CurrentOutChar - 0x10000); + OutString.Append(wchar_t(0xD800 + (Adjusted >> 10))); + OutString.Append(wchar_t(0xDC00 + (Adjusted & 0x3FF))); + } + else + { + OutString.Append(wchar_t(CurrentOutChar)); + } +#else OutString.Append(wchar_t(CurrentOutChar)); +#endif CurrentOutChar = 0; } } @@ -967,33 +982,131 @@ TEST_CASE("ExtendableWideStringBuilder") TEST_CASE("utf8") { + using namespace utf8test; + SUBCASE("utf8towide") { - // TODO: add more extensive testing here - this covers a very small space - WideStringBuilder<32> wout; Utf8ToWide(u8"abcdefghi", wout); CHECK(StringEquals(L"abcdefghi", wout.c_str())); wout.Reset(); + Utf8ToWide(u8"abc\xC3\xA4\xC3\xB6\xC3\xBC", wout); + CHECK(StringEquals(L"abc\u00E4\u00F6\u00FC", wout.c_str())); + + wout.Reset(); + Utf8ToWide(std::string_view(kLatin), wout); + CHECK(StringEquals(kLatinW, wout.c_str())); + + wout.Reset(); + Utf8ToWide(std::string_view(kCyrillic), wout); + CHECK(StringEquals(kCyrillicW, wout.c_str())); + + wout.Reset(); + Utf8ToWide(std::string_view(kCJK), wout); + CHECK(StringEquals(kCJKW, wout.c_str())); + + wout.Reset(); + Utf8ToWide(std::string_view(kMixed), wout); + CHECK(StringEquals(kMixedW, wout.c_str())); - Utf8ToWide(u8"abc���", wout); - CHECK(StringEquals(L"abc���", wout.c_str())); + wout.Reset(); + Utf8ToWide(std::string_view(kEmoji), wout); + CHECK(StringEquals(kEmojiW, wout.c_str())); } SUBCASE("widetoutf8") { - // TODO: add more extensive testing here - this covers a very small space - - StringBuilder<32> out; + StringBuilder<64> out; WideToUtf8(L"abcdefghi", out); CHECK(StringEquals("abcdefghi", out.c_str())); out.Reset(); + WideToUtf8(kLatinW, out); + CHECK(StringEquals(kLatin, out.c_str())); - WideToUtf8(L"abc���", out); - CHECK(StringEquals(u8"abc���", out.c_str())); + out.Reset(); + WideToUtf8(kCyrillicW, out); + CHECK(StringEquals(kCyrillic, out.c_str())); + + out.Reset(); + WideToUtf8(kCJKW, out); + CHECK(StringEquals(kCJK, out.c_str())); + + out.Reset(); + WideToUtf8(kMixedW, out); + CHECK(StringEquals(kMixed, out.c_str())); + + out.Reset(); + WideToUtf8(kEmojiW, out); + CHECK(StringEquals(kEmoji, out.c_str())); + } + + SUBCASE("roundtrip") + { + // UTF-8 -> Wide -> UTF-8 identity + const char* Utf8Strings[] = {kLatin, kCyrillic, kCJK, kMixed, kEmoji}; + for (const char* Utf8Str : Utf8Strings) + { + ExtendableWideStringBuilder<64> Wide; + Utf8ToWide(std::string_view(Utf8Str), Wide); + + ExtendableStringBuilder<64> Back; + WideToUtf8(std::wstring_view(Wide.c_str()), Back); + CHECK(StringEquals(Utf8Str, Back.c_str())); + } + + // Wide -> UTF-8 -> Wide identity + const wchar_t* WideStrings[] = {kLatinW, kCyrillicW, kCJKW, kMixedW, kEmojiW}; + for (const wchar_t* WideStr : WideStrings) + { + ExtendableStringBuilder<64> Utf8; + WideToUtf8(std::wstring_view(WideStr), Utf8); + + ExtendableWideStringBuilder<64> Back; + Utf8ToWide(std::string_view(Utf8.c_str()), Back); + CHECK(StringEquals(WideStr, Back.c_str())); + } + + // Empty string round-trip + { + ExtendableWideStringBuilder<8> Wide; + Utf8ToWide(std::string_view(""), Wide); + CHECK(Wide.Size() == 0); + + ExtendableStringBuilder<8> Narrow; + WideToUtf8(std::wstring_view(L""), Narrow); + CHECK(Narrow.Size() == 0); + } + } + + SUBCASE("IsValidUtf8") + { + // Valid inputs + CHECK(IsValidUtf8("")); + CHECK(IsValidUtf8("hello world")); + CHECK(IsValidUtf8(kLatin)); + CHECK(IsValidUtf8(kCyrillic)); + CHECK(IsValidUtf8(kCJK)); + CHECK(IsValidUtf8(kMixed)); + CHECK(IsValidUtf8(kEmoji)); + + // Invalid: truncated 2-byte sequence + CHECK(!IsValidUtf8(std::string_view("\xC3", 1))); + + // Invalid: truncated 3-byte sequence + CHECK(!IsValidUtf8(std::string_view("\xE6\x97", 2))); + + // Invalid: truncated 4-byte sequence + CHECK(!IsValidUtf8(std::string_view("\xF0\x9F\x93", 3))); + + // Invalid: bad start byte + CHECK(!IsValidUtf8(std::string_view("\xFF", 1))); + CHECK(!IsValidUtf8(std::string_view("\xFE", 1))); + + // Invalid: overlong encoding of '/' (U+002F) + CHECK(!IsValidUtf8(std::string_view("\xC0\xAF", 2))); } } diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp index ef8fb0480..6000bd95c 100644 --- a/src/zencore/testing.cpp +++ b/src/zencore/testing.cpp @@ -1,18 +1,23 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#define ZEN_TEST_WITH_RUNNER 1 + #include "zencore/testing.h" + +#include "zencore/filesystem.h" #include "zencore/logging.h" +#include "zencore/process.h" +#include "zencore/trace.h" #if ZEN_WITH_TESTS # include +# include # include # include # include # include -# include - namespace zen::testing { using namespace std::literals; @@ -149,6 +154,34 @@ TestRunner::Run() return m_Impl->Session.run(); } +int +RunTestMain(int argc, char* argv[], [[maybe_unused]] const char* traceName, void (*forceLink)()) +{ +# if ZEN_PLATFORM_WINDOWS + setlocale(LC_ALL, "en_us.UTF8"); +# endif + + forceLink(); + +# if ZEN_PLATFORM_LINUX + zen::IgnoreChildSignals(); +# endif + +# if ZEN_WITH_TRACE + zen::TraceInit(traceName); + zen::TraceOptions TraceCommandlineOptions; + if (GetTraceOptionsFromCommandline(TraceCommandlineOptions)) + { + TraceConfigure(TraceCommandlineOptions); + } +# endif + + zen::logging::InitializeLogging(); + zen::MaximizeOpenFileCount(); + + return ZEN_RUN_TESTS(argc, argv); +} + } // namespace zen::testing #endif // ZEN_WITH_TESTS -- cgit v1.2.3 From d604351cb5dc3032a7cb8c84d6ad5f1480325e5c Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 2 Mar 2026 09:37:14 +0100 Subject: Add test suites (#799) Makes all test cases part of a test suite. Test suites are named after the module and the name of the file containing the implementation of the test. * This allows for better and more predictable filtering of which test cases to run which should also be able to reduce the time CI spends in tests since it can filter on the tests for that particular module. Also improves `xmake test` behaviour: * instead of an explicit list of projects just enumerate the test projects which are available based on build system state * also introduces logic to avoid running `xmake config` unnecessarily which would invalidate the existing build and do lots of unnecessary work since dependencies were invalidated by the updated config * also invokes build only for the chosen test targets As a bonus, also adds `xmake sln --open` which allows opening IDE after generation of solution/xmake project is done. --- src/zencore/base64.cpp | 4 +++ src/zencore/basicfile.cpp | 4 +++ src/zencore/blake3.cpp | 4 +++ src/zencore/callstack.cpp | 4 +++ src/zencore/compactbinary.cpp | 8 ++---- src/zencore/compactbinarybuilder.cpp | 4 +++ src/zencore/compactbinaryjson.cpp | 4 +++ src/zencore/compactbinarypackage.cpp | 4 +++ src/zencore/compactbinaryvalidation.cpp | 4 +++ src/zencore/compactbinaryyaml.cpp | 4 +++ src/zencore/compositebuffer.cpp | 5 ++++ src/zencore/compress.cpp | 4 +++ src/zencore/crypto.cpp | 4 +++ src/zencore/filesystem.cpp | 4 +++ src/zencore/include/zencore/testing.h | 7 ++--- src/zencore/intmath.cpp | 4 +++ src/zencore/iobuffer.cpp | 4 +++ src/zencore/jobqueue.cpp | 4 +++ src/zencore/logging.cpp | 4 +++ src/zencore/md5.cpp | 4 +++ src/zencore/memoryview.cpp | 4 +++ src/zencore/mpscqueue.cpp | 2 ++ src/zencore/parallelwork.cpp | 4 +++ src/zencore/refcount.cpp | 4 +++ src/zencore/sha1.cpp | 4 +++ src/zencore/sharedbuffer.cpp | 4 +++ src/zencore/stream.cpp | 4 +++ src/zencore/string.cpp | 4 +++ src/zencore/testing.cpp | 47 ++++++++++++++++++++++++++------- src/zencore/uid.cpp | 4 +++ src/zencore/workthreadpool.cpp | 4 +++ src/zencore/zencore.cpp | 2 +- 32 files changed, 156 insertions(+), 19 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/base64.cpp b/src/zencore/base64.cpp index fdf5f2d66..96e121799 100644 --- a/src/zencore/base64.cpp +++ b/src/zencore/base64.cpp @@ -180,6 +180,8 @@ template bool Base64::Decode(const wchar_t* Source, uint32_t Length, ui using namespace std::string_literals; +TEST_SUITE_BEGIN("core.base64"); + TEST_CASE("Base64") { auto EncodeString = [](std::string_view Input) -> std::string { @@ -290,6 +292,8 @@ TEST_CASE("Base64") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index bd4d119fb..9dcf7663a 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -888,6 +888,8 @@ WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.basicfile"); + TEST_CASE("BasicFile") { ScopedCurrentDirectoryChange _; @@ -1081,6 +1083,8 @@ TEST_CASE("BasicFileBuffer") } } +TEST_SUITE_END(); + void basicfile_forcelink() { diff --git a/src/zencore/blake3.cpp b/src/zencore/blake3.cpp index 054f0d3a0..123918de5 100644 --- a/src/zencore/blake3.cpp +++ b/src/zencore/blake3.cpp @@ -200,6 +200,8 @@ BLAKE3Stream::GetHash() // return text; // } +TEST_SUITE_BEGIN("core.blake3"); + TEST_CASE("BLAKE3") { SUBCASE("Basics") @@ -237,6 +239,8 @@ TEST_CASE("BLAKE3") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/callstack.cpp b/src/zencore/callstack.cpp index 8aa1111bf..ee0b0625a 100644 --- a/src/zencore/callstack.cpp +++ b/src/zencore/callstack.cpp @@ -260,6 +260,8 @@ GetCallstackRaw(void* CaptureBuffer, int FramesToSkip, int FramesToCapture) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.callstack"); + TEST_CASE("Callstack.Basic") { void* Addresses[4]; @@ -272,6 +274,8 @@ TEST_CASE("Callstack.Basic") } } +TEST_SUITE_END(); + void callstack_forcelink() { diff --git a/src/zencore/compactbinary.cpp b/src/zencore/compactbinary.cpp index b43cc18f1..9c81305d0 100644 --- a/src/zencore/compactbinary.cpp +++ b/src/zencore/compactbinary.cpp @@ -1512,6 +1512,8 @@ uson_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinary"); + TEST_CASE("guid") { using namespace std::literals; @@ -1704,8 +1706,6 @@ TEST_CASE("uson.datetime") ////////////////////////////////////////////////////////////////////////// -TEST_SUITE_BEGIN("core.datetime"); - TEST_CASE("core.datetime.compare") { DateTime T1(2000, 12, 13); @@ -1732,10 +1732,6 @@ TEST_CASE("core.datetime.add") CHECK(dT + T1 - T2 == dT1); } -TEST_SUITE_END(); - -TEST_SUITE_BEGIN("core.timespan"); - TEST_CASE("core.timespan.compare") { TimeSpan T1(1000); diff --git a/src/zencore/compactbinarybuilder.cpp b/src/zencore/compactbinarybuilder.cpp index 63c0b9c5c..a9ba30750 100644 --- a/src/zencore/compactbinarybuilder.cpp +++ b/src/zencore/compactbinarybuilder.cpp @@ -710,6 +710,8 @@ usonbuilder_forcelink() // return ""; // } +TEST_SUITE_BEGIN("core.compactbinarybuilder"); + TEST_CASE("usonbuilder.object") { using namespace std::literals; @@ -1530,6 +1532,8 @@ TEST_CASE("usonbuilder.stream") CHECK(ValidateCompactBinary(Object.GetBuffer(), CbValidateMode::All) == CbValidateError::None); } } + +TEST_SUITE_END(); #endif } // namespace zen diff --git a/src/zencore/compactbinaryjson.cpp b/src/zencore/compactbinaryjson.cpp index abbec360a..da560a449 100644 --- a/src/zencore/compactbinaryjson.cpp +++ b/src/zencore/compactbinaryjson.cpp @@ -654,6 +654,8 @@ cbjson_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinaryjson"); + TEST_CASE("uson.json") { using namespace std::literals; @@ -872,6 +874,8 @@ TEST_CASE("json.uson") } } +TEST_SUITE_END(); + #endif // ZEN_WITH_TESTS } // namespace zen diff --git a/src/zencore/compactbinarypackage.cpp b/src/zencore/compactbinarypackage.cpp index ffe64f2e9..56a292ca6 100644 --- a/src/zencore/compactbinarypackage.cpp +++ b/src/zencore/compactbinarypackage.cpp @@ -805,6 +805,8 @@ usonpackage_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinarypackage"); + TEST_CASE("usonpackage") { using namespace std::literals; @@ -1343,6 +1345,8 @@ TEST_CASE("usonpackage.invalidpackage") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp index d7292f405..3e78f8ef1 100644 --- a/src/zencore/compactbinaryvalidation.cpp +++ b/src/zencore/compactbinaryvalidation.cpp @@ -753,10 +753,14 @@ usonvalidation_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinaryvalidation"); + TEST_CASE("usonvalidation") { SUBCASE("Basic") {} } + +TEST_SUITE_END(); #endif } // namespace zen diff --git a/src/zencore/compactbinaryyaml.cpp b/src/zencore/compactbinaryyaml.cpp index b308af418..b7f2c55df 100644 --- a/src/zencore/compactbinaryyaml.cpp +++ b/src/zencore/compactbinaryyaml.cpp @@ -412,6 +412,8 @@ cbyaml_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinaryyaml"); + TEST_CASE("uson.yaml") { using namespace std::literals; @@ -524,6 +526,8 @@ mixed_seq: )"sv); } } + +TEST_SUITE_END(); #endif } // namespace zen diff --git a/src/zencore/compositebuffer.cpp b/src/zencore/compositebuffer.cpp index 252ac9045..ed2b16384 100644 --- a/src/zencore/compositebuffer.cpp +++ b/src/zencore/compositebuffer.cpp @@ -297,6 +297,9 @@ CompositeBuffer::IterateRange(uint64_t Offset, } #if ZEN_WITH_TESTS + +TEST_SUITE_BEGIN("core.compositebuffer"); + TEST_CASE("CompositeBuffer Null") { CompositeBuffer Buffer; @@ -462,6 +465,8 @@ TEST_CASE("CompositeBuffer Composite") TestIterateRange(8, 0, MakeMemoryView(FlatArray).Mid(8, 0), FlatView2); } +TEST_SUITE_END(); + void compositebuffer_forcelink() { diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp index 25ed0fc46..6aa0adce0 100644 --- a/src/zencore/compress.cpp +++ b/src/zencore/compress.cpp @@ -2420,6 +2420,8 @@ private: #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.compress"); + TEST_CASE("CompressedBuffer") { uint8_t Zeroes[1024]{}; @@ -2967,6 +2969,8 @@ TEST_CASE("CompressedBufferReader") } } +TEST_SUITE_END(); + void compress_forcelink() { diff --git a/src/zencore/crypto.cpp b/src/zencore/crypto.cpp index 09eebb6ae..049854b42 100644 --- a/src/zencore/crypto.cpp +++ b/src/zencore/crypto.cpp @@ -449,6 +449,8 @@ crypto_forcelink() { } +TEST_SUITE_BEGIN("core.crypto"); + TEST_CASE("crypto.bits") { using CryptoBits256Bit = CryptoBits<256>; @@ -500,6 +502,8 @@ TEST_CASE("crypto.aes") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 03398860b..9885b2ada 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -3309,6 +3309,8 @@ filesystem_forcelink() { } +TEST_SUITE_BEGIN("core.filesystem"); + TEST_CASE("filesystem") { using namespace std::filesystem; @@ -3603,6 +3605,8 @@ TEST_CASE("SharedMemory") CHECK(!OpenSharedMemory("SharedMemoryTest0", 482, false)); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/include/zencore/testing.h b/src/zencore/include/zencore/testing.h index 43bdbbffe..8410216c4 100644 --- a/src/zencore/include/zencore/testing.h +++ b/src/zencore/include/zencore/testing.h @@ -43,8 +43,9 @@ public: TestRunner(); ~TestRunner(); - int ApplyCommandLine(int argc, char const* const* argv); - int Run(); + void SetDefaultSuiteFilter(const char* Pattern); + int ApplyCommandLine(int Argc, char const* const* Argv); + int Run(); private: struct Impl; @@ -59,7 +60,7 @@ private: return Runner.Run(); \ }() -int RunTestMain(int argc, char* argv[], const char* traceName, void (*forceLink)()); +int RunTestMain(int Argc, char* Argv[], const char* ExecutableName, void (*ForceLink)()); } // namespace zen::testing #endif diff --git a/src/zencore/intmath.cpp b/src/zencore/intmath.cpp index 32f82b486..fedf76edc 100644 --- a/src/zencore/intmath.cpp +++ b/src/zencore/intmath.cpp @@ -19,6 +19,8 @@ intmath_forcelink() { } +TEST_SUITE_BEGIN("core.intmath"); + TEST_CASE("intmath") { CHECK(FloorLog2(0x00) == 0); @@ -66,6 +68,8 @@ TEST_CASE("intmath") CHECK(ByteSwap(uint64_t(0x214d'6172'7469'6e21ull)) == 0x216e'6974'7261'4d21ull); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp index 1c31d6620..c47c54981 100644 --- a/src/zencore/iobuffer.cpp +++ b/src/zencore/iobuffer.cpp @@ -719,6 +719,8 @@ iobuffer_forcelink() { } +TEST_SUITE_BEGIN("core.iobuffer"); + TEST_CASE("IoBuffer") { zen::IoBuffer buffer1; @@ -756,6 +758,8 @@ TEST_CASE("IoBuffer.mmap") # endif } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/jobqueue.cpp b/src/zencore/jobqueue.cpp index 75c1be42b..35724b07a 100644 --- a/src/zencore/jobqueue.cpp +++ b/src/zencore/jobqueue.cpp @@ -460,6 +460,8 @@ jobqueue_forcelink() { } +TEST_SUITE_BEGIN("core.jobqueue"); + TEST_CASE("JobQueue") { std::unique_ptr Queue(MakeJobQueue(2, "queue")); @@ -580,6 +582,8 @@ TEST_CASE("JobQueue") } JobsLatch.Wait(); } + +TEST_SUITE_END(); #endif } // namespace zen diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp index e79c4b41c..e960a2729 100644 --- a/src/zencore/logging.cpp +++ b/src/zencore/logging.cpp @@ -540,6 +540,8 @@ logging_forcelink() using namespace std::literals; +TEST_SUITE_BEGIN("core.logging"); + TEST_CASE("simple.bread") { ExtendableStringBuilder<256> Crumbs; @@ -588,6 +590,8 @@ TEST_CASE("simple.bread") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/md5.cpp b/src/zencore/md5.cpp index 4ec145697..3baee91c2 100644 --- a/src/zencore/md5.cpp +++ b/src/zencore/md5.cpp @@ -437,6 +437,8 @@ md5_forcelink() // return md5text; // } +TEST_SUITE_BEGIN("core.md5"); + TEST_CASE("MD5") { using namespace std::literals; @@ -458,6 +460,8 @@ TEST_CASE("MD5") CHECK(Output.compare(Buffer)); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/memoryview.cpp b/src/zencore/memoryview.cpp index 1f6a6996c..1654b1766 100644 --- a/src/zencore/memoryview.cpp +++ b/src/zencore/memoryview.cpp @@ -18,6 +18,8 @@ namespace zen { #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.memoryview"); + TEST_CASE("MemoryView") { { @@ -35,6 +37,8 @@ TEST_CASE("MemoryView") CHECK(MakeMemoryView({1.0f, 1.2f}).GetSize() == 8); } +TEST_SUITE_END(); + void memory_forcelink() { diff --git a/src/zencore/mpscqueue.cpp b/src/zencore/mpscqueue.cpp index 29c76c3ca..f749f1c90 100644 --- a/src/zencore/mpscqueue.cpp +++ b/src/zencore/mpscqueue.cpp @@ -8,6 +8,7 @@ namespace zen { #if ZEN_WITH_TESTS && 0 +TEST_SUITE_BEGIN("core.mpscqueue"); TEST_CASE("mpsc") { MpscQueue Queue; @@ -15,6 +16,7 @@ TEST_CASE("mpsc") std::optional Value = Queue.Dequeue(); CHECK_EQ(Value, "hello"); } +TEST_SUITE_END(); #endif void diff --git a/src/zencore/parallelwork.cpp b/src/zencore/parallelwork.cpp index d86d5815f..94696f479 100644 --- a/src/zencore/parallelwork.cpp +++ b/src/zencore/parallelwork.cpp @@ -157,6 +157,8 @@ ParallelWork::RethrowErrors() #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.parallelwork"); + TEST_CASE("parallellwork.nowork") { std::atomic AbortFlag; @@ -255,6 +257,8 @@ TEST_CASE("parallellwork.limitqueue") Work.Wait(); } +TEST_SUITE_END(); + void parallellwork_forcelink() { diff --git a/src/zencore/refcount.cpp b/src/zencore/refcount.cpp index a6a86ee12..f19afe715 100644 --- a/src/zencore/refcount.cpp +++ b/src/zencore/refcount.cpp @@ -33,6 +33,8 @@ refcount_forcelink() { } +TEST_SUITE_BEGIN("core.refcount"); + TEST_CASE("RefPtr") { RefPtr Ref; @@ -60,6 +62,8 @@ TEST_CASE("RefPtr") CHECK(IsDestroyed == true); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/sha1.cpp b/src/zencore/sha1.cpp index 3ee74d7d8..807ae4c30 100644 --- a/src/zencore/sha1.cpp +++ b/src/zencore/sha1.cpp @@ -373,6 +373,8 @@ sha1_forcelink() // return sha1text; // } +TEST_SUITE_BEGIN("core.sha1"); + TEST_CASE("SHA1") { uint8_t sha1_empty[20] = {0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, @@ -438,6 +440,8 @@ TEST_CASE("SHA1") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/sharedbuffer.cpp b/src/zencore/sharedbuffer.cpp index 78efb9d42..8dc6d49d8 100644 --- a/src/zencore/sharedbuffer.cpp +++ b/src/zencore/sharedbuffer.cpp @@ -152,10 +152,14 @@ sharedbuffer_forcelink() { } +TEST_SUITE_BEGIN("core.sharedbuffer"); + TEST_CASE("SharedBuffer") { } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/stream.cpp b/src/zencore/stream.cpp index a800ce121..de67303a4 100644 --- a/src/zencore/stream.cpp +++ b/src/zencore/stream.cpp @@ -79,6 +79,8 @@ BufferReader::Serialize(void* V, int64_t Length) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.stream"); + TEST_CASE("binary.writer.span") { BinaryWriter Writer; @@ -91,6 +93,8 @@ TEST_CASE("binary.writer.span") CHECK(memcmp(Result.GetData(), "apa banan", 9) == 0); } +TEST_SUITE_END(); + void stream_forcelink() { diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp index ab1c7de58..27635a86c 100644 --- a/src/zencore/string.cpp +++ b/src/zencore/string.cpp @@ -546,6 +546,8 @@ UrlDecode(std::string_view InUrl) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.string"); + TEST_CASE("url") { using namespace std::literals; @@ -1222,6 +1224,8 @@ TEST_CASE("string") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp index 6000bd95c..0bae139bd 100644 --- a/src/zencore/testing.cpp +++ b/src/zencore/testing.cpp @@ -128,18 +128,24 @@ TestRunner::~TestRunner() { } +void +TestRunner::SetDefaultSuiteFilter(const char* Pattern) +{ + m_Impl->Session.setOption("test-suite", Pattern); +} + int -TestRunner::ApplyCommandLine(int argc, char const* const* argv) +TestRunner::ApplyCommandLine(int Argc, char const* const* Argv) { - m_Impl->Session.applyCommandLine(argc, argv); + m_Impl->Session.applyCommandLine(Argc, Argv); - for (int i = 1; i < argc; ++i) + for (int i = 1; i < Argc; ++i) { - if (argv[i] == "--debug"sv) + if (Argv[i] == "--debug"sv) { zen::logging::SetLogLevel(zen::logging::level::Debug); } - else if (argv[i] == "--verbose"sv) + else if (Argv[i] == "--verbose"sv) { zen::logging::SetLogLevel(zen::logging::level::Trace); } @@ -155,20 +161,20 @@ TestRunner::Run() } int -RunTestMain(int argc, char* argv[], [[maybe_unused]] const char* traceName, void (*forceLink)()) +RunTestMain(int Argc, char* Argv[], const char* ExecutableName, void (*ForceLink)()) { # if ZEN_PLATFORM_WINDOWS setlocale(LC_ALL, "en_us.UTF8"); # endif - forceLink(); + ForceLink(); # if ZEN_PLATFORM_LINUX zen::IgnoreChildSignals(); # endif # if ZEN_WITH_TRACE - zen::TraceInit(traceName); + zen::TraceInit(ExecutableName); zen::TraceOptions TraceCommandlineOptions; if (GetTraceOptionsFromCommandline(TraceCommandlineOptions)) { @@ -179,7 +185,30 @@ RunTestMain(int argc, char* argv[], [[maybe_unused]] const char* traceName, void zen::logging::InitializeLogging(); zen::MaximizeOpenFileCount(); - return ZEN_RUN_TESTS(argc, argv); + TestRunner Runner; + + // Derive default suite filter from ExecutableName: "zencore-test" -> "core.*" + if (ExecutableName) + { + std::string_view Name = ExecutableName; + if (Name.starts_with("zen")) + { + Name.remove_prefix(3); + } + if (Name.ends_with("-test")) + { + Name.remove_suffix(5); + } + if (!Name.empty()) + { + std::string Filter(Name); + Filter += ".*"; + Runner.SetDefaultSuiteFilter(Filter.c_str()); + } + } + + Runner.ApplyCommandLine(Argc, Argv); + return Runner.Run(); } } // namespace zen::testing diff --git a/src/zencore/uid.cpp b/src/zencore/uid.cpp index d7636f2ad..971683721 100644 --- a/src/zencore/uid.cpp +++ b/src/zencore/uid.cpp @@ -156,6 +156,8 @@ Oid::FromMemory(const void* Ptr) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.uid"); + TEST_CASE("Oid") { SUBCASE("Basic") @@ -185,6 +187,8 @@ TEST_CASE("Oid") } } +TEST_SUITE_END(); + void uid_forcelink() { diff --git a/src/zencore/workthreadpool.cpp b/src/zencore/workthreadpool.cpp index cb84bbe06..1cb338c66 100644 --- a/src/zencore/workthreadpool.cpp +++ b/src/zencore/workthreadpool.cpp @@ -354,6 +354,8 @@ workthreadpool_forcelink() using namespace std::literals; +TEST_SUITE_BEGIN("core.workthreadpool"); + TEST_CASE("threadpool.basic") { WorkerThreadPool Threadpool{1}; @@ -368,6 +370,8 @@ TEST_CASE("threadpool.basic") CHECK_THROWS(FutureThrow.get()); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/zencore.cpp b/src/zencore/zencore.cpp index 4ff79edc7..d82474705 100644 --- a/src/zencore/zencore.cpp +++ b/src/zencore/zencore.cpp @@ -285,7 +285,7 @@ zencore_forcelinktests() namespace zen { -TEST_SUITE_BEGIN("core.assert"); +TEST_SUITE_BEGIN("core.zencore"); TEST_CASE("Assert.Default") { -- cgit v1.2.3 From b67dac7c093cc82b7e8f12f9eb57bfa34dfe26d8 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 4 Mar 2026 08:35:32 +0100 Subject: unity build fixes (#802) Various fixes to make cpp files build in unity build mode as an aside using Unity build doesn't really seem to work on Linux, unsure why but it leads to link-time issues --- src/zencore/include/zencore/compactbinaryfile.h | 1 + src/zencore/include/zencore/meta.h | 1 + src/zencore/include/zencore/varint.h | 1 + src/zencore/md5.cpp | 19 ++++++++++++++++++- src/zencore/xmake.lua | 1 + 5 files changed, 22 insertions(+), 1 deletion(-) (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/compactbinaryfile.h b/src/zencore/include/zencore/compactbinaryfile.h index 00c37e941..33f3e7bea 100644 --- a/src/zencore/include/zencore/compactbinaryfile.h +++ b/src/zencore/include/zencore/compactbinaryfile.h @@ -1,4 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#pragma once #include #include diff --git a/src/zencore/include/zencore/meta.h b/src/zencore/include/zencore/meta.h index 82eb5cc30..20ec4ac6f 100644 --- a/src/zencore/include/zencore/meta.h +++ b/src/zencore/include/zencore/meta.h @@ -1,4 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#pragma once /* This file contains utility functions for meta programming * diff --git a/src/zencore/include/zencore/varint.h b/src/zencore/include/zencore/varint.h index 9fe905f25..43ca14d38 100644 --- a/src/zencore/include/zencore/varint.h +++ b/src/zencore/include/zencore/varint.h @@ -1,4 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#pragma once #include "intmath.h" diff --git a/src/zencore/md5.cpp b/src/zencore/md5.cpp index 3baee91c2..83ed53fc8 100644 --- a/src/zencore/md5.cpp +++ b/src/zencore/md5.cpp @@ -342,6 +342,23 @@ Transform(uint32_t* buf, uint32_t* in) #undef G #undef H #undef I +#undef ROTATE_LEFT +#undef S11 +#undef S12 +#undef S13 +#undef S14 +#undef S21 +#undef S22 +#undef S23 +#undef S24 +#undef S31 +#undef S32 +#undef S33 +#undef S34 +#undef S41 +#undef S42 +#undef S43 +#undef S44 namespace zen { @@ -391,7 +408,7 @@ MD5::FromHexString(const char* string) { MD5 md5; - ParseHexBytes(string, 40, md5.Hash); + ParseHexBytes(string, 2 * sizeof md5.Hash, md5.Hash); return md5; } diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index 9a67175a0..2f81b7ec8 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -15,6 +15,7 @@ target('zencore') set_configdir("include/zencore") add_files("**.cpp") add_files("trace.cpp", {unity_ignored = true }) + add_files("testing.cpp", {unity_ignored = true }) if has_config("zenrpmalloc") then add_deps("rpmalloc") -- cgit v1.2.3 From eafd4d78378c1a642445ed127fdbe51ac559d4e3 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 4 Mar 2026 09:40:49 +0100 Subject: HTTP improvements (#803) - Add GetTotalBytesReceived/GetTotalBytesSent to HttpServer with implementations in ASIO and http.sys backends - Add ExpectedErrorCodes to HttpClientSettings to suppress warn/info logs for anticipated HTTP error codes - Also fixes minor issues in `CprHttpClient::Download` --- src/zencore/process.cpp | 14 ++++++++++++++ src/zencore/windows.cpp | 12 +++++------- 2 files changed, 19 insertions(+), 7 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 226a94050..f657869dc 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -745,6 +746,8 @@ CreateProcElevated(const std::filesystem::path& Executable, std::string_view Com CreateProcResult CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options) { + ZEN_TRACE_CPU("CreateProc"); + #if ZEN_PLATFORM_WINDOWS if (Options.Flags & CreateProcOptions::Flag_Unelevated) { @@ -776,6 +779,17 @@ CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine ZEN_UNUSED(Result); } + if (!Options.StdoutFile.empty()) + { + int Fd = open(Options.StdoutFile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (Fd >= 0) + { + dup2(Fd, STDOUT_FILENO); + dup2(Fd, STDERR_FILENO); + close(Fd); + } + } + if (execv(Executable.c_str(), ArgV.data()) < 0) { ThrowLastError("Failed to exec() a new process image"); diff --git a/src/zencore/windows.cpp b/src/zencore/windows.cpp index d02fcd35e..87f854b90 100644 --- a/src/zencore/windows.cpp +++ b/src/zencore/windows.cpp @@ -12,14 +12,12 @@ namespace zen::windows { bool IsRunningOnWine() { - HMODULE NtDll = GetModuleHandleA("ntdll.dll"); + static bool s_Result = [] { + HMODULE NtDll = GetModuleHandleA("ntdll.dll"); + return NtDll && !!GetProcAddress(NtDll, "wine_get_version"); + }(); - if (NtDll) - { - return !!GetProcAddress(NtDll, "wine_get_version"); - } - - return false; + return s_Result; } FileMapping::FileMapping(_In_ FileMapping& orig) -- cgit v1.2.3 From 0763d09a81e5a1d3df11763a7ec75e7860c9510a Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 4 Mar 2026 14:13:46 +0100 Subject: compute orchestration (#763) - Added local process runners for Linux/Wine, Mac with some sandboxing support - Horde & Nomad provisioning for development and testing - Client session queues with lifecycle management (active/draining/cancelled), automatic retry with configurable limits, and manual reschedule API - Improved web UI for orchestrator, compute, and hub dashboards with WebSocket push updates - Some security hardening - Improved scalability and `zen exec` command Still experimental - compute support is disabled by default --- src/zencore/include/zencore/system.h | 36 +++- src/zencore/system.cpp | 336 +++++++++++++++++++++++++++-------- 2 files changed, 292 insertions(+), 80 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/system.h b/src/zencore/include/zencore/system.h index bf3c15d3d..fecbe2dbe 100644 --- a/src/zencore/include/zencore/system.h +++ b/src/zencore/include/zencore/system.h @@ -4,6 +4,8 @@ #include +#include +#include #include namespace zen { @@ -12,6 +14,7 @@ class CbWriter; std::string GetMachineName(); std::string_view GetOperatingSystemName(); +std::string_view GetRuntimePlatformName(); // "windows", "wine", "linux", or "macos" std::string_view GetCpuName(); struct SystemMetrics @@ -25,7 +28,13 @@ struct SystemMetrics uint64_t AvailVirtualMemoryMiB = 0; uint64_t PageFileMiB = 0; uint64_t AvailPageFileMiB = 0; - float CpuUsagePercent = 0.0f; +}; + +/// Extended metrics that include CPU usage percentage, which requires +/// stateful delta tracking via SystemMetricsTracker. +struct ExtendedSystemMetrics : SystemMetrics +{ + float CpuUsagePercent = 0.0f; }; SystemMetrics GetSystemMetrics(); @@ -33,6 +42,31 @@ SystemMetrics GetSystemMetrics(); void SetCpuCountForReporting(int FakeCpuCount); SystemMetrics GetSystemMetricsForReporting(); +ExtendedSystemMetrics ApplyReportingOverrides(ExtendedSystemMetrics Metrics); + void Describe(const SystemMetrics& Metrics, CbWriter& Writer); +void Describe(const ExtendedSystemMetrics& Metrics, CbWriter& Writer); + +/// Stateful tracker that computes CPU usage as a delta between consecutive +/// Query() calls. The first call returns CpuUsagePercent = 0 (no previous +/// sample). Thread-safe: concurrent calls are serialised internally. +/// CPU sampling is rate-limited to MinInterval (default 1 s); calls that +/// arrive sooner return the previously cached value. +class SystemMetricsTracker +{ +public: + explicit SystemMetricsTracker(std::chrono::milliseconds MinInterval = std::chrono::seconds(1)); + ~SystemMetricsTracker(); + + SystemMetricsTracker(const SystemMetricsTracker&) = delete; + SystemMetricsTracker& operator=(const SystemMetricsTracker&) = delete; + + /// Collect current metrics. CPU usage is computed as delta since last Query(). + ExtendedSystemMetrics Query(); + +private: + struct Impl; + std::unique_ptr m_Impl; +}; } // namespace zen diff --git a/src/zencore/system.cpp b/src/zencore/system.cpp index 267c87e12..833d3c04b 100644 --- a/src/zencore/system.cpp +++ b/src/zencore/system.cpp @@ -7,6 +7,8 @@ #include #include +#include + #if ZEN_PLATFORM_WINDOWS # include @@ -133,33 +135,6 @@ GetSystemMetrics() Metrics.AvailPageFileMiB = MemStatus.ullAvailPageFile / 1024 / 1024; } - // Query CPU usage using PDH - // - // TODO: This should be changed to not require a Sleep, perhaps by using some - // background metrics gathering mechanism. - - { - PDH_HQUERY QueryHandle = nullptr; - PDH_HCOUNTER CounterHandle = nullptr; - - if (PdhOpenQueryW(nullptr, 0, &QueryHandle) == ERROR_SUCCESS) - { - if (PdhAddEnglishCounterW(QueryHandle, L"\\Processor(_Total)\\% Processor Time", 0, &CounterHandle) == ERROR_SUCCESS) - { - PdhCollectQueryData(QueryHandle); - Sleep(100); - PdhCollectQueryData(QueryHandle); - - PDH_FMT_COUNTERVALUE CounterValue; - if (PdhGetFormattedCounterValue(CounterHandle, PDH_FMT_DOUBLE, nullptr, &CounterValue) == ERROR_SUCCESS) - { - Metrics.CpuUsagePercent = static_cast(CounterValue.doubleValue); - } - } - PdhCloseQuery(QueryHandle); - } - } - return Metrics; } #elif ZEN_PLATFORM_LINUX @@ -235,39 +210,6 @@ GetSystemMetrics() } } - // Query CPU usage - Metrics.CpuUsagePercent = 0.0f; - if (FILE* Stat = fopen("/proc/stat", "r")) - { - char Line[256]; - unsigned long User, Nice, System, Idle, IoWait, Irq, SoftIrq; - static unsigned long PrevUser = 0, PrevNice = 0, PrevSystem = 0, PrevIdle = 0, PrevIoWait = 0, PrevIrq = 0, PrevSoftIrq = 0; - - if (fgets(Line, sizeof(Line), Stat)) - { - if (sscanf(Line, "cpu %lu %lu %lu %lu %lu %lu %lu", &User, &Nice, &System, &Idle, &IoWait, &Irq, &SoftIrq) == 7) - { - unsigned long TotalDelta = (User + Nice + System + Idle + IoWait + Irq + SoftIrq) - - (PrevUser + PrevNice + PrevSystem + PrevIdle + PrevIoWait + PrevIrq + PrevSoftIrq); - unsigned long IdleDelta = Idle - PrevIdle; - - if (TotalDelta > 0) - { - Metrics.CpuUsagePercent = 100.0f * (TotalDelta - IdleDelta) / TotalDelta; - } - - PrevUser = User; - PrevNice = Nice; - PrevSystem = System; - PrevIdle = Idle; - PrevIoWait = IoWait; - PrevIrq = Irq; - PrevSoftIrq = SoftIrq; - } - } - fclose(Stat); - } - // Get memory information long Pages = sysconf(_SC_PHYS_PAGES); long PageSize = sysconf(_SC_PAGE_SIZE); @@ -348,25 +290,6 @@ GetSystemMetrics() sysctlbyname("hw.packages", &Packages, &Size, nullptr, 0); Metrics.CpuCount = Packages > 0 ? Packages : 1; - // Query CPU usage using host_statistics64 - Metrics.CpuUsagePercent = 0.0f; - host_cpu_load_info_data_t CpuLoad; - mach_msg_type_number_t CpuCount = sizeof(CpuLoad) / sizeof(natural_t); - if (host_statistics(mach_host_self(), HOST_CPU_LOAD_INFO, (host_info_t)&CpuLoad, &CpuCount) == KERN_SUCCESS) - { - unsigned long TotalTicks = 0; - for (int i = 0; i < CPU_STATE_MAX; ++i) - { - TotalTicks += CpuLoad.cpu_ticks[i]; - } - - if (TotalTicks > 0) - { - unsigned long IdleTicks = CpuLoad.cpu_ticks[CPU_STATE_IDLE]; - Metrics.CpuUsagePercent = 100.0f * (TotalTicks - IdleTicks) / TotalTicks; - } - } - // Get memory information uint64_t MemSize = 0; Size = sizeof(MemSize); @@ -401,6 +324,17 @@ GetSystemMetrics() # error "Unknown platform" #endif +ExtendedSystemMetrics +ApplyReportingOverrides(ExtendedSystemMetrics Metrics) +{ + if (g_FakeCpuCount) + { + Metrics.CoreCount = g_FakeCpuCount; + Metrics.LogicalProcessorCount = g_FakeCpuCount; + } + return Metrics; +} + SystemMetrics GetSystemMetricsForReporting() { @@ -415,12 +349,249 @@ GetSystemMetricsForReporting() return Sm; } +/////////////////////////////////////////////////////////////////////////// +// SystemMetricsTracker +/////////////////////////////////////////////////////////////////////////// + +// Per-platform CPU sampling helper. Called with m_Mutex held. + +#if ZEN_PLATFORM_WINDOWS || ZEN_PLATFORM_LINUX + +// Samples CPU usage by reading /proc/stat. Used natively on Linux and as a +// Wine fallback on Windows (where /proc/stat is accessible via the Z: drive). +struct ProcStatCpuSampler +{ + const char* Path = "/proc/stat"; + unsigned long PrevUser = 0; + unsigned long PrevNice = 0; + unsigned long PrevSystem = 0; + unsigned long PrevIdle = 0; + unsigned long PrevIoWait = 0; + unsigned long PrevIrq = 0; + unsigned long PrevSoftIrq = 0; + + explicit ProcStatCpuSampler(const char* InPath = "/proc/stat") : Path(InPath) {} + + float Sample() + { + float CpuUsage = 0.0f; + + if (FILE* Stat = fopen(Path, "r")) + { + char Line[256]; + unsigned long User, Nice, System, Idle, IoWait, Irq, SoftIrq; + + if (fgets(Line, sizeof(Line), Stat)) + { + if (sscanf(Line, "cpu %lu %lu %lu %lu %lu %lu %lu", &User, &Nice, &System, &Idle, &IoWait, &Irq, &SoftIrq) == 7) + { + unsigned long TotalDelta = (User + Nice + System + Idle + IoWait + Irq + SoftIrq) - + (PrevUser + PrevNice + PrevSystem + PrevIdle + PrevIoWait + PrevIrq + PrevSoftIrq); + unsigned long IdleDelta = Idle - PrevIdle; + + if (TotalDelta > 0) + { + CpuUsage = 100.0f * (TotalDelta - IdleDelta) / TotalDelta; + } + + PrevUser = User; + PrevNice = Nice; + PrevSystem = System; + PrevIdle = Idle; + PrevIoWait = IoWait; + PrevIrq = Irq; + PrevSoftIrq = SoftIrq; + } + } + fclose(Stat); + } + + return CpuUsage; + } +}; + +#endif + +#if ZEN_PLATFORM_WINDOWS + +struct CpuSampler +{ + PDH_HQUERY QueryHandle = nullptr; + PDH_HCOUNTER CounterHandle = nullptr; + bool HasPreviousSample = false; + bool IsWine = false; + ProcStatCpuSampler ProcStat{"Z:\\proc\\stat"}; + + CpuSampler() + { + IsWine = zen::windows::IsRunningOnWine(); + + if (!IsWine) + { + if (PdhOpenQueryW(nullptr, 0, &QueryHandle) == ERROR_SUCCESS) + { + if (PdhAddEnglishCounterW(QueryHandle, L"\\Processor(_Total)\\% Processor Time", 0, &CounterHandle) != ERROR_SUCCESS) + { + CounterHandle = nullptr; + } + } + } + } + + ~CpuSampler() + { + if (QueryHandle) + { + PdhCloseQuery(QueryHandle); + } + } + + float Sample() + { + if (IsWine) + { + return ProcStat.Sample(); + } + + if (!QueryHandle || !CounterHandle) + { + return 0.0f; + } + + PdhCollectQueryData(QueryHandle); + + if (!HasPreviousSample) + { + HasPreviousSample = true; + return 0.0f; + } + + PDH_FMT_COUNTERVALUE CounterValue; + if (PdhGetFormattedCounterValue(CounterHandle, PDH_FMT_DOUBLE, nullptr, &CounterValue) == ERROR_SUCCESS) + { + return static_cast(CounterValue.doubleValue); + } + + return 0.0f; + } +}; + +#elif ZEN_PLATFORM_LINUX + +struct CpuSampler +{ + ProcStatCpuSampler ProcStat; + + float Sample() { return ProcStat.Sample(); } +}; + +#elif ZEN_PLATFORM_MAC + +struct CpuSampler +{ + unsigned long PrevTotalTicks = 0; + unsigned long PrevIdleTicks = 0; + + float Sample() + { + float CpuUsage = 0.0f; + + host_cpu_load_info_data_t CpuLoad; + mach_msg_type_number_t Count = sizeof(CpuLoad) / sizeof(natural_t); + if (host_statistics(mach_host_self(), HOST_CPU_LOAD_INFO, (host_info_t)&CpuLoad, &Count) == KERN_SUCCESS) + { + unsigned long TotalTicks = 0; + for (int i = 0; i < CPU_STATE_MAX; ++i) + { + TotalTicks += CpuLoad.cpu_ticks[i]; + } + unsigned long IdleTicks = CpuLoad.cpu_ticks[CPU_STATE_IDLE]; + + unsigned long TotalDelta = TotalTicks - PrevTotalTicks; + unsigned long IdleDelta = IdleTicks - PrevIdleTicks; + + if (TotalDelta > 0 && PrevTotalTicks > 0) + { + CpuUsage = 100.0f * (TotalDelta - IdleDelta) / TotalDelta; + } + + PrevTotalTicks = TotalTicks; + PrevIdleTicks = IdleTicks; + } + + return CpuUsage; + } +}; + +#endif + +struct SystemMetricsTracker::Impl +{ + using Clock = std::chrono::steady_clock; + + std::mutex Mutex; + CpuSampler Sampler; + float CachedCpuPercent = 0.0f; + Clock::time_point NextSampleTime = Clock::now(); + std::chrono::milliseconds MinInterval; + + explicit Impl(std::chrono::milliseconds InMinInterval) : MinInterval(InMinInterval) {} + + float SampleCpu() + { + const auto Now = Clock::now(); + if (Now >= NextSampleTime) + { + CachedCpuPercent = Sampler.Sample(); + NextSampleTime = Now + MinInterval; + } + return CachedCpuPercent; + } +}; + +SystemMetricsTracker::SystemMetricsTracker(std::chrono::milliseconds MinInterval) : m_Impl(std::make_unique(MinInterval)) +{ +} + +SystemMetricsTracker::~SystemMetricsTracker() = default; + +ExtendedSystemMetrics +SystemMetricsTracker::Query() +{ + ExtendedSystemMetrics Metrics; + static_cast(Metrics) = GetSystemMetrics(); + + std::lock_guard Lock(m_Impl->Mutex); + Metrics.CpuUsagePercent = m_Impl->SampleCpu(); + return Metrics; +} + +/////////////////////////////////////////////////////////////////////////// + std::string_view GetOperatingSystemName() { return ZEN_PLATFORM_NAME; } +std::string_view +GetRuntimePlatformName() +{ +#if ZEN_PLATFORM_WINDOWS + if (zen::windows::IsRunningOnWine()) + { + return "wine"sv; + } + return "windows"sv; +#elif ZEN_PLATFORM_LINUX + return "linux"sv; +#elif ZEN_PLATFORM_MAC + return "macos"sv; +#else + return "unknown"sv; +#endif +} + std::string_view GetCpuName() { @@ -440,4 +611,11 @@ Describe(const SystemMetrics& Metrics, CbWriter& Writer) << "avail_pagefile_mb" << Metrics.AvailPageFileMiB; } +void +Describe(const ExtendedSystemMetrics& Metrics, CbWriter& Writer) +{ + Describe(static_cast(Metrics), Writer); + Writer << "cpu_usage_percent" << Metrics.CpuUsagePercent; +} + } // namespace zen -- cgit v1.2.3 From 6926c04dc4d7c5c0f0310b66c17c9a4e94d2e341 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 4 Mar 2026 16:07:14 +0100 Subject: more feedback during auth option parsing (#806) * remove stray std::unique_ptr Auth; causing crashes * add more feedback during parsing of auth options --- src/zencore/include/zencore/string.h | 2 ++ src/zencore/string.cpp | 56 ++++++++++++++++++++++++++++++------ 2 files changed, 49 insertions(+), 9 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h index 5a12ba5d2..250eb9f56 100644 --- a/src/zencore/include/zencore/string.h +++ b/src/zencore/include/zencore/string.h @@ -1265,6 +1265,8 @@ private: uint64_t LoMask, HiMask; }; +std::string HideSensitiveString(std::string_view String); + ////////////////////////////////////////////////////////////////////////// void string_forcelink(); // internal diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp index 27635a86c..3d0451e27 100644 --- a/src/zencore/string.cpp +++ b/src/zencore/string.cpp @@ -539,10 +539,33 @@ UrlDecode(std::string_view InUrl) return std::string(Url.ToView()); } -////////////////////////////////////////////////////////////////////////// -// -// Unit tests -// +std::string +HideSensitiveString(std::string_view String) +{ + const size_t Length = String.length(); + const size_t SourceLength = Length > 16 ? 4 : 0; + const size_t PadLength = Min(Length - SourceLength, 4u); + const bool AddEllipsis = (SourceLength + PadLength) < Length; + StringBuilder<16> SB; + if (SourceLength > 0) + { + SB << String.substr(0, SourceLength); + } + if (PadLength > 0) + { + SB << std::string(PadLength, 'X'); + } + if (AddEllipsis) + { + SB << "..."; + } + return SB.ToString(); +}; + + ////////////////////////////////////////////////////////////////////////// + // + // Unit tests + // #if ZEN_WITH_TESTS @@ -814,11 +837,6 @@ TEST_CASE("niceNum") } } -void -string_forcelink() -{ -} - TEST_CASE("StringBuilder") { StringBuilder<64> sb; @@ -1224,8 +1242,28 @@ TEST_CASE("string") } } +TEST_CASE("hidesensitivestring") +{ + using namespace std::literals; + + CHECK_EQ(HideSensitiveString(""sv), ""sv); + CHECK_EQ(HideSensitiveString("A"sv), "X"sv); + CHECK_EQ(HideSensitiveString("ABCD"sv), "XXXX"sv); + CHECK_EQ(HideSensitiveString("ABCDE"sv), "XXXX..."sv); + CHECK_EQ(HideSensitiveString("ABCDEFGH"sv), "XXXX..."sv); + CHECK_EQ(HideSensitiveString("ABCDEFGHIJKLMNOP"sv), "XXXX..."sv); + CHECK_EQ(HideSensitiveString("ABCDEFGHIJKLMNOPQ"sv), "ABCDXXXX..."sv); + CHECK_EQ(HideSensitiveString("ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"sv), "ABCDXXXX..."sv); + CHECK_EQ(HideSensitiveString("1234567890123456789"sv), "1234XXXX..."sv); +} + TEST_SUITE_END(); +void +string_forcelink() +{ +} + #endif } // namespace zen -- cgit v1.2.3 From 1f83b48a20bf90f41e18867620c5774f3be6280d Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 4 Mar 2026 17:23:36 +0100 Subject: Fixing various compiler issues (#807) Compile fixes for various versions of gcc,clang (non-UE) --- src/zencore/filesystem.cpp | 11 ++++++----- src/zencore/include/zencore/hashutils.h | 1 + src/zencore/xmake.lua | 7 ++++++- 3 files changed, 13 insertions(+), 6 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 9885b2ada..8ed63565c 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -194,7 +194,7 @@ WipeDirectory(const wchar_t* DirPath, bool KeepDotFiles) FindClose(hFind); } - return true; + return Success; } bool @@ -1022,7 +1022,7 @@ TryCloneFile(const std::filesystem::path& FromPath, const std::filesystem::path& return false; } fchmod(ToFd, 0666); - ScopedFd $To = { FromFd }; + ScopedFd $To = { ToFd }; ioctl(ToFd, FICLONE, FromFd); @@ -1112,7 +1112,8 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP size_t FileSizeBytes = Stat.st_size; - fchown(ToFd, Stat.st_uid, Stat.st_gid); + int $Ignore = fchown(ToFd, Stat.st_uid, Stat.st_gid); + ZEN_UNUSED($Ignore); // What's the appropriate error handling here? // Copy impl const size_t BufferSize = Min(FileSizeBytes, 64u << 10); @@ -1398,7 +1399,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer const uint64_t ChunkSize = Min(WriteSize, uint64_t(2) * 1024 * 1024 * 1024); #if ZEN_PLATFORM_WINDOWS - hRes = Outfile.Write(DataPtr, gsl::narrow_cast(WriteSize)); + hRes = Outfile.Write(DataPtr, gsl::narrow_cast(ChunkSize)); if (FAILED(hRes)) { Outfile.Close(); @@ -1407,7 +1408,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer ThrowSystemException(hRes, fmt::format("File write failed for '{}'", Path).c_str()); } #else - if (write(Fd, DataPtr, WriteSize) != int64_t(WriteSize)) + if (write(Fd, DataPtr, ChunkSize) != int64_t(ChunkSize)) { close(Fd); std::error_code DummyEc; diff --git a/src/zencore/include/zencore/hashutils.h b/src/zencore/include/zencore/hashutils.h index 6b9902b3a..8abfd4b6e 100644 --- a/src/zencore/include/zencore/hashutils.h +++ b/src/zencore/include/zencore/hashutils.h @@ -3,6 +3,7 @@ #pragma once #include +#include #include namespace zen { diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index 2f81b7ec8..ab842f6ed 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -14,7 +14,12 @@ target('zencore') end) set_configdir("include/zencore") add_files("**.cpp") - add_files("trace.cpp", {unity_ignored = true }) + if is_plat("linux") and not (get_config("toolchain") or ""):find("clang") then + -- GCC false positives in thirdparty trace.h (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100137) + add_files("trace.cpp", {unity_ignored = true, force = {cxxflags = {"-Wno-stringop-overread", "-Wno-dangling-pointer"}} }) + else + add_files("trace.cpp", {unity_ignored = true }) + end add_files("testing.cpp", {unity_ignored = true }) if has_config("zenrpmalloc") then -- cgit v1.2.3 From d8940b27c8a5c070c3b48ca9e575929df8d1d888 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 5 Mar 2026 00:08:19 +0100 Subject: added TEST_SUITE_BEGIN/END around some TEST_CASEs which didn't have them (#809) * added TEST_SUITE_BEGIN/END around some TEST_CASEs which didn't have them * fixed some stats issues * ScopedSpan should Initialize * annotated classes in stats.h with some documentation comments --- src/zencore/xxhash.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/zencore') diff --git a/src/zencore/xxhash.cpp b/src/zencore/xxhash.cpp index 6d1050531..88a48dd68 100644 --- a/src/zencore/xxhash.cpp +++ b/src/zencore/xxhash.cpp @@ -59,6 +59,8 @@ xxhash_forcelink() { } +TEST_SUITE_BEGIN("core.xxhash"); + TEST_CASE("XXH3_128") { using namespace std::literals; @@ -96,6 +98,8 @@ TEST_CASE("XXH3_128") } } +TEST_SUITE_END(); + #endif } // namespace zen -- cgit v1.2.3 From 1e731796187ad73b2dee44b48fcecdd487616394 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 6 Mar 2026 10:11:51 +0100 Subject: Claude config, some bug fixes (#813) * Claude config updates * Bug fixes and hardening across `zencore` and `zenhttp`, identified via static analysis. ### zencore - **`ZEN_ASSERT` macro** -- extended to accept an optional string message literal; added `ZEN_ASSERT_MSG_` helper for message formatting. Callers needing runtime fmt-style formatting should use `ZEN_ASSERT_FORMAT`. - **`MpscQueue`** -- fixed `TypeCompatibleStorage` to use a properly-sized `char Storage[sizeof(T)]` array instead of a single `char`; corrected `Data()` to cast `&Storage` rather than `this`; switched cache-line alignment to a fixed constant to avoid GCC's `-Winterference-size` warning. Enabled previously-disabled tests. - **`StringBuilderImpl`** -- initialized `m_Base`/`m_CurPos`/`m_End` to `nullptr`. Fixed `StringCompare` return type (`bool` -> `int`). Fixed `ParseInt` to reject strings with trailing non-numeric characters. Removed deprecated `` include. - **`NiceNumGeneral`** -- replaced `powl()` with integer `IntPow()` to avoid floating-point precision issues. - **`RwLock::ExclusiveLockScope`** -- added move constructor/assignment; initialized `m_Lock` to `nullptr`. - **`Latch::AddCount`** -- fixed variable type (`std::atomic_ptrdiff_t` -> `std::ptrdiff_t` for the return value of `fetch_add`). - **`thread.cpp`** -- fixed Linux `pthread_setname_np` 16-byte name truncation; added null check before dereferencing in `Event::Close()`; fixed `NamedEvent::Close()` to call `close(Fd)` outside the lock region; added null guard in `NamedMutex` destructor; `Sleep()` now returns early for non-positive durations. - **`MD5Stream`** -- was entirely stubbed out (no-op); now correctly calls `MD5Init`/`MD5Update`/`MD5Final`. Fixed `ToHexString` to use the correct string length. Fixed forward declarations. Fixed tests to compare `compare() == 0`. - **`sentryintegration.cpp`** -- guard against null `filename`/`funcname` in spdlog message handler to prevent a crash in `fmt::format`. - **`jobqueue.cpp`** -- fixed lost job ID when `IdGenerator` wraps around zero; fixed raw `Job*` in `RunningJobs` map (potential use-after-free) to `RefPtr`; fixed range-loop copies; fixed format string typo. - **`trace.cpp`** -- suppress GCC false-positive warnings in third-party `trace.h` include. ### zenhttp - **WebSocket close race** (`wsasio`, `wshttpsys`, `httpwsclient`) -- `m_CloseSent` promoted from `bool` to `std::atomic`; close check changed to `exchange(true)` to eliminate the check-then-set data race. - **`wsframecodec.cpp`** -- reject WebSocket frames with payload > 256 MB to prevent OOM from malformed/malicious frames. - **`oidc.cpp`** -- URL-encode refresh token and client ID in token requests (`FormUrlEncode`); parse `end_session_endpoint` and `device_authorization_endpoint` from OIDC discovery document. - **`httpclientcommon.cpp`** -- propagate error code from `AppendData` when flushing the cache buffer. - **`httpclient.h`** -- initialize all uninitialized members (`ErrorCode`, `UploadedBytes`, `DownloadedBytes`, `ElapsedSeconds`, `MultipartBoundary` fields). - **`httpserver.h`** -- fix `operator=` return type for `HttpRpcHandler` (missing `&`). - **`packageformat.h`** -- fix `~0u` (32-bit truncation) to `~uint64_t(0)` for a `uint64_t` field. - **`httpparser`** -- initialize `m_RequestVerb` in both declaration and `ResetState()`. - **`httpplugin.cpp`** -- initialize `m_BasePort`; fix format string missing quotes around connection name. - **`httptracer.h`** -- move `#pragma once` before includes. - **`websocket.h`** -- initialize `WebSocketMessage::Opcode`. ### zenserver - **`hubservice.cpp`** -- fix two `ZEN_ASSERT` calls that incorrectly used fmt-style format args; converted to `ZEN_ASSERT_FORMAT`. --- src/zencore/blake3.cpp | 2 +- src/zencore/commandline.cpp | 1 + src/zencore/include/zencore/md5.h | 2 ++ src/zencore/include/zencore/mpscqueue.h | 20 +++++++++------- src/zencore/include/zencore/string.h | 22 ++++++++--------- src/zencore/include/zencore/thread.h | 16 ++++++++++--- src/zencore/include/zencore/xxhash.h | 2 +- src/zencore/include/zencore/zencore.h | 34 ++++++++++++++++---------- src/zencore/jobqueue.cpp | 20 +++++++--------- src/zencore/logging.cpp | 2 +- src/zencore/md5.cpp | 24 +++++++++++-------- src/zencore/memtrack/tagtrace.cpp | 2 +- src/zencore/mpscqueue.cpp | 4 ++-- src/zencore/sentryintegration.cpp | 9 +++++-- src/zencore/string.cpp | 15 ++++++++++-- src/zencore/thread.cpp | 42 ++++++++++++++++++++++----------- src/zencore/trace.cpp | 9 +++++++ src/zencore/xmake.lua | 7 +----- 18 files changed, 146 insertions(+), 87 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/blake3.cpp b/src/zencore/blake3.cpp index 123918de5..55f9b74af 100644 --- a/src/zencore/blake3.cpp +++ b/src/zencore/blake3.cpp @@ -123,7 +123,7 @@ BLAKE3::ToHexString(StringBuilderBase& outBuilder) const char str[65]; ToHexString(str); - outBuilder.AppendRange(str, &str[65]); + outBuilder.AppendRange(str, &str[StringLength]); return outBuilder; } diff --git a/src/zencore/commandline.cpp b/src/zencore/commandline.cpp index 426cf23d6..718ef9678 100644 --- a/src/zencore/commandline.cpp +++ b/src/zencore/commandline.cpp @@ -14,6 +14,7 @@ ZEN_THIRD_PARTY_INCLUDES_END # include #endif +#include #include namespace zen { diff --git a/src/zencore/include/zencore/md5.h b/src/zencore/include/zencore/md5.h index d934dd86b..3b0b7cae6 100644 --- a/src/zencore/include/zencore/md5.h +++ b/src/zencore/include/zencore/md5.h @@ -43,6 +43,8 @@ public: MD5 GetHash(); private: + // Opaque storage for MD5_CTX (104 bytes, aligned to uint32_t) + alignas(4) uint8_t m_Context[104]; }; void md5_forcelink(); // internal diff --git a/src/zencore/include/zencore/mpscqueue.h b/src/zencore/include/zencore/mpscqueue.h index 19e410d85..d97c433fd 100644 --- a/src/zencore/include/zencore/mpscqueue.h +++ b/src/zencore/include/zencore/mpscqueue.h @@ -22,10 +22,10 @@ namespace zen { template struct TypeCompatibleStorage { - ElementType* Data() { return (ElementType*)this; } - const ElementType* Data() const { return (const ElementType*)this; } + ElementType* Data() { return reinterpret_cast(&Storage); } + const ElementType* Data() const { return reinterpret_cast(&Storage); } - alignas(ElementType) char DataMember; + alignas(ElementType) char Storage[sizeof(ElementType)]; }; /** Fast multi-producer/single-consumer unbounded concurrent queue. @@ -58,7 +58,7 @@ public: Tail = Next; Next = Tail->Next.load(std::memory_order_relaxed); - std::destroy_at((ElementType*)&Tail->Value); + std::destroy_at(Tail->Value.Data()); delete Tail; } } @@ -67,7 +67,7 @@ public: void Enqueue(ArgTypes&&... Args) { Node* New = new Node; - new (&New->Value) ElementType(std::forward(Args)...); + new (New->Value.Data()) ElementType(std::forward(Args)...); Node* Prev = Head.exchange(New, std::memory_order_acq_rel); Prev->Next.store(New, std::memory_order_release); @@ -82,7 +82,7 @@ public: return {}; } - ElementType* ValuePtr = (ElementType*)&Next->Value; + ElementType* ValuePtr = Next->Value.Data(); std::optional Res{std::move(*ValuePtr)}; std::destroy_at(ValuePtr); @@ -100,9 +100,11 @@ private: }; private: - std::atomic Head; // accessed only by producers - alignas(hardware_constructive_interference_size) - Node* Tail; // accessed only by consumer, hence should be on a different cache line than `Head` + // Use a fixed constant to avoid GCC's -Winterference-size warning with std::hardware_destructive_interference_size + static constexpr std::size_t CacheLineSize = 64; + + alignas(CacheLineSize) std::atomic Head; // accessed only by producers + alignas(CacheLineSize) Node* Tail; // accessed only by consumer, separate cache line from Head }; void mpscqueue_forcelink(); diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h index 250eb9f56..4deca63ed 100644 --- a/src/zencore/include/zencore/string.h +++ b/src/zencore/include/zencore/string.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -51,7 +50,7 @@ StringLength(const wchar_t* str) return wcslen(str); } -inline bool +inline int StringCompare(const char16_t* s1, const char16_t* s2) { char16_t c1, c2; @@ -66,7 +65,7 @@ StringCompare(const char16_t* s1, const char16_t* s2) ++s1; ++s2; } - return uint16_t(c1) - uint16_t(c2); + return int(uint16_t(c1)) - int(uint16_t(c2)); } inline bool @@ -122,10 +121,10 @@ public: StringBuilderImpl() = default; ~StringBuilderImpl(); - StringBuilderImpl(const StringBuilderImpl&) = delete; - StringBuilderImpl(const StringBuilderImpl&&) = delete; + StringBuilderImpl(const StringBuilderImpl&) = delete; + StringBuilderImpl(StringBuilderImpl&&) = delete; const StringBuilderImpl& operator=(const StringBuilderImpl&) = delete; - const StringBuilderImpl& operator=(const StringBuilderImpl&&) = delete; + StringBuilderImpl& operator=(StringBuilderImpl&&) = delete; inline size_t AddUninitialized(size_t Count) { @@ -374,9 +373,9 @@ protected: [[noreturn]] void Fail(const char* FailReason); // note: throws exception - C* m_Base; - C* m_CurPos; - C* m_End; + C* m_Base = nullptr; + C* m_CurPos = nullptr; + C* m_End = nullptr; bool m_IsDynamic = false; bool m_IsExtendable = false; }; @@ -773,8 +772,9 @@ std::optional ParseInt(const std::string_view& Input) { T Out = 0; - const std::from_chars_result Result = std::from_chars(Input.data(), Input.data() + Input.size(), Out); - if (Result.ec == std::errc::invalid_argument || Result.ec == std::errc::result_out_of_range) + const char* End = Input.data() + Input.size(); + const std::from_chars_result Result = std::from_chars(Input.data(), End, Out); + if (Result.ec == std::errc::invalid_argument || Result.ec == std::errc::result_out_of_range || Result.ptr != End) { return std::nullopt; } diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h index a1c68b0b2..d0d710ee8 100644 --- a/src/zencore/include/zencore/thread.h +++ b/src/zencore/include/zencore/thread.h @@ -58,7 +58,7 @@ public: } private: - RwLock* m_Lock; + RwLock* m_Lock = nullptr; }; inline auto WithSharedLock(auto&& Fun) @@ -69,6 +69,16 @@ public: struct ExclusiveLockScope { + ExclusiveLockScope(const ExclusiveLockScope& Rhs) = delete; + ExclusiveLockScope(ExclusiveLockScope&& Rhs) : m_Lock(Rhs.m_Lock) { Rhs.m_Lock = nullptr; } + ExclusiveLockScope& operator=(ExclusiveLockScope&& Rhs) + { + ReleaseNow(); + m_Lock = Rhs.m_Lock; + Rhs.m_Lock = nullptr; + return *this; + } + ExclusiveLockScope& operator=(const ExclusiveLockScope& Rhs) = delete; ExclusiveLockScope(RwLock& Lock) : m_Lock(&Lock) { Lock.AcquireExclusive(); } ~ExclusiveLockScope() { ReleaseNow(); } @@ -82,7 +92,7 @@ public: } private: - RwLock* m_Lock; + RwLock* m_Lock = nullptr; }; inline auto WithExclusiveLock(auto&& Fun) @@ -195,7 +205,7 @@ public: // false positive completion results. void AddCount(std::ptrdiff_t Count) { - std::atomic_ptrdiff_t Old = Counter.fetch_add(Count); + std::ptrdiff_t Old = Counter.fetch_add(Count); ZEN_ASSERT(Old > 0); } diff --git a/src/zencore/include/zencore/xxhash.h b/src/zencore/include/zencore/xxhash.h index fc55b513b..f79d39b61 100644 --- a/src/zencore/include/zencore/xxhash.h +++ b/src/zencore/include/zencore/xxhash.h @@ -87,7 +87,7 @@ struct XXH3_128Stream } private: - XXH3_state_s m_State; + XXH3_state_s m_State{}; }; struct XXH3_128Stream_deprecated diff --git a/src/zencore/include/zencore/zencore.h b/src/zencore/include/zencore/zencore.h index 177a19fff..a31950b0b 100644 --- a/src/zencore/include/zencore/zencore.h +++ b/src/zencore/include/zencore/zencore.h @@ -70,26 +70,36 @@ protected: } // namespace zen -#define ZEN_ASSERT(x, ...) \ - do \ - { \ - if (x) [[unlikely]] \ - break; \ - zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, #x); \ +#define ZEN_ASSERT(x, ...) \ + do \ + { \ + if (x) [[unlikely]] \ + break; \ + zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, ZEN_ASSERT_MSG_(#x, ##__VA_ARGS__)); \ } while (false) #ifndef NDEBUG -# define ZEN_ASSERT_SLOW(x, ...) \ - do \ - { \ - if (x) [[unlikely]] \ - break; \ - zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, #x); \ +# define ZEN_ASSERT_SLOW(x, ...) \ + do \ + { \ + if (x) [[unlikely]] \ + break; \ + zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, ZEN_ASSERT_MSG_(#x, ##__VA_ARGS__)); \ } while (false) #else # define ZEN_ASSERT_SLOW(x, ...) #endif +// Internal: select between "expr" and "expr: message" forms. +// With no extra args: ZEN_ASSERT_MSG_("expr") -> "expr" +// With a message arg: ZEN_ASSERT_MSG_("expr", "msg") -> "expr" ": " "msg" +// With fmt-style args: ZEN_ASSERT_MSG_("expr", "msg", args...) -> "expr" ": " "msg" +// The extra fmt args are silently discarded here — use ZEN_ASSERT_FORMAT for those. +#define ZEN_ASSERT_MSG_SELECT_(_1, _2, N, ...) N +#define ZEN_ASSERT_MSG_1_(expr) expr +#define ZEN_ASSERT_MSG_2_(expr, msg, ...) expr ": " msg +#define ZEN_ASSERT_MSG_(expr, ...) ZEN_ASSERT_MSG_SELECT_(unused, ##__VA_ARGS__, ZEN_ASSERT_MSG_2_, ZEN_ASSERT_MSG_1_)(expr, ##__VA_ARGS__) + ////////////////////////////////////////////////////////////////////////// #define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false, __VA_ARGS__) diff --git a/src/zencore/jobqueue.cpp b/src/zencore/jobqueue.cpp index 35724b07a..d6a8a6479 100644 --- a/src/zencore/jobqueue.cpp +++ b/src/zencore/jobqueue.cpp @@ -90,7 +90,7 @@ public: uint64_t NewJobId = IdGenerator.fetch_add(1); if (NewJobId == 0) { - IdGenerator.fetch_add(1); + NewJobId = IdGenerator.fetch_add(1); } RefPtr NewJob(new Job()); NewJob->Queue = this; @@ -129,7 +129,7 @@ public: QueuedJobs.erase(It); } }); - ZEN_ERROR("Failed to schedule job {}:'{}' to job queue. Reason: ''", NewJob->Id.Id, NewJob->Name, Ex.what()); + ZEN_ERROR("Failed to schedule job {}:'{}' to job queue. Reason: '{}'", NewJob->Id.Id, NewJob->Name, Ex.what()); throw; } } @@ -221,11 +221,11 @@ public: std::vector Jobs; QueueLock.WithSharedLock([&]() { - for (auto It : RunningJobs) + for (const auto& It : RunningJobs) { Jobs.push_back({.Id = JobId{It.first}, .Status = JobStatus::Running}); } - for (auto It : CompletedJobs) + for (const auto& It : CompletedJobs) { if (IsStale(It.second->EndTick)) { @@ -234,7 +234,7 @@ public: } Jobs.push_back({.Id = JobId{It.first}, .Status = JobStatus::Completed}); } - for (auto It : AbortedJobs) + for (const auto& It : AbortedJobs) { if (IsStale(It.second->EndTick)) { @@ -243,7 +243,7 @@ public: } Jobs.push_back({.Id = JobId{It.first}, .Status = JobStatus::Aborted}); } - for (auto It : QueuedJobs) + for (const auto& It : QueuedJobs) { Jobs.push_back({.Id = It->Id, .Status = JobStatus::Queued}); } @@ -337,7 +337,7 @@ public: std::atomic_bool InitializedFlag = false; RwLock QueueLock; std::deque> QueuedJobs; - std::unordered_map RunningJobs; + std::unordered_map> RunningJobs; std::unordered_map> CompletedJobs; std::unordered_map> AbortedJobs; @@ -429,20 +429,16 @@ JobQueue::ToString(JobStatus Status) { case JobQueue::JobStatus::Queued: return "Queued"sv; - break; case JobQueue::JobStatus::Running: return "Running"sv; - break; case JobQueue::JobStatus::Aborted: return "Aborted"sv; - break; case JobQueue::JobStatus::Completed: return "Completed"sv; - break; default: ZEN_ASSERT(false); + return ""sv; } - return ""sv; } std::unique_ptr diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp index e960a2729..ebd68de09 100644 --- a/src/zencore/logging.cpp +++ b/src/zencore/logging.cpp @@ -303,7 +303,7 @@ GetLogLevel() LoggerRef Default() { - ZEN_ASSERT(TheDefaultLogger); + ZEN_ASSERT(TheDefaultLogger, "logging::InitializeLogging() must be called before using the logger"); return TheDefaultLogger; } diff --git a/src/zencore/md5.cpp b/src/zencore/md5.cpp index 83ed53fc8..f8cfee3ac 100644 --- a/src/zencore/md5.cpp +++ b/src/zencore/md5.cpp @@ -56,9 +56,9 @@ struct MD5_CTX unsigned char digest[16]; /* actual digest after MD5Final call */ }; -void MD5Init(); -void MD5Update(); -void MD5Final(); +void MD5Init(MD5_CTX* mdContext); +void MD5Update(MD5_CTX* mdContext, unsigned char* inBuf, unsigned int inLen); +void MD5Final(MD5_CTX* mdContext); /* ********************************************************************** @@ -370,28 +370,32 @@ MD5 MD5::Zero; // Initialized to all zeroes MD5Stream::MD5Stream() { + static_assert(sizeof(MD5_CTX) <= sizeof(m_Context)); Reset(); } void MD5Stream::Reset() { + MD5Init(reinterpret_cast(m_Context)); } MD5Stream& MD5Stream::Append(const void* Data, size_t ByteCount) { - ZEN_UNUSED(Data); - ZEN_UNUSED(ByteCount); - + MD5Update(reinterpret_cast(m_Context), (unsigned char*)Data, (unsigned int)ByteCount); return *this; } MD5 MD5Stream::GetHash() { - MD5 md5{}; + MD5_CTX FinalCtx; + memcpy(&FinalCtx, m_Context, sizeof(MD5_CTX)); + MD5Final(&FinalCtx); + MD5 md5{}; + memcpy(md5.Hash, FinalCtx.digest, 16); return md5; } @@ -428,7 +432,7 @@ MD5::ToHexString(StringBuilderBase& outBuilder) const char str[41]; ToHexString(str); - outBuilder.AppendRange(str, &str[40]); + outBuilder.AppendRange(str, &str[StringLength]); return outBuilder; } @@ -470,11 +474,11 @@ TEST_CASE("MD5") MD5::String_t Buffer; Result.ToHexString(Buffer); - CHECK(Output.compare(Buffer)); + CHECK(Output.compare(Buffer) == 0); MD5 Reresult = MD5::FromHexString(Buffer); Reresult.ToHexString(Buffer); - CHECK(Output.compare(Buffer)); + CHECK(Output.compare(Buffer) == 0); } TEST_SUITE_END(); diff --git a/src/zencore/memtrack/tagtrace.cpp b/src/zencore/memtrack/tagtrace.cpp index 70a74365d..fca4a2ec3 100644 --- a/src/zencore/memtrack/tagtrace.cpp +++ b/src/zencore/memtrack/tagtrace.cpp @@ -186,7 +186,7 @@ FTagTrace::AnnounceSpecialTags() const { auto EmitTag = [](const char16_t* DisplayString, int32_t Tag, int32_t ParentTag) { const uint32_t DisplayLen = (uint32_t)StringLength(DisplayString); - UE_TRACE_LOG(Memory, TagSpec, MemAllocChannel, DisplayLen * sizeof(ANSICHAR)) + UE_TRACE_LOG(Memory, TagSpec, MemAllocChannel, DisplayLen * sizeof(char16_t)) << TagSpec.Tag(Tag) << TagSpec.Parent(ParentTag) << TagSpec.Display(DisplayString, DisplayLen); }; diff --git a/src/zencore/mpscqueue.cpp b/src/zencore/mpscqueue.cpp index f749f1c90..bdd22e20c 100644 --- a/src/zencore/mpscqueue.cpp +++ b/src/zencore/mpscqueue.cpp @@ -7,7 +7,7 @@ namespace zen { -#if ZEN_WITH_TESTS && 0 +#if ZEN_WITH_TESTS TEST_SUITE_BEGIN("core.mpscqueue"); TEST_CASE("mpsc") { @@ -24,4 +24,4 @@ mpscqueue_forcelink() { } -} // namespace zen \ No newline at end of file +} // namespace zen diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index 636e182b4..bfff114c3 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -81,8 +81,13 @@ sentry_sink::sink_it_(const spdlog::details::log_msg& msg) } try { - std::string Message = fmt::format("{}\n{}({}) [{}]", msg.payload, msg.source.filename, msg.source.line, msg.source.funcname); - sentry_value_t event = sentry_value_new_message_event( + auto MaybeNullString = [](const char* Ptr) { return Ptr ? Ptr : ""; }; + std::string Message = fmt::format("{}\n{}({}) [{}]", + msg.payload, + MaybeNullString(msg.source.filename), + msg.source.line, + MaybeNullString(msg.source.funcname)); + sentry_value_t event = sentry_value_new_message_event( /* level */ MapToSentryLevel[msg.level], /* logger */ nullptr, /* message */ Message.c_str()); diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp index 3d0451e27..ed0ba6f46 100644 --- a/src/zencore/string.cpp +++ b/src/zencore/string.cpp @@ -268,6 +268,17 @@ namespace { /* kNicenumTime */ 1000}; } // namespace +uint64_t +IntPow(uint64_t Base, int Exp) +{ + uint64_t Result = 1; + for (int I = 0; I < Exp; ++I) + { + Result *= Base; + } + return Result; +} + /* * Convert a number to an appropriately human-readable output. */ @@ -315,7 +326,7 @@ NiceNumGeneral(uint64_t Num, std::span Buffer, NicenumFormat Format) const char* u = UnitStrings[Format][Index]; - if ((Index == 0) || ((Num % (uint64_t)powl((int)KiloUnit[Format], Index)) == 0)) + if ((Index == 0) || ((Num % IntPow(KiloUnit[Format], Index)) == 0)) { /* * If this is an even multiple of the base, always display @@ -339,7 +350,7 @@ NiceNumGeneral(uint64_t Num, std::span Buffer, NicenumFormat Format) for (int i = 2; i >= 0; i--) { - double Value = (double)Num / (uint64_t)powl((int)KiloUnit[Format], Index); + double Value = (double)Num / IntPow(KiloUnit[Format], Index); /* * Don't print floating point values for time. Note, diff --git a/src/zencore/thread.cpp b/src/zencore/thread.cpp index 9e3486e49..54459cbaa 100644 --- a/src/zencore/thread.cpp +++ b/src/zencore/thread.cpp @@ -133,7 +133,10 @@ SetCurrentThreadName([[maybe_unused]] std::string_view ThreadName) #elif ZEN_PLATFORM_MAC pthread_setname_np(ThreadNameZ.c_str()); #else - pthread_setname_np(pthread_self(), ThreadNameZ.c_str()); + // Linux pthread_setname_np has a 16-byte limit (15 chars + NUL) + StringBuilder<16> LinuxThreadName; + LinuxThreadName << LimitedThreadName.substr(0, 15); + pthread_setname_np(pthread_self(), LinuxThreadName.c_str()); #endif } // namespace zen @@ -233,12 +236,15 @@ Event::Close() #else std::atomic_thread_fence(std::memory_order_acquire); auto* Inner = (EventInner*)m_EventHandle.load(); + if (Inner) { - std::unique_lock Lock(Inner->Mutex); - Inner->bSet.store(true); - m_EventHandle = nullptr; + { + std::unique_lock Lock(Inner->Mutex); + Inner->bSet.store(true); + m_EventHandle = nullptr; + } + delete Inner; } - delete Inner; #endif } @@ -351,7 +357,7 @@ NamedEvent::NamedEvent(std::string_view EventName) intptr_t Packed; Packed = intptr_t(Sem) << 32; Packed |= intptr_t(Fd) & 0xffff'ffff; - m_EventHandle = (void*)Packed; + m_EventHandle = (void*)Packed; #endif ZEN_ASSERT(m_EventHandle != nullptr); } @@ -372,7 +378,9 @@ NamedEvent::Close() #if ZEN_PLATFORM_WINDOWS CloseHandle(m_EventHandle); #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - int Fd = int(intptr_t(m_EventHandle.load()) & 0xffff'ffff); + const intptr_t Handle = intptr_t(m_EventHandle.load()); + const int Fd = int(Handle & 0xffff'ffff); + const int Sem = int(Handle >> 32); if (flock(Fd, LOCK_EX | LOCK_NB) == 0) { @@ -388,11 +396,10 @@ NamedEvent::Close() } flock(Fd, LOCK_UN | LOCK_NB); - close(Fd); - - int Sem = int(intptr_t(m_EventHandle.load()) >> 32); semctl(Sem, 0, IPC_RMID); } + + close(Fd); #endif m_EventHandle = nullptr; @@ -481,9 +488,12 @@ NamedMutex::~NamedMutex() CloseHandle(m_MutexHandle); } #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - int Inner = int(intptr_t(m_MutexHandle)); - flock(Inner, LOCK_UN); - close(Inner); + if (m_MutexHandle) + { + int Inner = int(intptr_t(m_MutexHandle)); + flock(Inner, LOCK_UN); + close(Inner); + } #endif } @@ -516,7 +526,6 @@ NamedMutex::Create(std::string_view MutexName) if (flock(Inner, LOCK_EX) != 0) { close(Inner); - Inner = 0; return false; } @@ -583,6 +592,11 @@ GetCurrentThreadId() void Sleep(int ms) { + if (ms <= 0) + { + return; + } + #if ZEN_PLATFORM_WINDOWS ::Sleep(ms); #else diff --git a/src/zencore/trace.cpp b/src/zencore/trace.cpp index a026974c0..7c195e69f 100644 --- a/src/zencore/trace.cpp +++ b/src/zencore/trace.cpp @@ -10,7 +10,16 @@ # define TRACE_IMPLEMENT 1 # undef _WINSOCK_DEPRECATED_NO_WARNINGS +// GCC false positives in thirdparty trace.h (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100137) +# if ZEN_COMPILER_GCC +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wstringop-overread" +# pragma GCC diagnostic ignored "-Wdangling-pointer" +# endif # include +# if ZEN_COMPILER_GCC +# pragma GCC diagnostic pop +# endif # include # include diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index ab842f6ed..2f81b7ec8 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -14,12 +14,7 @@ target('zencore') end) set_configdir("include/zencore") add_files("**.cpp") - if is_plat("linux") and not (get_config("toolchain") or ""):find("clang") then - -- GCC false positives in thirdparty trace.h (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100137) - add_files("trace.cpp", {unity_ignored = true, force = {cxxflags = {"-Wno-stringop-overread", "-Wno-dangling-pointer"}} }) - else - add_files("trace.cpp", {unity_ignored = true }) - end + add_files("trace.cpp", {unity_ignored = true }) add_files("testing.cpp", {unity_ignored = true }) if has_config("zenrpmalloc") then -- cgit v1.2.3 From 19a117889c2db6b817af9458c04c04f324162e75 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 9 Mar 2026 10:50:47 +0100 Subject: Eliminate spdlog dependency (#773) Removes the vendored spdlog library (~12,000 lines) and replaces it with a purpose-built logging system in zencore (~1,800 lines). The new implementation provides the same functionality with fewer abstractions, no shared_ptr overhead, and full control over the logging pipeline. ### What changed **New logging core in zencore/logging/:** - LogMessage, Formatter, Sink, Logger, Registry - core abstractions matching spdlog's model but simplified - AnsiColorStdoutSink - ANSI color console output (replaces spdlog stdout_color_sink) - MsvcSink - OutputDebugString on Windows (replaces spdlog msvc_sink) - AsyncSink - async logging via BlockingQueue worker thread (replaces spdlog async_logger) - NullSink, MessageOnlyFormatter - utility types - Thread-safe timestamp caching in formatters using RwLock **Moved to zenutil/logging/:** - FullFormatter - full log formatting with timestamp, logger name, level, source location, multiline alignment - JsonFormatter - structured JSON log output - RotatingFileSink - rotating file sink with atomic size tracking **API changes:** - Log levels are now an enum (LogLevel) instead of int, eliminating the zen::logging::level namespace - LoggerRef no longer wraps shared_ptr - it holds a raw pointer with the registry owning lifetime - Logger error handler is wired through Registry and propagated to all loggers on registration - Logger::Log() now populates ThreadId on every message **Cleanup:** - Deleted thirdparty/spdlog/ entirely (110+ files) - Deleted full_test_formatter (was ~80% duplicate of FullFormatter) - Renamed snake_case classes to PascalCase (full_formatter -> FullFormatter, json_formatter -> JsonFormatter, sentry_sink -> SentrySink) - Removed spdlog from xmake dependency graph ### Build / test impact - zencore no longer depends on spdlog - zenutil and zenvfs xmake.lua updated to drop spdlog dep - zentelemetry xmake.lua updated to drop spdlog dep - All existing tests pass, no test changes required beyond formatter class renames --- src/zencore/include/zencore/blockingqueue.h | 2 + src/zencore/include/zencore/logbase.h | 113 ++++--- src/zencore/include/zencore/logging.h | 214 ++++++------- .../include/zencore/logging/ansicolorsink.h | 26 ++ src/zencore/include/zencore/logging/asyncsink.h | 30 ++ src/zencore/include/zencore/logging/formatter.h | 20 ++ src/zencore/include/zencore/logging/helpers.h | 122 ++++++++ src/zencore/include/zencore/logging/logger.h | 63 ++++ src/zencore/include/zencore/logging/logmsg.h | 66 +++++ src/zencore/include/zencore/logging/memorybuffer.h | 11 + .../include/zencore/logging/messageonlyformatter.h | 22 ++ src/zencore/include/zencore/logging/msvcsink.h | 30 ++ src/zencore/include/zencore/logging/nullsink.h | 17 ++ src/zencore/include/zencore/logging/registry.h | 70 +++++ src/zencore/include/zencore/logging/sink.h | 34 +++ src/zencore/include/zencore/logging/tracesink.h | 23 ++ src/zencore/include/zencore/sentryintegration.h | 8 +- src/zencore/logging.cpp | 328 +++++++++----------- src/zencore/logging/ansicolorsink.cpp | 178 +++++++++++ src/zencore/logging/asyncsink.cpp | 212 +++++++++++++ src/zencore/logging/logger.cpp | 142 +++++++++ src/zencore/logging/msvcsink.cpp | 80 +++++ src/zencore/logging/registry.cpp | 330 +++++++++++++++++++++ src/zencore/logging/tracesink.cpp | 88 ++++++ src/zencore/sentryintegration.cpp | 128 ++++---- src/zencore/testing.cpp | 4 +- src/zencore/xmake.lua | 1 - src/zencore/zencore.cpp | 2 +- 28 files changed, 1912 insertions(+), 452 deletions(-) create mode 100644 src/zencore/include/zencore/logging/ansicolorsink.h create mode 100644 src/zencore/include/zencore/logging/asyncsink.h create mode 100644 src/zencore/include/zencore/logging/formatter.h create mode 100644 src/zencore/include/zencore/logging/helpers.h create mode 100644 src/zencore/include/zencore/logging/logger.h create mode 100644 src/zencore/include/zencore/logging/logmsg.h create mode 100644 src/zencore/include/zencore/logging/memorybuffer.h create mode 100644 src/zencore/include/zencore/logging/messageonlyformatter.h create mode 100644 src/zencore/include/zencore/logging/msvcsink.h create mode 100644 src/zencore/include/zencore/logging/nullsink.h create mode 100644 src/zencore/include/zencore/logging/registry.h create mode 100644 src/zencore/include/zencore/logging/sink.h create mode 100644 src/zencore/include/zencore/logging/tracesink.h create mode 100644 src/zencore/logging/ansicolorsink.cpp create mode 100644 src/zencore/logging/asyncsink.cpp create mode 100644 src/zencore/logging/logger.cpp create mode 100644 src/zencore/logging/msvcsink.cpp create mode 100644 src/zencore/logging/registry.cpp create mode 100644 src/zencore/logging/tracesink.cpp (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/blockingqueue.h b/src/zencore/include/zencore/blockingqueue.h index e91fdc659..b6c93e937 100644 --- a/src/zencore/include/zencore/blockingqueue.h +++ b/src/zencore/include/zencore/blockingqueue.h @@ -2,6 +2,8 @@ #pragma once +#include // For ZEN_ASSERT + #include #include #include diff --git a/src/zencore/include/zencore/logbase.h b/src/zencore/include/zencore/logbase.h index 00af68b0a..ece17a85e 100644 --- a/src/zencore/include/zencore/logbase.h +++ b/src/zencore/include/zencore/logbase.h @@ -4,96 +4,85 @@ #include -#define ZEN_LOG_LEVEL_TRACE 0 -#define ZEN_LOG_LEVEL_DEBUG 1 -#define ZEN_LOG_LEVEL_INFO 2 -#define ZEN_LOG_LEVEL_WARN 3 -#define ZEN_LOG_LEVEL_ERROR 4 -#define ZEN_LOG_LEVEL_CRITICAL 5 -#define ZEN_LOG_LEVEL_OFF 6 - -#define ZEN_LEVEL_NAME_TRACE std::string_view("trace", 5) -#define ZEN_LEVEL_NAME_DEBUG std::string_view("debug", 5) -#define ZEN_LEVEL_NAME_INFO std::string_view("info", 4) -#define ZEN_LEVEL_NAME_WARNING std::string_view("warning", 7) -#define ZEN_LEVEL_NAME_ERROR std::string_view("error", 5) -#define ZEN_LEVEL_NAME_CRITICAL std::string_view("critical", 8) -#define ZEN_LEVEL_NAME_OFF std::string_view("off", 3) - -namespace zen::logging::level { +namespace zen::logging { enum LogLevel : int { - Trace = ZEN_LOG_LEVEL_TRACE, - Debug = ZEN_LOG_LEVEL_DEBUG, - Info = ZEN_LOG_LEVEL_INFO, - Warn = ZEN_LOG_LEVEL_WARN, - Err = ZEN_LOG_LEVEL_ERROR, - Critical = ZEN_LOG_LEVEL_CRITICAL, - Off = ZEN_LOG_LEVEL_OFF, + Trace, + Debug, + Info, + Warn, + Err, + Critical, + Off, LogLevelCount }; LogLevel ParseLogLevelString(std::string_view String); std::string_view ToStringView(LogLevel Level); -} // namespace zen::logging::level - -namespace zen::logging { - -void SetLogLevel(level::LogLevel NewLogLevel); -level::LogLevel GetLogLevel(); +void SetLogLevel(LogLevel NewLogLevel); +LogLevel GetLogLevel(); -} // namespace zen::logging +struct SourceLocation +{ + constexpr SourceLocation() = default; + constexpr SourceLocation(const char* InFilename, int InLine) : Filename(InFilename), Line(InLine) {} -namespace spdlog { -class logger; -} + constexpr operator bool() const noexcept { return Line != 0; } -namespace zen::logging { + const char* Filename{nullptr}; + int Line{0}; +}; -struct SourceLocation +/** This encodes the constant parts of a log message which can be emitted once + * and then referred to by log events. + * + * It's *critical* that instances of this struct are permanent and never + * destroyed, as log messages will refer to them by pointer. The easiest way + * to ensure this is to create them as function-local statics. + * + * The logging macros already do this for you so this should not be something + * you normally would need to worry about. + */ +struct LogPoint { - constexpr SourceLocation() = default; - constexpr SourceLocation(const char* filename_in, int line_in, const char* funcname_in) - : filename(filename_in) - , line(line_in) - , funcname(funcname_in) - { - } - - constexpr bool empty() const noexcept { return line == 0; } - - // IMPORTANT NOTE: the layout of this class must match the spdlog::source_loc class - // since we currently pass a pointer to it into spdlog after casting it to - // spdlog::source_loc* - // - // This is intended to be an intermediate state, before we (probably) transition off - // spdlog entirely - - const char* filename{nullptr}; - int line{0}; - const char* funcname{nullptr}; + SourceLocation Location; + LogLevel Level; + std::string_view FormatString; }; +class Logger; + } // namespace zen::logging namespace zen { +// Lightweight non-owning handle to a Logger. Loggers are owned by the Registry +// via Ref; LoggerRef exists as a cheap (raw pointer) handle that can be +// stored in members and passed through logging macros without requiring the +// complete Logger type or incurring refcount overhead on every log call. struct LoggerRef { LoggerRef() = default; - LoggerRef(spdlog::logger& InLogger) : SpdLogger(&InLogger) {} + LoggerRef(logging::Logger& InLogger) : m_Logger(&InLogger) {} + // This exists so that logging macros can pass LoggerRef or LogCategory + // to ZEN_LOG without needing to know which one it is LoggerRef Logger() { return *this; } - bool ShouldLog(int Level) const; - inline operator bool() const { return SpdLogger != nullptr; } + bool ShouldLog(logging::LogLevel Level) const; + inline operator bool() const { return m_Logger != nullptr; } + + inline logging::Logger* operator->() const { return m_Logger; } + inline logging::Logger& operator*() const { return *m_Logger; } - void SetLogLevel(logging::level::LogLevel NewLogLevel); - logging::level::LogLevel GetLogLevel(); + void SetLogLevel(logging::LogLevel NewLogLevel); + logging::LogLevel GetLogLevel(); + void Flush(); - spdlog::logger* SpdLogger = nullptr; +private: + logging::Logger* m_Logger = nullptr; }; } // namespace zen diff --git a/src/zencore/include/zencore/logging.h b/src/zencore/include/zencore/logging.h index 74a44d028..4b593c19e 100644 --- a/src/zencore/include/zencore/logging.h +++ b/src/zencore/include/zencore/logging.h @@ -9,16 +9,9 @@ #if ZEN_PLATFORM_WINDOWS # define ZEN_LOG_SECTION(Id) ZEN_DATA_SECTION(Id) -# pragma section(".zlog$f", read) # pragma section(".zlog$l", read) -# pragma section(".zlog$m", read) -# pragma section(".zlog$s", read) -# define ZEN_DECLARE_FUNCTION static constinit ZEN_LOG_SECTION(".zlog$f") char FuncName[] = __FUNCTION__; -# define ZEN_LOG_FUNCNAME FuncName #else # define ZEN_LOG_SECTION(Id) -# define ZEN_DECLARE_FUNCTION -# define ZEN_LOG_FUNCNAME static_cast(__func__) #endif namespace zen::logging { @@ -37,34 +30,29 @@ LoggerRef ErrorLog(); void SetErrorLog(std::string_view LoggerId); LoggerRef Get(std::string_view Name); -void ConfigureLogLevels(level::LogLevel Level, std::string_view Loggers); +void ConfigureLogLevels(LogLevel Level, std::string_view Loggers); void RefreshLogLevels(); -void RefreshLogLevels(level::LogLevel DefaultLevel); - +void RefreshLogLevels(LogLevel DefaultLevel); + +/** LogCategory allows for the creation of log categories that can be used with + * the logging macros just like a logger reference. The main purpose of this is + * to allow for static log categories in global scope where we can't actually + * go ahead and instantiate a logger immediately because the logging system may + * not be initialized yet. + */ struct LogCategory { - inline LogCategory(std::string_view InCategory) : CategoryName(InCategory) {} - - inline zen::LoggerRef Logger() - { - if (LoggerRef) - { - return LoggerRef; - } + inline LogCategory(std::string_view InCategory) : m_CategoryName(InCategory) {} - LoggerRef = zen::logging::Get(CategoryName); - return LoggerRef; - } + LoggerRef Logger(); - std::string CategoryName; - zen::LoggerRef LoggerRef; +private: + std::string m_CategoryName; + LoggerRef m_LoggerRef; }; -void EmitConsoleLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args); -void EmitLogMessage(LoggerRef& Logger, int LogLevel, std::string_view Message); -void EmitLogMessage(LoggerRef& Logger, const SourceLocation& Location, int LogLevel, std::string_view Message); -void EmitLogMessage(LoggerRef& Logger, int LogLevel, std::string_view Format, fmt::format_args Args); -void EmitLogMessage(LoggerRef& Logger, const SourceLocation& Location, int LogLevel, std::string_view Format, fmt::format_args Args); +void EmitConsoleLogMessage(const LogPoint& Lp, fmt::format_args Args); +void EmitLogMessage(LoggerRef& Logger, const LogPoint& Lp, fmt::format_args Args); template auto @@ -79,15 +67,14 @@ namespace zen { extern LoggerRef TheDefaultLogger; -inline LoggerRef -Log() -{ - if (TheDefaultLogger) - { - return TheDefaultLogger; - } - return zen::logging::ConsoleLog(); -} +/** + * This is the default logger, which any ZEN_INFO et al will get if there's + * no Log() function declared in the current scope. + * + * Typically, classes which want to log to its own channel will declare a Log() + * member function which returns a LoggerRef created at construction time. + */ +LoggerRef Log(); using logging::ConsoleLog; using logging::ErrorLog; @@ -98,12 +85,6 @@ using zen::ConsoleLog; using zen::ErrorLog; using zen::Log; -inline consteval bool -LogIsErrorLevel(int LogLevel) -{ - return (LogLevel == zen::logging::level::Err || LogLevel == zen::logging::level::Critical); -}; - #if ZEN_BUILD_DEBUG # define ZEN_CHECK_FORMAT_STRING(fmtstr, ...) \ while (false) \ @@ -117,75 +98,66 @@ LogIsErrorLevel(int LogLevel) } #endif -#define ZEN_LOG_WITH_LOCATION(InLogger, InLevel, fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - ZEN_DECLARE_FUNCTION \ - static constinit ZEN_LOG_SECTION(".zlog$s") char FileName[] = __FILE__; \ - static constinit ZEN_LOG_SECTION(".zlog$m") char FormatString[] = fmtstr; \ - static constinit ZEN_LOG_SECTION(".zlog$l") zen::logging::SourceLocation Location{FileName, __LINE__, ZEN_LOG_FUNCNAME}; \ - zen::LoggerRef Logger = InLogger; \ - ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ - if (Logger.ShouldLog(InLevel)) \ - { \ - zen::logging::EmitLogMessage(Logger, \ - Location, \ - InLevel, \ - std::string_view(FormatString, sizeof FormatString - 1), \ - zen::logging::LogCaptureArguments(__VA_ARGS__)); \ - } \ +#define ZEN_LOG_WITH_LOCATION(InLogger, InLevel, fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + static constinit ZEN_LOG_SECTION(".zlog$l") \ + zen::logging::LogPoint LogPoint{zen::logging::SourceLocation{__FILE__, __LINE__}, InLevel, std::string_view(fmtstr)}; \ + zen::LoggerRef Logger = InLogger; \ + ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ + if (Logger.ShouldLog(InLevel)) \ + { \ + zen::logging::EmitLogMessage(Logger, LogPoint, zen::logging::LogCaptureArguments(__VA_ARGS__)); \ + } \ } while (false); -#define ZEN_LOG(InLogger, InLevel, fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - static constinit ZEN_LOG_SECTION(".zlog$m") char FormatString[] = fmtstr; \ - zen::LoggerRef Logger = InLogger; \ - ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ - if (Logger.ShouldLog(InLevel)) \ - { \ - zen::logging::EmitLogMessage(Logger, \ - InLevel, \ - std::string_view(FormatString, sizeof FormatString - 1), \ - zen::logging::LogCaptureArguments(__VA_ARGS__)); \ - } \ +#define ZEN_LOG(InLogger, InLevel, fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + static constinit ZEN_LOG_SECTION(".zlog$l") zen::logging::LogPoint LogPoint{{}, InLevel, std::string_view(fmtstr)}; \ + zen::LoggerRef Logger = InLogger; \ + ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ + if (Logger.ShouldLog(InLevel)) \ + { \ + zen::logging::EmitLogMessage(Logger, LogPoint, zen::logging::LogCaptureArguments(__VA_ARGS__)); \ + } \ } while (false); #define ZEN_DEFINE_LOG_CATEGORY_STATIC(Category, Name) \ static zen::logging::LogCategory Category { Name } -#define ZEN_LOG_TRACE(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::level::Trace, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_DEBUG(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::level::Debug, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_INFO(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::level::Info, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_WARN(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::level::Warn, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_ERROR(Category, fmtstr, ...) ZEN_LOG_WITH_LOCATION(Category.Logger(), zen::logging::level::Err, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_CRITICAL(Category, fmtstr, ...) \ - ZEN_LOG_WITH_LOCATION(Category.Logger(), zen::logging::level::Critical, fmtstr, ##__VA_ARGS__) - -#define ZEN_TRACE(fmtstr, ...) ZEN_LOG(Log(), zen::logging::level::Trace, fmtstr, ##__VA_ARGS__) -#define ZEN_DEBUG(fmtstr, ...) ZEN_LOG(Log(), zen::logging::level::Debug, fmtstr, ##__VA_ARGS__) -#define ZEN_INFO(fmtstr, ...) ZEN_LOG(Log(), zen::logging::level::Info, fmtstr, ##__VA_ARGS__) -#define ZEN_WARN(fmtstr, ...) ZEN_LOG(Log(), zen::logging::level::Warn, fmtstr, ##__VA_ARGS__) -#define ZEN_ERROR(fmtstr, ...) ZEN_LOG_WITH_LOCATION(Log(), zen::logging::level::Err, fmtstr, ##__VA_ARGS__) -#define ZEN_CRITICAL(fmtstr, ...) ZEN_LOG_WITH_LOCATION(Log(), zen::logging::level::Critical, fmtstr, ##__VA_ARGS__) - -#define ZEN_CONSOLE_LOG(InLevel, fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ - zen::logging::EmitConsoleLogMessage(InLevel, fmtstr, zen::logging::LogCaptureArguments(__VA_ARGS__)); \ +#define ZEN_LOG_TRACE(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::Trace, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_DEBUG(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::Debug, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_INFO(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::Info, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_WARN(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::Warn, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_ERROR(Category, fmtstr, ...) ZEN_LOG_WITH_LOCATION(Category.Logger(), zen::logging::Err, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_CRITICAL(Category, fmtstr, ...) ZEN_LOG_WITH_LOCATION(Category.Logger(), zen::logging::Critical, fmtstr, ##__VA_ARGS__) + +#define ZEN_TRACE(fmtstr, ...) ZEN_LOG(Log(), zen::logging::Trace, fmtstr, ##__VA_ARGS__) +#define ZEN_DEBUG(fmtstr, ...) ZEN_LOG(Log(), zen::logging::Debug, fmtstr, ##__VA_ARGS__) +#define ZEN_INFO(fmtstr, ...) ZEN_LOG(Log(), zen::logging::Info, fmtstr, ##__VA_ARGS__) +#define ZEN_WARN(fmtstr, ...) ZEN_LOG(Log(), zen::logging::Warn, fmtstr, ##__VA_ARGS__) +#define ZEN_ERROR(fmtstr, ...) ZEN_LOG_WITH_LOCATION(Log(), zen::logging::Err, fmtstr, ##__VA_ARGS__) +#define ZEN_CRITICAL(fmtstr, ...) ZEN_LOG_WITH_LOCATION(Log(), zen::logging::Critical, fmtstr, ##__VA_ARGS__) + +#define ZEN_CONSOLE_LOG(InLevel, fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + static constinit ZEN_LOG_SECTION(".zlog$l") zen::logging::LogPoint LogPoint{{}, InLevel, std::string_view(fmtstr)}; \ + ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ + zen::logging::EmitConsoleLogMessage(LogPoint, zen::logging::LogCaptureArguments(__VA_ARGS__)); \ } while (false) -#define ZEN_CONSOLE(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_TRACE(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Trace, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_DEBUG(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Debug, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_INFO(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_WARN(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Warn, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_ERROR(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Err, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_CRITICAL(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Critical, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_TRACE(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Trace, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_DEBUG(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Debug, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_INFO(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_WARN(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Warn, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_ERROR(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Err, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_CRITICAL(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Critical, fmtstr, ##__VA_ARGS__) ////////////////////////////////////////////////////////////////////////// @@ -240,28 +212,28 @@ std::string_view EmitActivitiesForLogging(StringBuilderBase& OutString); #define ZEN_LOG_SCOPE(...) ScopedLazyActivity $Activity##__LINE__([&](StringBuilderBase& Out) { Out << fmt::format(__VA_ARGS__); }) -#define ZEN_SCOPED_WARN(fmtstr, ...) \ - do \ - { \ - ExtendableStringBuilder<256> ScopeString; \ - const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ - ZEN_LOG(Log(), zen::logging::level::Warn, fmtstr "{}", ##__VA_ARGS__, Scopes); \ +#define ZEN_SCOPED_WARN(fmtstr, ...) \ + do \ + { \ + ExtendableStringBuilder<256> ScopeString; \ + const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ + ZEN_LOG(Log(), zen::logging::Warn, fmtstr "{}", ##__VA_ARGS__, Scopes); \ } while (false) -#define ZEN_SCOPED_ERROR(fmtstr, ...) \ - do \ - { \ - ExtendableStringBuilder<256> ScopeString; \ - const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ - ZEN_LOG_WITH_LOCATION(Log(), zen::logging::level::Err, fmtstr "{}", ##__VA_ARGS__, Scopes); \ +#define ZEN_SCOPED_ERROR(fmtstr, ...) \ + do \ + { \ + ExtendableStringBuilder<256> ScopeString; \ + const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ + ZEN_LOG_WITH_LOCATION(Log(), zen::logging::Err, fmtstr "{}", ##__VA_ARGS__, Scopes); \ } while (false) -#define ZEN_SCOPED_CRITICAL(fmtstr, ...) \ - do \ - { \ - ExtendableStringBuilder<256> ScopeString; \ - const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ - ZEN_LOG_WITH_LOCATION(Log(), zen::logging::level::Critical, fmtstr "{}", ##__VA_ARGS__, Scopes); \ +#define ZEN_SCOPED_CRITICAL(fmtstr, ...) \ + do \ + { \ + ExtendableStringBuilder<256> ScopeString; \ + const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ + ZEN_LOG_WITH_LOCATION(Log(), zen::logging::Critical, fmtstr "{}", ##__VA_ARGS__, Scopes); \ } while (false) ScopedActivityBase* GetThreadActivity(); diff --git a/src/zencore/include/zencore/logging/ansicolorsink.h b/src/zencore/include/zencore/logging/ansicolorsink.h new file mode 100644 index 000000000..9f859e8d7 --- /dev/null +++ b/src/zencore/include/zencore/logging/ansicolorsink.h @@ -0,0 +1,26 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +namespace zen::logging { + +class AnsiColorStdoutSink : public Sink +{ +public: + AnsiColorStdoutSink(); + ~AnsiColorStdoutSink() override; + + void Log(const LogMessage& Msg) override; + void Flush() override; + void SetFormatter(std::unique_ptr InFormatter) override; + +private: + struct Impl; + std::unique_ptr m_Impl; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/asyncsink.h b/src/zencore/include/zencore/logging/asyncsink.h new file mode 100644 index 000000000..c49a1ccce --- /dev/null +++ b/src/zencore/include/zencore/logging/asyncsink.h @@ -0,0 +1,30 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include + +namespace zen::logging { + +class AsyncSink : public Sink +{ +public: + explicit AsyncSink(std::vector InSinks); + ~AsyncSink() override; + + AsyncSink(const AsyncSink&) = delete; + AsyncSink& operator=(const AsyncSink&) = delete; + + void Log(const LogMessage& Msg) override; + void Flush() override; + void SetFormatter(std::unique_ptr InFormatter) override; + +private: + struct Impl; + std::unique_ptr m_Impl; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/formatter.h b/src/zencore/include/zencore/logging/formatter.h new file mode 100644 index 000000000..11904d71d --- /dev/null +++ b/src/zencore/include/zencore/logging/formatter.h @@ -0,0 +1,20 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +#include + +namespace zen::logging { + +class Formatter +{ +public: + virtual ~Formatter() = default; + virtual void Format(const LogMessage& Msg, MemoryBuffer& Dest) = 0; + virtual std::unique_ptr Clone() const = 0; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/helpers.h b/src/zencore/include/zencore/logging/helpers.h new file mode 100644 index 000000000..ce021e1a5 --- /dev/null +++ b/src/zencore/include/zencore/logging/helpers.h @@ -0,0 +1,122 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +#include +#include +#include + +namespace zen::logging::helpers { + +inline void +AppendStringView(std::string_view Sv, MemoryBuffer& Dest) +{ + Dest.append(Sv.data(), Sv.data() + Sv.size()); +} + +inline void +AppendInt(int N, MemoryBuffer& Dest) +{ + fmt::format_int Formatted(N); + Dest.append(Formatted.data(), Formatted.data() + Formatted.size()); +} + +inline void +Pad2(int N, MemoryBuffer& Dest) +{ + if (N >= 0 && N < 100) + { + Dest.push_back(static_cast('0' + N / 10)); + Dest.push_back(static_cast('0' + N % 10)); + } + else + { + fmt::format_int Formatted(N); + Dest.append(Formatted.data(), Formatted.data() + Formatted.size()); + } +} + +inline void +Pad3(uint32_t N, MemoryBuffer& Dest) +{ + if (N < 1000) + { + Dest.push_back(static_cast('0' + N / 100)); + Dest.push_back(static_cast('0' + (N / 10) % 10)); + Dest.push_back(static_cast('0' + N % 10)); + } + else + { + AppendInt(static_cast(N), Dest); + } +} + +inline void +PadUint(size_t N, unsigned int Width, MemoryBuffer& Dest) +{ + fmt::format_int Formatted(N); + auto StrLen = static_cast(Formatted.size()); + if (Width > StrLen) + { + for (unsigned int Pad = 0; Pad < Width - StrLen; ++Pad) + { + Dest.push_back('0'); + } + } + Dest.append(Formatted.data(), Formatted.data() + Formatted.size()); +} + +template +inline ToDuration +TimeFraction(std::chrono::system_clock::time_point Tp) +{ + using std::chrono::duration_cast; + using std::chrono::seconds; + auto Duration = Tp.time_since_epoch(); + auto Secs = duration_cast(Duration); + return duration_cast(Duration) - duration_cast(Secs); +} + +inline std::tm +SafeLocaltime(std::time_t Time) +{ + std::tm Result{}; +#if defined(_WIN32) + localtime_s(&Result, &Time); +#else + localtime_r(&Time, &Result); +#endif + return Result; +} + +inline const char* +ShortFilename(const char* Path) +{ + if (Path == nullptr) + { + return Path; + } + + const char* It = Path; + const char* LastSep = Path; + while (*It) + { + if (*It == '/' || *It == '\\') + { + LastSep = It + 1; + } + ++It; + } + return LastSep; +} + +inline std::string_view +LevelToShortString(LogLevel Level) +{ + return ToStringView(Level); +} + +} // namespace zen::logging::helpers diff --git a/src/zencore/include/zencore/logging/logger.h b/src/zencore/include/zencore/logging/logger.h new file mode 100644 index 000000000..39d1139a5 --- /dev/null +++ b/src/zencore/include/zencore/logging/logger.h @@ -0,0 +1,63 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include +#include + +namespace zen::logging { + +class ErrorHandler +{ +public: + virtual ~ErrorHandler() = default; + virtual void HandleError(const std::string_view& Msg) = 0; +}; + +class Logger : public RefCounted +{ +public: + Logger(std::string_view InName, SinkPtr InSink); + Logger(std::string_view InName, std::span InSinks); + ~Logger(); + + Logger(const Logger&) = delete; + Logger& operator=(const Logger&) = delete; + + void Log(const LogPoint& Point, fmt::format_args Args); + + bool ShouldLog(LogLevel InLevel) const { return InLevel >= m_Level.load(std::memory_order_relaxed); } + + void SetLevel(LogLevel InLevel) { m_Level.store(InLevel, std::memory_order_relaxed); } + LogLevel GetLevel() const { return m_Level.load(std::memory_order_relaxed); } + + void SetFlushLevel(LogLevel InLevel) { m_FlushLevel.store(InLevel, std::memory_order_relaxed); } + LogLevel GetFlushLevel() const { return m_FlushLevel.load(std::memory_order_relaxed); } + + std::string_view Name() const; + + void SetSinks(std::vector InSinks); + void AddSink(SinkPtr InSink); + + void SetFormatter(std::unique_ptr InFormatter); + + void SetErrorHandler(ErrorHandler* Handler); + + void Flush(); + + Ref Clone(std::string_view NewName) const; + +private: + void SinkIt(const LogMessage& Msg); + void FlushIfNeeded(LogLevel InLevel); + + struct Impl; + std::unique_ptr m_Impl; + std::atomic m_Level{Info}; + std::atomic m_FlushLevel{Off}; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/logmsg.h b/src/zencore/include/zencore/logging/logmsg.h new file mode 100644 index 000000000..1d8b6b1b7 --- /dev/null +++ b/src/zencore/include/zencore/logging/logmsg.h @@ -0,0 +1,66 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include + +namespace zen::logging { + +using LogClock = std::chrono::system_clock; + +struct LogMessage +{ + LogMessage() = default; + + LogMessage(const LogPoint& InPoint, std::string_view InLoggerName, std::string_view InPayload) + : m_LoggerName(InLoggerName) + , m_Level(InPoint.Level) + , m_Time(LogClock::now()) + , m_Source(InPoint.Location) + , m_Payload(InPayload) + , m_Point(&InPoint) + { + } + + std::string_view GetPayload() const { return m_Payload; } + int GetThreadId() const { return m_ThreadId; } + LogClock::time_point GetTime() const { return m_Time; } + LogLevel GetLevel() const { return m_Level; } + std::string_view GetLoggerName() const { return m_LoggerName; } + const SourceLocation& GetSource() const { return m_Source; } + const LogPoint& GetLogPoint() const { return *m_Point; } + + void SetThreadId(int InThreadId) { m_ThreadId = InThreadId; } + void SetPayload(std::string_view InPayload) { m_Payload = InPayload; } + void SetLoggerName(std::string_view InName) { m_LoggerName = InName; } + void SetLevel(LogLevel InLevel) { m_Level = InLevel; } + void SetTime(LogClock::time_point InTime) { m_Time = InTime; } + void SetSource(const SourceLocation& InSource) { m_Source = InSource; } + + mutable size_t ColorRangeStart = 0; + mutable size_t ColorRangeEnd = 0; + +private: + static constexpr LogPoint s_DefaultPoints[LogLevelCount] = { + {{}, Trace, {}}, + {{}, Debug, {}}, + {{}, Info, {}}, + {{}, Warn, {}}, + {{}, Err, {}}, + {{}, Critical, {}}, + {{}, Off, {}}, + }; + + std::string_view m_LoggerName; + LogLevel m_Level = Off; + std::chrono::system_clock::time_point m_Time; + SourceLocation m_Source; + std::string_view m_Payload; + const LogPoint* m_Point = &s_DefaultPoints[Off]; + int m_ThreadId = 0; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/memorybuffer.h b/src/zencore/include/zencore/logging/memorybuffer.h new file mode 100644 index 000000000..cd0ff324f --- /dev/null +++ b/src/zencore/include/zencore/logging/memorybuffer.h @@ -0,0 +1,11 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +namespace zen::logging { + +using MemoryBuffer = fmt::basic_memory_buffer; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/messageonlyformatter.h b/src/zencore/include/zencore/logging/messageonlyformatter.h new file mode 100644 index 000000000..ce25fe9a6 --- /dev/null +++ b/src/zencore/include/zencore/logging/messageonlyformatter.h @@ -0,0 +1,22 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen::logging { + +class MessageOnlyFormatter : public Formatter +{ +public: + void Format(const LogMessage& Msg, MemoryBuffer& Dest) override + { + helpers::AppendStringView(Msg.GetPayload(), Dest); + Dest.push_back('\n'); + } + + std::unique_ptr Clone() const override { return std::make_unique(); } +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/msvcsink.h b/src/zencore/include/zencore/logging/msvcsink.h new file mode 100644 index 000000000..48ea1b915 --- /dev/null +++ b/src/zencore/include/zencore/logging/msvcsink.h @@ -0,0 +1,30 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#if ZEN_PLATFORM_WINDOWS + +# include + +namespace zen::logging { + +class MsvcSink : public Sink +{ +public: + MsvcSink(); + ~MsvcSink() override = default; + + void Log(const LogMessage& Msg) override; + void Flush() override; + void SetFormatter(std::unique_ptr InFormatter) override; + +private: + std::mutex m_Mutex; + std::unique_ptr m_Formatter; +}; + +} // namespace zen::logging + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/src/zencore/include/zencore/logging/nullsink.h b/src/zencore/include/zencore/logging/nullsink.h new file mode 100644 index 000000000..7ac5677c6 --- /dev/null +++ b/src/zencore/include/zencore/logging/nullsink.h @@ -0,0 +1,17 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +namespace zen::logging { + +class NullSink : public Sink +{ +public: + void Log(const LogMessage& /*Msg*/) override {} + void Flush() override {} + void SetFormatter(std::unique_ptr /*InFormatter*/) override {} +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/registry.h b/src/zencore/include/zencore/logging/registry.h new file mode 100644 index 000000000..a4d3692d2 --- /dev/null +++ b/src/zencore/include/zencore/logging/registry.h @@ -0,0 +1,70 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace zen::logging { + +class Registry +{ +public: + using LogLevels = std::span>; + + static Registry& Instance(); + void Shutdown(); + + void Register(Ref InLogger); + void Drop(const std::string& Name); + Ref Get(const std::string& Name); + + void SetDefaultLogger(Ref InLogger); + Logger* DefaultLoggerRaw(); + Ref DefaultLogger(); + + void SetGlobalLevel(LogLevel Level); + LogLevel GetGlobalLevel() const; + void SetLevels(LogLevels Levels, LogLevel* DefaultLevel); + + void FlushAll(); + void FlushOn(LogLevel Level); + void FlushEvery(std::chrono::seconds Interval); + + // Change formatter on all registered loggers + void SetFormatter(std::unique_ptr InFormatter); + + // Apply function to all registered loggers. Note that the function will + // be called while the registry mutex is held, so it should be fast and + // not attempt to call back into the registry. + template + void ApplyAll(Func&& F) + { + ApplyAllImpl([](void* Ctx, Ref L) { (*static_cast*>(Ctx))(std::move(L)); }, &F); + } + + // Set error handler for all loggers in the registry. The handler is called + // if any logger encounters an error during logging or flushing. + // The caller must ensure the handler outlives the registry. + void SetErrorHandler(ErrorHandler* Handler); + +private: + void ApplyAllImpl(void (*Func)(void*, Ref), void* Context); + + Registry(); + ~Registry(); + + Registry(const Registry&) = delete; + Registry& operator=(const Registry&) = delete; + + struct Impl; + std::unique_ptr m_Impl; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/sink.h b/src/zencore/include/zencore/logging/sink.h new file mode 100644 index 000000000..172176a4e --- /dev/null +++ b/src/zencore/include/zencore/logging/sink.h @@ -0,0 +1,34 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +#include +#include + +namespace zen::logging { + +class Sink : public RefCounted +{ +public: + virtual ~Sink() = default; + + virtual void Log(const LogMessage& Msg) = 0; + virtual void Flush() = 0; + + virtual void SetFormatter(std::unique_ptr InFormatter) = 0; + + bool ShouldLog(LogLevel InLevel) const { return InLevel >= m_Level.load(std::memory_order_relaxed); } + void SetLevel(LogLevel InLevel) { m_Level.store(InLevel, std::memory_order_relaxed); } + LogLevel GetLevel() const { return m_Level.load(std::memory_order_relaxed); } + +protected: + std::atomic m_Level{Trace}; +}; + +using SinkPtr = Ref; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/tracesink.h b/src/zencore/include/zencore/logging/tracesink.h new file mode 100644 index 000000000..e63d838b4 --- /dev/null +++ b/src/zencore/include/zencore/logging/tracesink.h @@ -0,0 +1,23 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +namespace zen::logging { + +/** + * A logging sink that forwards log messages to the trace system. + * + * Work-in-progress, not fully implemented. + */ + +class TraceSink : public Sink +{ +public: + void Log(const LogMessage& Msg) override; + void Flush() override; + void SetFormatter(std::unique_ptr InFormatter) override; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h index faf1238b7..a4e33d69e 100644 --- a/src/zencore/include/zencore/sentryintegration.h +++ b/src/zencore/include/zencore/sentryintegration.h @@ -11,11 +11,9 @@ #if ZEN_USE_SENTRY -# include +# include -ZEN_THIRD_PARTY_INCLUDES_START -# include -ZEN_THIRD_PARTY_INCLUDES_END +# include namespace sentry { @@ -53,7 +51,7 @@ private: std::string m_SentryUserName; std::string m_SentryHostName; std::string m_SentryId; - std::shared_ptr m_SentryLogger; + Ref m_SentryLogger; }; } // namespace zen diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp index ebd68de09..099518637 100644 --- a/src/zencore/logging.cpp +++ b/src/zencore/logging.cpp @@ -2,208 +2,128 @@ #include "zencore/logging.h" +#include +#include +#include +#include +#include #include #include #include #include -ZEN_THIRD_PARTY_INCLUDES_START -#include -#include -#include -#include -ZEN_THIRD_PARTY_INCLUDES_END +#include #if ZEN_PLATFORM_WINDOWS # pragma section(".zlog$a", read) -# pragma section(".zlog$f", read) -# pragma section(".zlog$m", read) -# pragma section(".zlog$s", read) +# pragma section(".zlog$l", read) # pragma section(".zlog$z", read) #endif namespace zen { -// We shadow the underlying spdlog default logger, in order to avoid a bunch of overhead LoggerRef TheDefaultLogger; +LoggerRef +Log() +{ + if (TheDefaultLogger) + { + return TheDefaultLogger; + } + return zen::logging::ConsoleLog(); +} + } // namespace zen namespace zen::logging { -using MemoryBuffer_t = fmt::basic_memory_buffer; - -struct LoggingContext -{ - inline LoggingContext(); - inline ~LoggingContext(); - - zen::logging::MemoryBuffer_t MessageBuffer; - - inline std::string_view Message() const { return std::string_view(MessageBuffer.data(), MessageBuffer.size()); } -}; +////////////////////////////////////////////////////////////////////////// -LoggingContext::LoggingContext() +LoggerRef +LogCategory::Logger() { -} + // This should be thread safe since zen::logging::Get() will return + // the same logger instance for the same category name. Also the + // LoggerRef is simply a pointer. + if (!m_LoggerRef) + { + m_LoggerRef = zen::logging::Get(m_CategoryName); + } -LoggingContext::~LoggingContext() -{ + return m_LoggerRef; } -////////////////////////////////////////////////////////////////////////// - static inline bool -IsErrorLevel(int LogLevel) +IsErrorLevel(LogLevel InLevel) { - return (LogLevel == zen::logging::level::Err || LogLevel == zen::logging::level::Critical); + return (InLevel == Err || InLevel == Critical); }; -static_assert(sizeof(spdlog::source_loc) == sizeof(SourceLocation)); -static_assert(offsetof(spdlog::source_loc, filename) == offsetof(SourceLocation, filename)); -static_assert(offsetof(spdlog::source_loc, line) == offsetof(SourceLocation, line)); -static_assert(offsetof(spdlog::source_loc, funcname) == offsetof(SourceLocation, funcname)); - void -EmitLogMessage(LoggerRef& Logger, int LogLevel, const std::string_view Message) +EmitLogMessage(LoggerRef& Logger, const LogPoint& Lp, fmt::format_args Args) { ZEN_MEMSCOPE(ELLMTag::Logging); - const spdlog::level::level_enum InLevel = (spdlog::level::level_enum)LogLevel; - Logger.SpdLogger->log(InLevel, Message); - if (IsErrorLevel(LogLevel)) - { - if (LoggerRef ErrLogger = zen::logging::ErrorLog()) - { - ErrLogger.SpdLogger->log(InLevel, Message); - } - } -} -void -EmitLogMessage(LoggerRef& Logger, int LogLevel, std::string_view Format, fmt::format_args Args) -{ - ZEN_MEMSCOPE(ELLMTag::Logging); - zen::logging::LoggingContext LogCtx; - fmt::vformat_to(fmt::appender(LogCtx.MessageBuffer), Format, Args); - zen::logging::EmitLogMessage(Logger, LogLevel, LogCtx.Message()); -} + Logger->Log(Lp, Args); -void -EmitLogMessage(LoggerRef& Logger, const SourceLocation& InLocation, int LogLevel, const std::string_view Message) -{ - ZEN_MEMSCOPE(ELLMTag::Logging); - const spdlog::source_loc& Location = *reinterpret_cast(&InLocation); - const spdlog::level::level_enum InLevel = (spdlog::level::level_enum)LogLevel; - Logger.SpdLogger->log(Location, InLevel, Message); - if (IsErrorLevel(LogLevel)) + if (IsErrorLevel(Lp.Level)) { if (LoggerRef ErrLogger = zen::logging::ErrorLog()) { - ErrLogger.SpdLogger->log(Location, InLevel, Message); + ErrLogger->Log(Lp, Args); } } } void -EmitLogMessage(LoggerRef& Logger, const SourceLocation& InLocation, int LogLevel, std::string_view Format, fmt::format_args Args) -{ - ZEN_MEMSCOPE(ELLMTag::Logging); - zen::logging::LoggingContext LogCtx; - fmt::vformat_to(fmt::appender(LogCtx.MessageBuffer), Format, Args); - zen::logging::EmitLogMessage(Logger, InLocation, LogLevel, LogCtx.Message()); -} - -void -EmitConsoleLogMessage(int LogLevel, const std::string_view Message) -{ - ZEN_MEMSCOPE(ELLMTag::Logging); - const spdlog::level::level_enum InLevel = (spdlog::level::level_enum)LogLevel; - ConsoleLog().SpdLogger->log(InLevel, Message); -} - -#define ZEN_COLOR_YELLOW "\033[0;33m" -#define ZEN_COLOR_RED "\033[0;31m" -#define ZEN_BRIGHT_COLOR_RED "\033[1;31m" -#define ZEN_COLOR_RESET "\033[0m" - -void -EmitConsoleLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) +EmitConsoleLogMessage(const LogPoint& Lp, fmt::format_args Args) { ZEN_MEMSCOPE(ELLMTag::Logging); - zen::logging::LoggingContext LogCtx; - - // We are not using a format option for console which include log level since it would interfere with normal console output - - const spdlog::level::level_enum InLevel = (spdlog::level::level_enum)LogLevel; - switch (InLevel) - { - case spdlog::level::level_enum::warn: - fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_COLOR_YELLOW "Warning: " ZEN_COLOR_RESET); - break; - case spdlog::level::level_enum::err: - fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_BRIGHT_COLOR_RED "Error: " ZEN_COLOR_RESET); - break; - case spdlog::level::level_enum::critical: - fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_COLOR_RED "Critical: " ZEN_COLOR_RESET); - break; - default: - break; - } - fmt::vformat_to(fmt::appender(LogCtx.MessageBuffer), Format, Args); - zen::logging::EmitConsoleLogMessage(LogLevel, LogCtx.Message()); + ConsoleLog()->Log(Lp, Args); } } // namespace zen::logging -namespace zen::logging::level { +namespace zen::logging { -spdlog::level::level_enum -to_spdlog_level(LogLevel NewLogLevel) -{ - return static_cast((int)NewLogLevel); -} +constinit std::string_view LevelNames[] = {std::string_view("trace", 5), + std::string_view("debug", 5), + std::string_view("info", 4), + std::string_view("warning", 7), + std::string_view("error", 5), + std::string_view("critical", 8), + std::string_view("off", 3)}; LogLevel -to_logging_level(spdlog::level::level_enum NewLogLevel) -{ - return static_cast((int)NewLogLevel); -} - -constinit std::string_view LevelNames[] = {ZEN_LEVEL_NAME_TRACE, - ZEN_LEVEL_NAME_DEBUG, - ZEN_LEVEL_NAME_INFO, - ZEN_LEVEL_NAME_WARNING, - ZEN_LEVEL_NAME_ERROR, - ZEN_LEVEL_NAME_CRITICAL, - ZEN_LEVEL_NAME_OFF}; - -level::LogLevel ParseLogLevelString(std::string_view Name) { - for (int Level = 0; Level < level::LogLevelCount; ++Level) + for (int Level = 0; Level < LogLevelCount; ++Level) { if (LevelNames[Level] == Name) - return static_cast(Level); + { + return static_cast(Level); + } } if (Name == "warn") { - return level::Warn; + return Warn; } if (Name == "err") { - return level::Err; + return Err; } - return level::Off; + return Off; } std::string_view -ToStringView(level::LogLevel Level) +ToStringView(LogLevel Level) { - if (int(Level) < level::LogLevelCount) + if (int(Level) < LogLevelCount) { return LevelNames[int(Level)]; } @@ -211,17 +131,17 @@ ToStringView(level::LogLevel Level) return "None"; } -} // namespace zen::logging::level +} // namespace zen::logging ////////////////////////////////////////////////////////////////////////// namespace zen::logging { RwLock LogLevelsLock; -std::string LogLevels[level::LogLevelCount]; +std::string LogLevels[LogLevelCount]; void -ConfigureLogLevels(level::LogLevel Level, std::string_view Loggers) +ConfigureLogLevels(LogLevel Level, std::string_view Loggers) { ZEN_MEMSCOPE(ELLMTag::Logging); @@ -230,18 +150,18 @@ ConfigureLogLevels(level::LogLevel Level, std::string_view Loggers) } void -RefreshLogLevels(level::LogLevel* DefaultLevel) +RefreshLogLevels(LogLevel* DefaultLevel) { ZEN_MEMSCOPE(ELLMTag::Logging); - spdlog::details::registry::log_levels Levels; + std::vector> Levels; { RwLock::SharedLockScope _(LogLevelsLock); - for (int i = 0; i < level::LogLevelCount; ++i) + for (int i = 0; i < LogLevelCount; ++i) { - level::LogLevel CurrentLevel{i}; + LogLevel CurrentLevel{i}; std::string_view Spec = LogLevels[i]; @@ -260,24 +180,16 @@ RefreshLogLevels(level::LogLevel* DefaultLevel) Spec = {}; } - Levels[LoggerName] = to_spdlog_level(CurrentLevel); + Levels.emplace_back(std::move(LoggerName), CurrentLevel); } } } - if (DefaultLevel) - { - spdlog::level::level_enum SpdDefaultLevel = to_spdlog_level(*DefaultLevel); - spdlog::details::registry::instance().set_levels(Levels, &SpdDefaultLevel); - } - else - { - spdlog::details::registry::instance().set_levels(Levels, nullptr); - } + Registry::Instance().SetLevels(Levels, DefaultLevel); } void -RefreshLogLevels(level::LogLevel DefaultLevel) +RefreshLogLevels(LogLevel DefaultLevel) { RefreshLogLevels(&DefaultLevel); } @@ -289,15 +201,15 @@ RefreshLogLevels() } void -SetLogLevel(level::LogLevel NewLogLevel) +SetLogLevel(LogLevel NewLogLevel) { - spdlog::set_level(to_spdlog_level(NewLogLevel)); + Registry::Instance().SetGlobalLevel(NewLogLevel); } -level::LogLevel +LogLevel GetLogLevel() { - return level::to_logging_level(spdlog::get_level()); + return Registry::Instance().GetGlobalLevel(); } LoggerRef @@ -312,10 +224,10 @@ SetDefault(std::string_view NewDefaultLoggerId) { ZEN_MEMSCOPE(ELLMTag::Logging); - auto NewDefaultLogger = spdlog::get(std::string(NewDefaultLoggerId)); + Ref NewDefaultLogger = Registry::Instance().Get(std::string(NewDefaultLoggerId)); ZEN_ASSERT(NewDefaultLogger); - spdlog::set_default_logger(NewDefaultLogger); + Registry::Instance().SetDefaultLogger(NewDefaultLogger); TheDefaultLogger = LoggerRef(*NewDefaultLogger); } @@ -338,11 +250,11 @@ SetErrorLog(std::string_view NewErrorLoggerId) } else { - auto NewErrorLogger = spdlog::get(std::string(NewErrorLoggerId)); + Ref NewErrorLogger = Registry::Instance().Get(std::string(NewErrorLoggerId)); ZEN_ASSERT(NewErrorLogger); - TheErrorLogger = LoggerRef(*NewErrorLogger.get()); + TheErrorLogger = LoggerRef(*NewErrorLogger.Get()); } } @@ -353,39 +265,75 @@ Get(std::string_view Name) { ZEN_MEMSCOPE(ELLMTag::Logging); - std::shared_ptr Logger = spdlog::get(std::string(Name)); + Ref FoundLogger = Registry::Instance().Get(std::string(Name)); - if (!Logger) + if (!FoundLogger) { g_LoggerMutex.WithExclusiveLock([&] { - Logger = spdlog::get(std::string(Name)); + FoundLogger = Registry::Instance().Get(std::string(Name)); - if (!Logger) + if (!FoundLogger) { - Logger = Default().SpdLogger->clone(std::string(Name)); - spdlog::apply_logger_env_levels(Logger); - spdlog::register_logger(Logger); + FoundLogger = Default()->Clone(std::string(Name)); + Registry::Instance().Register(FoundLogger); } }); } - return *Logger; + return *FoundLogger; } -std::once_flag ConsoleInitFlag; -std::shared_ptr ConLogger; +std::once_flag ConsoleInitFlag; +Ref ConLogger; void SuppressConsoleLog() { + ZEN_MEMSCOPE(ELLMTag::Logging); + if (ConLogger) { - spdlog::drop("console"); + Registry::Instance().Drop("console"); ConLogger = {}; } - ConLogger = spdlog::null_logger_mt("console"); + + SinkPtr NullSinkPtr(new NullSink()); + ConLogger = Ref(new Logger("console", std::vector{NullSinkPtr})); + Registry::Instance().Register(ConLogger); } +#define ZEN_COLOR_YELLOW "\033[0;33m" +#define ZEN_COLOR_RED "\033[0;31m" +#define ZEN_BRIGHT_COLOR_RED "\033[1;31m" +#define ZEN_COLOR_RESET "\033[0m" + +class ConsoleFormatter : public Formatter +{ +public: + void Format(const LogMessage& Msg, MemoryBuffer& Dest) override + { + switch (Msg.GetLevel()) + { + case Warn: + fmt::format_to(fmt::appender(Dest), ZEN_COLOR_YELLOW "Warning: " ZEN_COLOR_RESET); + break; + case Err: + fmt::format_to(fmt::appender(Dest), ZEN_BRIGHT_COLOR_RED "Error: " ZEN_COLOR_RESET); + break; + case Critical: + fmt::format_to(fmt::appender(Dest), ZEN_COLOR_RED "Critical: " ZEN_COLOR_RESET); + break; + default: + break; + } + + helpers::AppendStringView(Msg.GetPayload(), Dest); + Dest.push_back('\n'); + } + + std::unique_ptr Clone() const override { return std::make_unique(); } +}; + LoggerRef ConsoleLog() { @@ -394,10 +342,10 @@ ConsoleLog() std::call_once(ConsoleInitFlag, [&] { if (!ConLogger) { - ConLogger = spdlog::stdout_color_mt("console"); - spdlog::apply_logger_env_levels(ConLogger); - - ConLogger->set_pattern("%v"); + SinkPtr ConsoleSink(new AnsiColorStdoutSink()); + ConsoleSink->SetFormatter(std::make_unique()); + ConLogger = Ref(new Logger("console", std::vector{ConsoleSink})); + Registry::Instance().Register(ConLogger); } }); @@ -407,9 +355,11 @@ ConsoleLog() void ResetConsoleLog() { + ZEN_MEMSCOPE(ELLMTag::Logging); + LoggerRef ConLog = ConsoleLog(); - ConLog.SpdLogger->set_pattern("%v"); + ConLog->SetFormatter(std::make_unique()); } void @@ -417,13 +367,15 @@ InitializeLogging() { ZEN_MEMSCOPE(ELLMTag::Logging); - TheDefaultLogger = *spdlog::default_logger_raw(); + TheDefaultLogger = *Registry::Instance().DefaultLoggerRaw(); } void ShutdownLogging() { - spdlog::shutdown(); + ZEN_MEMSCOPE(ELLMTag::Logging); + + Registry::Instance().Shutdown(); TheDefaultLogger = {}; } @@ -457,7 +409,7 @@ EnableVTMode() void FlushLogging() { - spdlog::details::registry::instance().flush_all(); + Registry::Instance().FlushAll(); } } // namespace zen::logging @@ -465,21 +417,27 @@ FlushLogging() namespace zen { bool -LoggerRef::ShouldLog(int Level) const +LoggerRef::ShouldLog(logging::LogLevel Level) const { - return SpdLogger->should_log(static_cast(Level)); + return m_Logger->ShouldLog(Level); } void -LoggerRef::SetLogLevel(logging::level::LogLevel NewLogLevel) +LoggerRef::SetLogLevel(logging::LogLevel NewLogLevel) { - SpdLogger->set_level(to_spdlog_level(NewLogLevel)); + m_Logger->SetLevel(NewLogLevel); } -logging::level::LogLevel +logging::LogLevel LoggerRef::GetLogLevel() { - return logging::level::to_logging_level(SpdLogger->level()); + return m_Logger->GetLevel(); +} + +void +LoggerRef::Flush() +{ + m_Logger->Flush(); } thread_local ScopedActivityBase* t_ScopeStack = nullptr; diff --git a/src/zencore/logging/ansicolorsink.cpp b/src/zencore/logging/ansicolorsink.cpp new file mode 100644 index 000000000..9b9959862 --- /dev/null +++ b/src/zencore/logging/ansicolorsink.cpp @@ -0,0 +1,178 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include +#include +#include + +#include +#include + +namespace zen::logging { + +// Default formatter replicating spdlog's %+ pattern: +// [YYYY-MM-DD HH:MM:SS.mmm] [logger_name] [level] message\n +class DefaultConsoleFormatter : public Formatter +{ +public: + void Format(const LogMessage& Msg, MemoryBuffer& Dest) override + { + // timestamp + auto Secs = std::chrono::duration_cast(Msg.GetTime().time_since_epoch()); + if (Secs != m_LastLogSecs) + { + m_LastLogSecs = Secs; + m_CachedLocalTm = helpers::SafeLocaltime(LogClock::to_time_t(Msg.GetTime())); + } + + Dest.push_back('['); + helpers::AppendInt(m_CachedLocalTm.tm_year + 1900, Dest); + Dest.push_back('-'); + helpers::Pad2(m_CachedLocalTm.tm_mon + 1, Dest); + Dest.push_back('-'); + helpers::Pad2(m_CachedLocalTm.tm_mday, Dest); + Dest.push_back(' '); + helpers::Pad2(m_CachedLocalTm.tm_hour, Dest); + Dest.push_back(':'); + helpers::Pad2(m_CachedLocalTm.tm_min, Dest); + Dest.push_back(':'); + helpers::Pad2(m_CachedLocalTm.tm_sec, Dest); + Dest.push_back('.'); + auto Millis = helpers::TimeFraction(Msg.GetTime()); + helpers::Pad3(static_cast(Millis.count()), Dest); + Dest.push_back(']'); + Dest.push_back(' '); + + // logger name + if (Msg.GetLoggerName().size() > 0) + { + Dest.push_back('['); + helpers::AppendStringView(Msg.GetLoggerName(), Dest); + Dest.push_back(']'); + Dest.push_back(' '); + } + + // level (colored range) + Dest.push_back('['); + Msg.ColorRangeStart = Dest.size(); + helpers::AppendStringView(helpers::LevelToShortString(Msg.GetLevel()), Dest); + Msg.ColorRangeEnd = Dest.size(); + Dest.push_back(']'); + Dest.push_back(' '); + + // message + helpers::AppendStringView(Msg.GetPayload(), Dest); + Dest.push_back('\n'); + } + + std::unique_ptr Clone() const override { return std::make_unique(); } + +private: + std::chrono::seconds m_LastLogSecs{0}; + std::tm m_CachedLocalTm{}; +}; + +static constexpr std::string_view s_Reset = "\033[m"; + +static std::string_view +GetColorForLevel(LogLevel InLevel) +{ + using namespace std::string_view_literals; + switch (InLevel) + { + case Trace: + return "\033[37m"sv; // white + case Debug: + return "\033[36m"sv; // cyan + case Info: + return "\033[32m"sv; // green + case Warn: + return "\033[33m\033[1m"sv; // bold yellow + case Err: + return "\033[31m\033[1m"sv; // bold red + case Critical: + return "\033[1m\033[41m"sv; // bold on red background + default: + return s_Reset; + } +} + +struct AnsiColorStdoutSink::Impl +{ + Impl() : m_Formatter(std::make_unique()) {} + + void Log(const LogMessage& Msg) + { + std::lock_guard Lock(m_Mutex); + + MemoryBuffer Formatted; + m_Formatter->Format(Msg, Formatted); + + if (Msg.ColorRangeEnd > Msg.ColorRangeStart) + { + // Print pre-color range + fwrite(Formatted.data(), 1, Msg.ColorRangeStart, m_File); + + // Print color + std::string_view Color = GetColorForLevel(Msg.GetLevel()); + fwrite(Color.data(), 1, Color.size(), m_File); + + // Print colored range + fwrite(Formatted.data() + Msg.ColorRangeStart, 1, Msg.ColorRangeEnd - Msg.ColorRangeStart, m_File); + + // Reset color + fwrite(s_Reset.data(), 1, s_Reset.size(), m_File); + + // Print remainder + fwrite(Formatted.data() + Msg.ColorRangeEnd, 1, Formatted.size() - Msg.ColorRangeEnd, m_File); + } + else + { + fwrite(Formatted.data(), 1, Formatted.size(), m_File); + } + + fflush(m_File); + } + + void Flush() + { + std::lock_guard Lock(m_Mutex); + fflush(m_File); + } + + void SetFormatter(std::unique_ptr InFormatter) + { + std::lock_guard Lock(m_Mutex); + m_Formatter = std::move(InFormatter); + } + +private: + std::mutex m_Mutex; + std::unique_ptr m_Formatter; + FILE* m_File = stdout; +}; + +AnsiColorStdoutSink::AnsiColorStdoutSink() : m_Impl(std::make_unique()) +{ +} + +AnsiColorStdoutSink::~AnsiColorStdoutSink() = default; + +void +AnsiColorStdoutSink::Log(const LogMessage& Msg) +{ + m_Impl->Log(Msg); +} + +void +AnsiColorStdoutSink::Flush() +{ + m_Impl->Flush(); +} + +void +AnsiColorStdoutSink::SetFormatter(std::unique_ptr InFormatter) +{ + m_Impl->SetFormatter(std::move(InFormatter)); +} + +} // namespace zen::logging diff --git a/src/zencore/logging/asyncsink.cpp b/src/zencore/logging/asyncsink.cpp new file mode 100644 index 000000000..02bf9f3ba --- /dev/null +++ b/src/zencore/logging/asyncsink.cpp @@ -0,0 +1,212 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include + +#include +#include +#include + +namespace zen::logging { + +struct AsyncLogMessage +{ + enum class Type : uint8_t + { + Log, + Flush, + Shutdown + }; + + Type MsgType = Type::Log; + + // Points to the LogPoint from upstream logging code. LogMessage guarantees + // this is always valid (either a static LogPoint from ZEN_LOG macros or one + // of the per-level default LogPoints). + const LogPoint* Point = nullptr; + + int ThreadId = 0; + std::string OwnedPayload; + std::string OwnedLoggerName; + std::chrono::system_clock::time_point Time; + + std::shared_ptr> FlushPromise; +}; + +struct AsyncSink::Impl +{ + explicit Impl(std::vector InSinks) : m_Sinks(std::move(InSinks)) + { + m_WorkerThread = std::thread([this]() { + zen::SetCurrentThreadName("AsyncLog"); + WorkerLoop(); + }); + } + + ~Impl() + { + AsyncLogMessage ShutdownMsg; + ShutdownMsg.MsgType = AsyncLogMessage::Type::Shutdown; + m_Queue.Enqueue(std::move(ShutdownMsg)); + + if (m_WorkerThread.joinable()) + { + m_WorkerThread.join(); + } + } + + void Log(const LogMessage& Msg) + { + AsyncLogMessage AsyncMsg; + AsyncMsg.OwnedPayload = std::string(Msg.GetPayload()); + AsyncMsg.OwnedLoggerName = std::string(Msg.GetLoggerName()); + AsyncMsg.ThreadId = Msg.GetThreadId(); + AsyncMsg.Time = Msg.GetTime(); + AsyncMsg.Point = &Msg.GetLogPoint(); + AsyncMsg.MsgType = AsyncLogMessage::Type::Log; + + m_Queue.Enqueue(std::move(AsyncMsg)); + } + + void Flush() + { + auto Promise = std::make_shared>(); + auto Future = Promise->get_future(); + + AsyncLogMessage FlushMsg; + FlushMsg.MsgType = AsyncLogMessage::Type::Flush; + FlushMsg.FlushPromise = std::move(Promise); + + m_Queue.Enqueue(std::move(FlushMsg)); + + Future.get(); + } + + void SetFormatter(std::unique_ptr InFormatter) + { + for (auto& CurrentSink : m_Sinks) + { + CurrentSink->SetFormatter(InFormatter->Clone()); + } + } + +private: + void ForwardLogToSinks(const AsyncLogMessage& AsyncMsg) + { + LogMessage Reconstructed(*AsyncMsg.Point, AsyncMsg.OwnedLoggerName, AsyncMsg.OwnedPayload); + Reconstructed.SetTime(AsyncMsg.Time); + Reconstructed.SetThreadId(AsyncMsg.ThreadId); + + for (auto& CurrentSink : m_Sinks) + { + if (CurrentSink->ShouldLog(Reconstructed.GetLevel())) + { + try + { + CurrentSink->Log(Reconstructed); + } + catch (const std::exception&) + { + } + } + } + } + + void FlushSinks() + { + for (auto& CurrentSink : m_Sinks) + { + try + { + CurrentSink->Flush(); + } + catch (const std::exception&) + { + } + } + } + + void WorkerLoop() + { + AsyncLogMessage Msg; + while (m_Queue.WaitAndDequeue(Msg)) + { + switch (Msg.MsgType) + { + case AsyncLogMessage::Type::Log: + { + ForwardLogToSinks(Msg); + break; + } + + case AsyncLogMessage::Type::Flush: + { + FlushSinks(); + if (Msg.FlushPromise) + { + Msg.FlushPromise->set_value(); + } + break; + } + + case AsyncLogMessage::Type::Shutdown: + { + m_Queue.CompleteAdding(); + + AsyncLogMessage Remaining; + while (m_Queue.WaitAndDequeue(Remaining)) + { + if (Remaining.MsgType == AsyncLogMessage::Type::Log) + { + ForwardLogToSinks(Remaining); + } + else if (Remaining.MsgType == AsyncLogMessage::Type::Flush) + { + FlushSinks(); + if (Remaining.FlushPromise) + { + Remaining.FlushPromise->set_value(); + } + } + } + + FlushSinks(); + return; + } + } + } + } + + std::vector m_Sinks; + BlockingQueue m_Queue; + std::thread m_WorkerThread; +}; + +AsyncSink::AsyncSink(std::vector InSinks) : m_Impl(std::make_unique(std::move(InSinks))) +{ +} + +AsyncSink::~AsyncSink() = default; + +void +AsyncSink::Log(const LogMessage& Msg) +{ + m_Impl->Log(Msg); +} + +void +AsyncSink::Flush() +{ + m_Impl->Flush(); +} + +void +AsyncSink::SetFormatter(std::unique_ptr InFormatter) +{ + m_Impl->SetFormatter(std::move(InFormatter)); +} + +} // namespace zen::logging diff --git a/src/zencore/logging/logger.cpp b/src/zencore/logging/logger.cpp new file mode 100644 index 000000000..dd1675bb1 --- /dev/null +++ b/src/zencore/logging/logger.cpp @@ -0,0 +1,142 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include +#include + +#include +#include + +namespace zen::logging { + +struct Logger::Impl +{ + std::string m_Name; + std::vector m_Sinks; + ErrorHandler* m_ErrorHandler = nullptr; +}; + +Logger::Logger(std::string_view InName, SinkPtr InSink) : m_Impl(std::make_unique()) +{ + m_Impl->m_Name = InName; + m_Impl->m_Sinks.push_back(std::move(InSink)); +} + +Logger::Logger(std::string_view InName, std::span InSinks) : m_Impl(std::make_unique()) +{ + m_Impl->m_Name = InName; + m_Impl->m_Sinks.assign(InSinks.begin(), InSinks.end()); +} + +Logger::~Logger() = default; + +void +Logger::Log(const LogPoint& Point, fmt::format_args Args) +{ + if (!ShouldLog(Point.Level)) + { + return; + } + + fmt::basic_memory_buffer Buffer; + fmt::vformat_to(fmt::appender(Buffer), Point.FormatString, Args); + + LogMessage LogMsg(Point, m_Impl->m_Name, std::string_view(Buffer.data(), Buffer.size())); + LogMsg.SetThreadId(GetCurrentThreadId()); + SinkIt(LogMsg); + FlushIfNeeded(Point.Level); +} + +void +Logger::SinkIt(const LogMessage& Msg) +{ + for (auto& CurrentSink : m_Impl->m_Sinks) + { + if (CurrentSink->ShouldLog(Msg.GetLevel())) + { + try + { + CurrentSink->Log(Msg); + } + catch (const std::exception& Ex) + { + if (m_Impl->m_ErrorHandler) + { + m_Impl->m_ErrorHandler->HandleError(Ex.what()); + } + } + } + } +} + +void +Logger::FlushIfNeeded(LogLevel InLevel) +{ + if (InLevel >= m_FlushLevel.load(std::memory_order_relaxed)) + { + Flush(); + } +} + +void +Logger::Flush() +{ + for (auto& CurrentSink : m_Impl->m_Sinks) + { + try + { + CurrentSink->Flush(); + } + catch (const std::exception& Ex) + { + if (m_Impl->m_ErrorHandler) + { + m_Impl->m_ErrorHandler->HandleError(Ex.what()); + } + } + } +} + +void +Logger::SetSinks(std::vector InSinks) +{ + m_Impl->m_Sinks = std::move(InSinks); +} + +void +Logger::AddSink(SinkPtr InSink) +{ + m_Impl->m_Sinks.push_back(std::move(InSink)); +} + +void +Logger::SetErrorHandler(ErrorHandler* Handler) +{ + m_Impl->m_ErrorHandler = Handler; +} + +void +Logger::SetFormatter(std::unique_ptr InFormatter) +{ + for (auto& CurrentSink : m_Impl->m_Sinks) + { + CurrentSink->SetFormatter(InFormatter->Clone()); + } +} + +std::string_view +Logger::Name() const +{ + return m_Impl->m_Name; +} + +Ref +Logger::Clone(std::string_view NewName) const +{ + Ref Cloned(new Logger(NewName, m_Impl->m_Sinks)); + Cloned->SetLevel(m_Level.load(std::memory_order_relaxed)); + Cloned->SetFlushLevel(m_FlushLevel.load(std::memory_order_relaxed)); + Cloned->SetErrorHandler(m_Impl->m_ErrorHandler); + return Cloned; +} + +} // namespace zen::logging diff --git a/src/zencore/logging/msvcsink.cpp b/src/zencore/logging/msvcsink.cpp new file mode 100644 index 000000000..457a4d6e1 --- /dev/null +++ b/src/zencore/logging/msvcsink.cpp @@ -0,0 +1,80 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#if ZEN_PLATFORM_WINDOWS + +# include +# include +# include + +ZEN_THIRD_PARTY_INCLUDES_START +# include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen::logging { + +// Default formatter for MSVC debug output: [level] message\n +// For error/critical messages with source info, prepends file(line): so that +// the message is clickable in the Visual Studio Output window. +class DefaultMsvcFormatter : public Formatter +{ +public: + void Format(const LogMessage& Msg, MemoryBuffer& Dest) override + { + const auto& Source = Msg.GetSource(); + if (Msg.GetLevel() >= LogLevel::Err && Source) + { + helpers::AppendStringView(Source.Filename, Dest); + Dest.push_back('('); + helpers::AppendInt(Source.Line, Dest); + Dest.push_back(')'); + Dest.push_back(':'); + Dest.push_back(' '); + } + + Dest.push_back('['); + helpers::AppendStringView(helpers::LevelToShortString(Msg.GetLevel()), Dest); + Dest.push_back(']'); + Dest.push_back(' '); + helpers::AppendStringView(Msg.GetPayload(), Dest); + Dest.push_back('\n'); + } + + std::unique_ptr Clone() const override { return std::make_unique(); } +}; + +MsvcSink::MsvcSink() : m_Formatter(std::make_unique()) +{ +} + +void +MsvcSink::Log(const LogMessage& Msg) +{ + std::lock_guard Lock(m_Mutex); + + MemoryBuffer Formatted; + m_Formatter->Format(Msg, Formatted); + + // Null-terminate for OutputDebugStringA + Formatted.push_back('\0'); + + OutputDebugStringA(Formatted.data()); +} + +void +MsvcSink::Flush() +{ + // Nothing to flush for OutputDebugString +} + +void +MsvcSink::SetFormatter(std::unique_ptr InFormatter) +{ + std::lock_guard Lock(m_Mutex); + m_Formatter = std::move(InFormatter); +} + +} // namespace zen::logging + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/src/zencore/logging/registry.cpp b/src/zencore/logging/registry.cpp new file mode 100644 index 000000000..3ed1fb0df --- /dev/null +++ b/src/zencore/logging/registry.cpp @@ -0,0 +1,330 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace zen::logging { + +struct Registry::Impl +{ + Impl() + { + // Create default logger with a stdout color sink + SinkPtr DefaultSink(new AnsiColorStdoutSink()); + m_DefaultLogger = Ref(new Logger("", DefaultSink)); + m_Loggers[""] = m_DefaultLogger; + } + + ~Impl() { StopPeriodicFlush(); } + + void Register(Ref InLogger) + { + std::lock_guard Lock(m_Mutex); + if (m_ErrorHandler) + { + InLogger->SetErrorHandler(m_ErrorHandler); + } + m_Loggers[std::string(InLogger->Name())] = std::move(InLogger); + } + + void Drop(const std::string& Name) + { + std::lock_guard Lock(m_Mutex); + m_Loggers.erase(Name); + } + + Ref Get(const std::string& Name) + { + std::lock_guard Lock(m_Mutex); + auto It = m_Loggers.find(Name); + if (It != m_Loggers.end()) + { + return It->second; + } + return {}; + } + + void SetDefaultLogger(Ref InLogger) + { + std::lock_guard Lock(m_Mutex); + if (InLogger) + { + m_Loggers[std::string(InLogger->Name())] = InLogger; + } + m_DefaultLogger = std::move(InLogger); + } + + Logger* DefaultLoggerRaw() { return m_DefaultLogger.Get(); } + + Ref DefaultLogger() + { + std::lock_guard Lock(m_Mutex); + return m_DefaultLogger; + } + + void SetGlobalLevel(LogLevel Level) + { + m_GlobalLevel.store(Level, std::memory_order_relaxed); + std::lock_guard Lock(m_Mutex); + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetLevel(Level); + } + } + + LogLevel GetGlobalLevel() const { return m_GlobalLevel.load(std::memory_order_relaxed); } + + void SetLevels(Registry::LogLevels Levels, LogLevel* DefaultLevel) + { + std::lock_guard Lock(m_Mutex); + + if (DefaultLevel) + { + m_GlobalLevel.store(*DefaultLevel, std::memory_order_relaxed); + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetLevel(*DefaultLevel); + } + } + + for (auto& [LoggerName, Level] : Levels) + { + auto It = m_Loggers.find(LoggerName); + if (It != m_Loggers.end()) + { + It->second->SetLevel(Level); + } + } + } + + void FlushAll() + { + std::lock_guard Lock(m_Mutex); + for (auto& [Name, CurLogger] : m_Loggers) + { + try + { + CurLogger->Flush(); + } + catch (const std::exception&) + { + } + } + } + + void FlushOn(LogLevel Level) + { + std::lock_guard Lock(m_Mutex); + m_FlushLevel = Level; + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetFlushLevel(Level); + } + } + + void FlushEvery(std::chrono::seconds Interval) + { + StopPeriodicFlush(); + + m_PeriodicFlushRunning.store(true, std::memory_order_relaxed); + + m_FlushThread = std::thread([this, Interval] { + while (m_PeriodicFlushRunning.load(std::memory_order_relaxed)) + { + { + std::unique_lock Lock(m_PeriodicFlushMutex); + m_PeriodicFlushCv.wait_for(Lock, Interval, [this] { return !m_PeriodicFlushRunning.load(std::memory_order_relaxed); }); + } + + if (m_PeriodicFlushRunning.load(std::memory_order_relaxed)) + { + FlushAll(); + } + } + }); + } + + void SetFormatter(std::unique_ptr InFormatter) + { + std::lock_guard Lock(m_Mutex); + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetFormatter(InFormatter->Clone()); + } + } + + void ApplyAll(void (*Func)(void*, Ref), void* Context) + { + std::lock_guard Lock(m_Mutex); + for (auto& [Name, CurLogger] : m_Loggers) + { + Func(Context, CurLogger); + } + } + + void SetErrorHandler(ErrorHandler* Handler) + { + std::lock_guard Lock(m_Mutex); + m_ErrorHandler = Handler; + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetErrorHandler(Handler); + } + } + + void Shutdown() + { + StopPeriodicFlush(); + FlushAll(); + + std::lock_guard Lock(m_Mutex); + m_Loggers.clear(); + m_DefaultLogger = nullptr; + } + +private: + void StopPeriodicFlush() + { + if (m_FlushThread.joinable()) + { + m_PeriodicFlushRunning.store(false, std::memory_order_relaxed); + { + std::lock_guard Lock(m_PeriodicFlushMutex); + m_PeriodicFlushCv.notify_one(); + } + m_FlushThread.join(); + } + } + + std::mutex m_Mutex; + std::unordered_map> m_Loggers; + Ref m_DefaultLogger; + std::atomic m_GlobalLevel{Trace}; + LogLevel m_FlushLevel{Off}; + ErrorHandler* m_ErrorHandler = nullptr; + + // Periodic flush + std::atomic m_PeriodicFlushRunning{false}; + std::mutex m_PeriodicFlushMutex; + std::condition_variable m_PeriodicFlushCv; + std::thread m_FlushThread; +}; + +Registry& +Registry::Instance() +{ + static Registry s_Instance; + return s_Instance; +} + +Registry::Registry() : m_Impl(std::make_unique()) +{ +} + +Registry::~Registry() = default; + +void +Registry::Register(Ref InLogger) +{ + m_Impl->Register(std::move(InLogger)); +} + +void +Registry::Drop(const std::string& Name) +{ + m_Impl->Drop(Name); +} + +Ref +Registry::Get(const std::string& Name) +{ + return m_Impl->Get(Name); +} + +void +Registry::SetDefaultLogger(Ref InLogger) +{ + m_Impl->SetDefaultLogger(std::move(InLogger)); +} + +Logger* +Registry::DefaultLoggerRaw() +{ + return m_Impl->DefaultLoggerRaw(); +} + +Ref +Registry::DefaultLogger() +{ + return m_Impl->DefaultLogger(); +} + +void +Registry::SetGlobalLevel(LogLevel Level) +{ + m_Impl->SetGlobalLevel(Level); +} + +LogLevel +Registry::GetGlobalLevel() const +{ + return m_Impl->GetGlobalLevel(); +} + +void +Registry::SetLevels(LogLevels Levels, LogLevel* DefaultLevel) +{ + m_Impl->SetLevels(Levels, DefaultLevel); +} + +void +Registry::FlushAll() +{ + m_Impl->FlushAll(); +} + +void +Registry::FlushOn(LogLevel Level) +{ + m_Impl->FlushOn(Level); +} + +void +Registry::FlushEvery(std::chrono::seconds Interval) +{ + m_Impl->FlushEvery(Interval); +} + +void +Registry::SetFormatter(std::unique_ptr InFormatter) +{ + m_Impl->SetFormatter(std::move(InFormatter)); +} + +void +Registry::ApplyAllImpl(void (*Func)(void*, Ref), void* Context) +{ + m_Impl->ApplyAll(Func, Context); +} + +void +Registry::SetErrorHandler(ErrorHandler* Handler) +{ + m_Impl->SetErrorHandler(Handler); +} + +void +Registry::Shutdown() +{ + m_Impl->Shutdown(); +} + +} // namespace zen::logging diff --git a/src/zencore/logging/tracesink.cpp b/src/zencore/logging/tracesink.cpp new file mode 100644 index 000000000..e3533327b --- /dev/null +++ b/src/zencore/logging/tracesink.cpp @@ -0,0 +1,88 @@ + +// Copyright Epic Games, Inc. All Rights Reserved. + +#include +#include +#include +#include +#include + +namespace zen::logging { + +UE_TRACE_CHANNEL_DEFINE(LogChannel) + +UE_TRACE_EVENT_BEGIN(Logging, LogCategory, NoSync | Important) + UE_TRACE_EVENT_FIELD(const void*, CategoryPointer) + UE_TRACE_EVENT_FIELD(uint8_t, DefaultVerbosity) + UE_TRACE_EVENT_FIELD(UE::Trace::AnsiString, Name) +UE_TRACE_EVENT_END() + +UE_TRACE_EVENT_BEGIN(Logging, LogMessageSpec, NoSync | Important) + UE_TRACE_EVENT_FIELD(const void*, LogPoint) + UE_TRACE_EVENT_FIELD(const void*, CategoryPointer) + UE_TRACE_EVENT_FIELD(int32_t, Line) + UE_TRACE_EVENT_FIELD(uint8_t, Verbosity) + UE_TRACE_EVENT_FIELD(UE::Trace::AnsiString, FileName) + UE_TRACE_EVENT_FIELD(UE::Trace::AnsiString, FormatString) +UE_TRACE_EVENT_END() + +UE_TRACE_EVENT_BEGIN(Logging, LogMessage, NoSync) + UE_TRACE_EVENT_FIELD(const void*, LogPoint) + UE_TRACE_EVENT_FIELD(uint64_t, Cycle) + UE_TRACE_EVENT_FIELD(uint8_t[], FormatArgs) +UE_TRACE_EVENT_END() + +void +TraceLogCategory(const logging::Logger* Category, const char* Name, logging::LogLevel DefaultVerbosity) +{ + uint16_t NameLen = uint16_t(strlen(Name)); + UE_TRACE_LOG(Logging, LogCategory, LogChannel, NameLen * sizeof(ANSICHAR)) + << LogCategory.CategoryPointer(Category) << LogCategory.DefaultVerbosity(uint8_t(DefaultVerbosity)) + << LogCategory.Name(Name, NameLen); +} + +void +TraceLogMessageSpec(const void* LogPoint, + const logging::Logger* Category, + logging::LogLevel Verbosity, + const std::string_view File, + int32_t Line, + const std::string_view Format) +{ + uint16_t FileNameLen = uint16_t(File.size()); + uint16_t FormatStringLen = uint16_t(Format.size()); + uint32_t DataSize = (FileNameLen * sizeof(ANSICHAR)) + (FormatStringLen * sizeof(ANSICHAR)); + UE_TRACE_LOG(Logging, LogMessageSpec, LogChannel, DataSize) + << LogMessageSpec.LogPoint(LogPoint) << LogMessageSpec.CategoryPointer(Category) << LogMessageSpec.Line(Line) + << LogMessageSpec.Verbosity(uint8_t(Verbosity)) << LogMessageSpec.FileName(File.data(), FileNameLen) + << LogMessageSpec.FormatString(Format.data(), FormatStringLen); +} + +void +TraceLogMessageInternal(const void* LogPoint, int32_t EncodedFormatArgsSize, const uint8_t* EncodedFormatArgs) +{ + UE_TRACE_LOG(Logging, LogMessage, LogChannel) << LogMessage.LogPoint(LogPoint) << LogMessage.Cycle(GetHifreqTimerValue()) + << LogMessage.FormatArgs(EncodedFormatArgs, EncodedFormatArgsSize); +} + +////////////////////////////////////////////////////////////////////////// + +void +TraceSink::Log(const LogMessage& Msg) +{ + ZEN_UNUSED(Msg); +} + +void +TraceSink::Flush() +{ +} + +void +TraceSink::SetFormatter(std::unique_ptr /*InFormatter*/) +{ + // This sink doesn't use a formatter since it just forwards the raw format + // args to the trace system +} + +} // namespace zen::logging diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index bfff114c3..e39b8438d 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -4,29 +4,23 @@ #include #include +#include +#include #include #include #include #include -#if ZEN_PLATFORM_LINUX +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC # include +# include #endif -#if ZEN_PLATFORM_MAC -# include -#endif - -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END - #if ZEN_USE_SENTRY # define SENTRY_BUILD_STATIC 1 ZEN_THIRD_PARTY_INCLUDES_START # include -# include ZEN_THIRD_PARTY_INCLUDES_END namespace sentry { @@ -44,76 +38,58 @@ struct SentryAssertImpl : zen::AssertImpl const zen::CallstackFrames* Callstack) override; }; -class sentry_sink final : public spdlog::sinks::base_sink +static constexpr sentry_level_t MapToSentryLevel[zen::logging::LogLevelCount] = {SENTRY_LEVEL_DEBUG, + SENTRY_LEVEL_DEBUG, + SENTRY_LEVEL_INFO, + SENTRY_LEVEL_WARNING, + SENTRY_LEVEL_ERROR, + SENTRY_LEVEL_FATAL, + SENTRY_LEVEL_DEBUG}; + +class SentrySink final : public zen::logging::Sink { public: - sentry_sink(); - ~sentry_sink(); + SentrySink() = default; + ~SentrySink() = default; + + void Log(const zen::logging::LogMessage& Msg) override + { + if (Msg.GetLevel() != zen::logging::Err && Msg.GetLevel() != zen::logging::Critical) + { + return; + } + try + { + std::string Message = fmt::format("{}\n{}({})", Msg.GetPayload(), Msg.GetSource().Filename, Msg.GetSource().Line); + sentry_value_t Event = sentry_value_new_message_event( + /* level */ MapToSentryLevel[Msg.GetLevel()], + /* logger */ nullptr, + /* message */ Message.c_str()); + sentry_event_value_add_stacktrace(Event, NULL, 0); + sentry_capture_event(Event); + } + catch (const std::exception&) + { + // If our logging with Message formatting fails we do a non-allocating version and just post the payload raw + char TmpBuffer[256]; + size_t MaxCopy = zen::Min(Msg.GetPayload().size(), size_t(255)); + memcpy(TmpBuffer, Msg.GetPayload().data(), MaxCopy); + TmpBuffer[MaxCopy] = '\0'; + sentry_value_t Event = sentry_value_new_message_event( + /* level */ SENTRY_LEVEL_ERROR, + /* logger */ nullptr, + /* message */ TmpBuffer); + sentry_event_value_add_stacktrace(Event, NULL, 0); + sentry_capture_event(Event); + } + } -protected: - void sink_it_(const spdlog::details::log_msg& msg) override; - void flush_() override; + void Flush() override {} + void SetFormatter(std::unique_ptr) override {} }; ////////////////////////////////////////////////////////////////////////// -static constexpr sentry_level_t MapToSentryLevel[spdlog::level::level_enum::n_levels] = {SENTRY_LEVEL_DEBUG, - SENTRY_LEVEL_DEBUG, - SENTRY_LEVEL_INFO, - SENTRY_LEVEL_WARNING, - SENTRY_LEVEL_ERROR, - SENTRY_LEVEL_FATAL, - SENTRY_LEVEL_DEBUG}; - -sentry_sink::sentry_sink() -{ -} -sentry_sink::~sentry_sink() -{ -} - -void -sentry_sink::sink_it_(const spdlog::details::log_msg& msg) -{ - if (msg.level != spdlog::level::err && msg.level != spdlog::level::critical) - { - return; - } - try - { - auto MaybeNullString = [](const char* Ptr) { return Ptr ? Ptr : ""; }; - std::string Message = fmt::format("{}\n{}({}) [{}]", - msg.payload, - MaybeNullString(msg.source.filename), - msg.source.line, - MaybeNullString(msg.source.funcname)); - sentry_value_t event = sentry_value_new_message_event( - /* level */ MapToSentryLevel[msg.level], - /* logger */ nullptr, - /* message */ Message.c_str()); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); - } - catch (const std::exception&) - { - // If our logging with Message formatting fails we do a non-allocating version and just post the msg.payload raw - char TmpBuffer[256]; - size_t MaxCopy = zen::Min(msg.payload.size(), size_t(255)); - memcpy(TmpBuffer, msg.payload.data(), MaxCopy); - TmpBuffer[MaxCopy] = '\0'; - sentry_value_t event = sentry_value_new_message_event( - /* level */ SENTRY_LEVEL_ERROR, - /* logger */ nullptr, - /* message */ TmpBuffer); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); - } -} -void -sentry_sink::flush_() -{ -} - void SentryAssertImpl::OnAssert(const char* Filename, int LineNumber, @@ -340,7 +316,9 @@ SentryIntegration::Initialize(const Config& Conf, const std::string& CommandLine sentry_set_user(SentryUserObject); - m_SentryLogger = spdlog::create("sentry"); + logging::SinkPtr SentrySink(new sentry::SentrySink()); + m_SentryLogger = Ref(new logging::Logger("sentry", std::vector{SentrySink})); + logging::Registry::Instance().Register(m_SentryLogger); logging::SetErrorLog("sentry"); m_SentryAssert = std::make_unique(); @@ -354,7 +332,7 @@ SentryIntegration::LogStartupInformation() { // Initialize the sentry-sdk log category at Warn level to reduce startup noise. // The level can be overridden via --log-debug=sentry-sdk or --log-info=sentry-sdk - LogSentry.Logger().SetLogLevel(logging::level::Warn); + LogSentry.Logger().SetLogLevel(logging::Warn); if (m_IsInitialized) { diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp index 0bae139bd..089e376bb 100644 --- a/src/zencore/testing.cpp +++ b/src/zencore/testing.cpp @@ -143,11 +143,11 @@ TestRunner::ApplyCommandLine(int Argc, char const* const* Argv) { if (Argv[i] == "--debug"sv) { - zen::logging::SetLogLevel(zen::logging::level::Debug); + zen::logging::SetLogLevel(zen::logging::Debug); } else if (Argv[i] == "--verbose"sv) { - zen::logging::SetLogLevel(zen::logging::level::Trace); + zen::logging::SetLogLevel(zen::logging::Trace); } } diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index 2f81b7ec8..171f4c533 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -26,7 +26,6 @@ target('zencore') end add_deps("zenbase") - add_deps("spdlog") add_deps("utfcpp") add_deps("oodle") add_deps("blake3") diff --git a/src/zencore/zencore.cpp b/src/zencore/zencore.cpp index d82474705..8c29a8962 100644 --- a/src/zencore/zencore.cpp +++ b/src/zencore/zencore.cpp @@ -147,7 +147,7 @@ AssertImpl::OnAssert(const char* Filename, int LineNumber, const char* FunctionN Message.push_back('\0'); // We use direct ZEN_LOG here instead of ZEN_ERROR as we don't care about *this* code location in the log - ZEN_LOG(Log(), zen::logging::level::Err, "{}", Message.data()); + ZEN_LOG(Log(), zen::logging::Err, "{}", Message.data()); zen::logging::FlushLogging(); } -- cgit v1.2.3 From 07649104761ee910b667adb2b865c4e2fd0979c9 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 9 Mar 2026 12:03:25 +0100 Subject: added auto-detection logic for console colour output (#817) Add auto-detection of colour support to `AnsicolourStdoutSink`. **New `colorMode` enum** (`On`, `Off`, `Auto`) added to the header, accepted by the `AnsicolorStdoutSink` constructor. Defaults to `Auto`, so all existing call sites are unaffected. **`Auto` mode detection logic** (in `IscolourTerminal()`): 1. **TTY check** -- if stdout is not a terminal, colour is disabled. 2. **`NO_COLOR`** -- respects the no-colour.org convention. If set, colour is disabled. 3. **`COLORTERM`** -- if set (e.g. `truecolour`, `24bit`), colour is enabled. 4. **`TERM`** -- rejects `dumb`; accepts known colour-capable terminals via substring match: `alacritty`, `ansi`, `colour`, `console`, `cygwin`, `gnome`, `konsole`, `kterm`, `linux`, `msys`, `putty`, `rxvt`, `screen`, `tmux`, `vt100`, `vt102`, `xterm`. Substring matching covers variants like `xterm-256color` and `rxvt-unicode`. 5. **Fallback** -- Windows defaults to colour enabled (modern console supports ANSI natively); other platforms default to disabled. When colour is disabled, ANSI escape sequences are omitted entirely from the output. NOTE: this doesn't currently apply to all paths which do logging in zen as they may be determining their colour output mode separately from `AnsicolorStdoutSink`. --- .../include/zencore/logging/ansicolorsink.h | 9 +- src/zencore/logging/ansicolorsink.cpp | 103 ++++++++++++++++++++- 2 files changed, 107 insertions(+), 5 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/logging/ansicolorsink.h b/src/zencore/include/zencore/logging/ansicolorsink.h index 9f859e8d7..5060a8393 100644 --- a/src/zencore/include/zencore/logging/ansicolorsink.h +++ b/src/zencore/include/zencore/logging/ansicolorsink.h @@ -8,10 +8,17 @@ namespace zen::logging { +enum class ColorMode +{ + On, + Off, + Auto +}; + class AnsiColorStdoutSink : public Sink { public: - AnsiColorStdoutSink(); + explicit AnsiColorStdoutSink(ColorMode Mode = ColorMode::Auto); ~AnsiColorStdoutSink() override; void Log(const LogMessage& Msg) override; diff --git a/src/zencore/logging/ansicolorsink.cpp b/src/zencore/logging/ansicolorsink.cpp index 9b9959862..540d22359 100644 --- a/src/zencore/logging/ansicolorsink.cpp +++ b/src/zencore/logging/ansicolorsink.cpp @@ -5,8 +5,19 @@ #include #include +#include #include +#if defined(_WIN32) +# include +# define ZEN_ISATTY _isatty +# define ZEN_FILENO _fileno +#else +# include +# define ZEN_ISATTY isatty +# define ZEN_FILENO fileno +#endif + namespace zen::logging { // Default formatter replicating spdlog's %+ pattern: @@ -98,7 +109,90 @@ GetColorForLevel(LogLevel InLevel) struct AnsiColorStdoutSink::Impl { - Impl() : m_Formatter(std::make_unique()) {} + explicit Impl(ColorMode Mode) : m_Formatter(std::make_unique()), m_UseColor(ResolveColorMode(Mode)) {} + + static bool IsColorTerminal() + { + // If stdout is not a TTY, no color + if (ZEN_ISATTY(ZEN_FILENO(stdout)) == 0) + { + return false; + } + + // NO_COLOR convention (https://no-color.org/) + if (std::getenv("NO_COLOR") != nullptr) + { + return false; + } + + // COLORTERM is set by terminals that support color (e.g. "truecolor", "24bit") + if (std::getenv("COLORTERM") != nullptr) + { + return true; + } + + // Check TERM for known color-capable values + const char* Term = std::getenv("TERM"); + if (Term != nullptr) + { + std::string_view TermView(Term); + // "dumb" terminals do not support color + if (TermView == "dumb") + { + return false; + } + // Match against known color-capable terminal types. + // TERM often includes suffixes like "-256color", so we use substring matching. + constexpr std::string_view ColorTerms[] = { + "alacritty", + "ansi", + "color", + "console", + "cygwin", + "gnome", + "konsole", + "kterm", + "linux", + "msys", + "putty", + "rxvt", + "screen", + "tmux", + "vt100", + "vt102", + "xterm", + }; + for (std::string_view Candidate : ColorTerms) + { + if (TermView.find(Candidate) != std::string_view::npos) + { + return true; + } + } + } + +#if defined(_WIN32) + // Windows console supports ANSI color by default in modern versions + return true; +#else + // Unknown terminal — be conservative + return false; +#endif + } + + static bool ResolveColorMode(ColorMode Mode) + { + switch (Mode) + { + case ColorMode::On: + return true; + case ColorMode::Off: + return false; + case ColorMode::Auto: + default: + return IsColorTerminal(); + } + } void Log(const LogMessage& Msg) { @@ -107,7 +201,7 @@ struct AnsiColorStdoutSink::Impl MemoryBuffer Formatted; m_Formatter->Format(Msg, Formatted); - if (Msg.ColorRangeEnd > Msg.ColorRangeStart) + if (m_UseColor && Msg.ColorRangeEnd > Msg.ColorRangeStart) { // Print pre-color range fwrite(Formatted.data(), 1, Msg.ColorRangeStart, m_File); @@ -148,10 +242,11 @@ struct AnsiColorStdoutSink::Impl private: std::mutex m_Mutex; std::unique_ptr m_Formatter; - FILE* m_File = stdout; + FILE* m_File = stdout; + bool m_UseColor = true; }; -AnsiColorStdoutSink::AnsiColorStdoutSink() : m_Impl(std::make_unique()) +AnsiColorStdoutSink::AnsiColorStdoutSink(ColorMode Mode) : m_Impl(std::make_unique(Mode)) { } -- cgit v1.2.3 From b37b34ea6ad906f54e8104526e77ba66aed997da Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 9 Mar 2026 17:43:08 +0100 Subject: Dashboard overhaul, compute integration (#814) - **Frontend dashboard overhaul**: Unified compute/main dashboards into a single shared UI. Added new pages for cache, projects, metrics, sessions, info (build/runtime config, system stats). Added live-update via WebSockets with pause control, sortable detail tables, themed styling. Refactored compute/hub/orchestrator pages into modular JS. - **HTTP server fixes and stats**: Fixed http.sys local-only fallback when default port is in use, implemented root endpoint redirect for http.sys, fixed Linux/Mac port reuse. Added /stats endpoint exposing HTTP server metrics (bytes transferred, request rates). Added WebSocket stats tracking. - **OTEL/diagnostics hardening**: Improved OTLP HTTP exporter with better error handling and resilience. Extended diagnostics services configuration. - **Session management**: Added new sessions service with HTTP endpoints for registering, updating, querying, and removing sessions. Includes session log file support. This is still WIP. - **CLI subcommand support**: Added support for commands with subcommands in the zen CLI tool, with improved command dispatch. - **Misc**: Exposed CPU usage/hostname to frontend, fixed JS compact binary float32/float64 decoding, limited projects displayed on front page to 25 sorted by last access, added vscode:// link support. Also contains some fixes from TSAN analysis. --- src/zencore/include/zencore/logging/tracesink.h | 4 ++ src/zencore/include/zencore/sentryintegration.h | 1 + src/zencore/include/zencore/system.h | 2 + src/zencore/logging/tracesink.cpp | 4 ++ src/zencore/sentryintegration.cpp | 19 ++++++-- src/zencore/system.cpp | 60 ++++++++++++++++++++++++- src/zencore/testutils.cpp | 2 +- 7 files changed, 87 insertions(+), 5 deletions(-) (limited to 'src/zencore') diff --git a/src/zencore/include/zencore/logging/tracesink.h b/src/zencore/include/zencore/logging/tracesink.h index e63d838b4..785c51e10 100644 --- a/src/zencore/include/zencore/logging/tracesink.h +++ b/src/zencore/include/zencore/logging/tracesink.h @@ -6,6 +6,8 @@ namespace zen::logging { +#if ZEN_WITH_TRACE + /** * A logging sink that forwards log messages to the trace system. * @@ -20,4 +22,6 @@ public: void SetFormatter(std::unique_ptr InFormatter) override; }; +#endif + } // namespace zen::logging diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h index a4e33d69e..27e5a8a82 100644 --- a/src/zencore/include/zencore/sentryintegration.h +++ b/src/zencore/include/zencore/sentryintegration.h @@ -40,6 +40,7 @@ public: }; void Initialize(const Config& Conf, const std::string& CommandLine); + void Close(); void LogStartupInformation(); static void ClearCaches(); diff --git a/src/zencore/include/zencore/system.h b/src/zencore/include/zencore/system.h index fecbe2dbe..a67999e52 100644 --- a/src/zencore/include/zencore/system.h +++ b/src/zencore/include/zencore/system.h @@ -14,6 +14,7 @@ class CbWriter; std::string GetMachineName(); std::string_view GetOperatingSystemName(); +std::string GetOperatingSystemVersion(); std::string_view GetRuntimePlatformName(); // "windows", "wine", "linux", or "macos" std::string_view GetCpuName(); @@ -28,6 +29,7 @@ struct SystemMetrics uint64_t AvailVirtualMemoryMiB = 0; uint64_t PageFileMiB = 0; uint64_t AvailPageFileMiB = 0; + uint64_t UptimeSeconds = 0; }; /// Extended metrics that include CPU usage percentage, which requires diff --git a/src/zencore/logging/tracesink.cpp b/src/zencore/logging/tracesink.cpp index e3533327b..8a6f4e40c 100644 --- a/src/zencore/logging/tracesink.cpp +++ b/src/zencore/logging/tracesink.cpp @@ -7,6 +7,8 @@ #include #include +#if ZEN_WITH_TRACE + namespace zen::logging { UE_TRACE_CHANNEL_DEFINE(LogChannel) @@ -86,3 +88,5 @@ TraceSink::SetFormatter(std::unique_ptr /*InFormatter*/) } } // namespace zen::logging + +#endif diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index e39b8438d..8d087e8c6 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -128,6 +128,8 @@ namespace zen { # if ZEN_USE_SENTRY ZEN_DEFINE_LOG_CATEGORY_STATIC(LogSentry, "sentry-sdk"); +static std::atomic s_SentryLogEnabled{true}; + static void SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[maybe_unused]] void* Userdata) { @@ -147,14 +149,15 @@ SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[may } // SentryLogFunction can be called before the logging system is initialized - // (during sentry_init which runs before InitializeLogging). Fall back to - // console logging when the category logger is not yet available. + // (during sentry_init which runs before InitializeLogging), or after it has + // been shut down (during sentry_close on a background worker thread). Fall + // back to console logging when the category logger is not available. // // Since we want to default to WARN level but this runs before logging has // been configured, we ignore the callbacks for DEBUG/INFO explicitly here // which means users don't see every possible log message if they're trying // to configure the levels using --log-debug=sentry-sdk - if (!TheDefaultLogger) + if (!TheDefaultLogger || !s_SentryLogEnabled.load(std::memory_order_acquire)) { switch (Level) { @@ -211,12 +214,22 @@ SentryIntegration::SentryIntegration() } SentryIntegration::~SentryIntegration() +{ + Close(); +} + +void +SentryIntegration::Close() { if (m_IsInitialized && m_SentryErrorCode == 0) { logging::SetErrorLog(""); m_SentryAssert.reset(); + // Disable spdlog forwarding before sentry_close() since its background + // worker thread may still log during shutdown via SentryLogFunction + s_SentryLogEnabled.store(false, std::memory_order_release); sentry_close(); + m_IsInitialized = false; } } diff --git a/src/zencore/system.cpp b/src/zencore/system.cpp index 833d3c04b..141450b84 100644 --- a/src/zencore/system.cpp +++ b/src/zencore/system.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -135,6 +136,8 @@ GetSystemMetrics() Metrics.AvailPageFileMiB = MemStatus.ullAvailPageFile / 1024 / 1024; } + Metrics.UptimeSeconds = GetTickCount64() / 1000; + return Metrics; } #elif ZEN_PLATFORM_LINUX @@ -226,6 +229,17 @@ GetSystemMetrics() Metrics.VirtualMemoryMiB = Metrics.SystemMemoryMiB; Metrics.AvailVirtualMemoryMiB = Metrics.AvailSystemMemoryMiB; + // System uptime + if (FILE* UptimeFile = fopen("/proc/uptime", "r")) + { + double UptimeSec = 0; + if (fscanf(UptimeFile, "%lf", &UptimeSec) == 1) + { + Metrics.UptimeSeconds = static_cast(UptimeSec); + } + fclose(UptimeFile); + } + // Parse /proc/meminfo for swap/page file information Metrics.PageFileMiB = 0; Metrics.AvailPageFileMiB = 0; @@ -318,6 +332,18 @@ GetSystemMetrics() Metrics.PageFileMiB = SwapUsage.xsu_total / 1024 / 1024; Metrics.AvailPageFileMiB = (SwapUsage.xsu_total - SwapUsage.xsu_used) / 1024 / 1024; + // System uptime via boot time + { + struct timeval BootTime + { + }; + Size = sizeof(BootTime); + if (sysctlbyname("kern.boottime", &BootTime, &Size, nullptr, 0) == 0) + { + Metrics.UptimeSeconds = static_cast(time(nullptr) - BootTime.tv_sec); + } + } + return Metrics; } #else @@ -574,6 +600,38 @@ GetOperatingSystemName() return ZEN_PLATFORM_NAME; } +std::string +GetOperatingSystemVersion() +{ +#if ZEN_PLATFORM_WINDOWS + // Use RtlGetVersion to avoid the compatibility shim that GetVersionEx applies + using RtlGetVersionFn = LONG(WINAPI*)(PRTL_OSVERSIONINFOW); + RTL_OSVERSIONINFOW OsVer{.dwOSVersionInfoSize = sizeof(OsVer)}; + if (auto Fn = (RtlGetVersionFn)GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "RtlGetVersion")) + { + Fn(&OsVer); + } + return fmt::format("Windows {}.{} Build {}", OsVer.dwMajorVersion, OsVer.dwMinorVersion, OsVer.dwBuildNumber); +#elif ZEN_PLATFORM_LINUX + struct utsname Info + { + }; + if (uname(&Info) == 0) + { + return fmt::format("{} {}", Info.sysname, Info.release); + } + return "Linux"; +#elif ZEN_PLATFORM_MAC + char OsVersion[64] = ""; + size_t Size = sizeof(OsVersion); + if (sysctlbyname("kern.osproductversion", OsVersion, &Size, nullptr, 0) == 0) + { + return fmt::format("macOS {}", OsVersion); + } + return "macOS"; +#endif +} + std::string_view GetRuntimePlatformName() { @@ -608,7 +666,7 @@ Describe(const SystemMetrics& Metrics, CbWriter& Writer) Writer << "cpu_count" << Metrics.CpuCount << "core_count" << Metrics.CoreCount << "lp_count" << Metrics.LogicalProcessorCount << "total_memory_mb" << Metrics.SystemMemoryMiB << "avail_memory_mb" << Metrics.AvailSystemMemoryMiB << "total_virtual_mb" << Metrics.VirtualMemoryMiB << "avail_virtual_mb" << Metrics.AvailVirtualMemoryMiB << "total_pagefile_mb" << Metrics.PageFileMiB - << "avail_pagefile_mb" << Metrics.AvailPageFileMiB; + << "avail_pagefile_mb" << Metrics.AvailPageFileMiB << "uptime_seconds" << Metrics.UptimeSeconds; } void diff --git a/src/zencore/testutils.cpp b/src/zencore/testutils.cpp index 5bc2841ae..0cd3f8121 100644 --- a/src/zencore/testutils.cpp +++ b/src/zencore/testutils.cpp @@ -46,7 +46,7 @@ ScopedTemporaryDirectory::~ScopedTemporaryDirectory() IoBuffer CreateRandomBlob(uint64_t Size) { - static FastRandom Rand{.Seed = 0x7CEBF54E45B9F5D1}; + thread_local FastRandom Rand{.Seed = 0x7CEBF54E45B9F5D1}; return CreateRandomBlob(Rand, Size); }; -- cgit v1.2.3