diff options
| author | Liam Mitchell <[email protected]> | 2026-03-09 19:06:36 -0700 |
|---|---|---|
| committer | Liam Mitchell <[email protected]> | 2026-03-09 19:06:36 -0700 |
| commit | d1abc50ee9d4fb72efc646e17decafea741caa34 (patch) | |
| tree | e4288e00f2f7ca0391b83d986efcb69d3ba66a83 /src/zencore | |
| parent | Allow requests with invalid content-types unless specified in command line or... (diff) | |
| parent | updated chunk–block analyser (#818) (diff) | |
| download | zen-d1abc50ee9d4fb72efc646e17decafea741caa34.tar.xz zen-d1abc50ee9d4fb72efc646e17decafea741caa34.zip | |
Merge branch 'main' into lm/restrict-content-type
Diffstat (limited to 'src/zencore')
86 files changed, 4112 insertions, 828 deletions
diff --git a/src/zencore/base64.cpp b/src/zencore/base64.cpp index 1f56ee6c3..96e121799 100644 --- a/src/zencore/base64.cpp +++ b/src/zencore/base64.cpp @@ -1,6 +1,10 @@ // Copyright Epic Games, Inc. All Rights Reserved. #include <zencore/base64.h> +#include <zencore/string.h> +#include <zencore/testing.h> + +#include <string> namespace zen { @@ -11,7 +15,6 @@ static const uint8_t EncodingAlphabet[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; /** The table used to convert an ascii character into a 6 bit value */ -#if 0 static const uint8_t DecodingAlphabet[256] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x00-0x0f 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x10-0x1f @@ -30,7 +33,6 @@ static const uint8_t DecodingAlphabet[256] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xe0-0xef 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // 0xf0-0xff }; -#endif // 0 template<typename CharType> uint32_t @@ -104,4 +106,194 @@ Base64::Encode(const uint8_t* Source, uint32_t Length, CharType* Dest) template uint32_t Base64::Encode<char>(const uint8_t* Source, uint32_t Length, char* Dest); template uint32_t Base64::Encode<wchar_t>(const uint8_t* Source, uint32_t Length, wchar_t* Dest); +template<typename CharType> +bool +Base64::Decode(const CharType* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength) +{ + // Length must be a multiple of 4 + if (Length % 4 != 0) + { + OutLength = 0; + return false; + } + + uint8_t* DecodedBytes = Dest; + + // Process 4 encoded characters at a time, producing 3 decoded bytes + while (Length > 0) + { + // Count padding characters at the end + uint32_t PadCount = 0; + if (Source[3] == '=') + { + PadCount++; + if (Source[2] == '=') + { + PadCount++; + } + } + + // Look up each character in the decoding table + uint8_t A = DecodingAlphabet[static_cast<uint8_t>(Source[0])]; + uint8_t B = DecodingAlphabet[static_cast<uint8_t>(Source[1])]; + uint8_t C = (PadCount >= 2) ? 0 : DecodingAlphabet[static_cast<uint8_t>(Source[2])]; + uint8_t D = (PadCount >= 1) ? 0 : DecodingAlphabet[static_cast<uint8_t>(Source[3])]; + + // Check for invalid characters (0xFF means not in the base64 alphabet) + if (A == 0xFF || B == 0xFF || C == 0xFF || D == 0xFF) + { + OutLength = 0; + return false; + } + + // Reconstruct the 24-bit value from 4 6-bit chunks + uint32_t ByteTriplet = (A << 18) | (B << 12) | (C << 6) | D; + + // Extract the 3 bytes + *DecodedBytes++ = static_cast<uint8_t>(ByteTriplet >> 16); + if (PadCount < 2) + { + *DecodedBytes++ = static_cast<uint8_t>((ByteTriplet >> 8) & 0xFF); + } + if (PadCount < 1) + { + *DecodedBytes++ = static_cast<uint8_t>(ByteTriplet & 0xFF); + } + + Source += 4; + Length -= 4; + } + + OutLength = uint32_t(DecodedBytes - Dest); + return true; +} + +template bool Base64::Decode<char>(const char* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength); +template bool Base64::Decode<wchar_t>(const wchar_t* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength); + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +#if ZEN_WITH_TESTS + +using namespace std::string_literals; + +TEST_SUITE_BEGIN("core.base64"); + +TEST_CASE("Base64") +{ + auto EncodeString = [](std::string_view Input) -> std::string { + std::string Result; + Result.resize(Base64::GetEncodedDataSize(uint32_t(Input.size()))); + Base64::Encode(reinterpret_cast<const uint8_t*>(Input.data()), uint32_t(Input.size()), Result.data()); + return Result; + }; + + auto DecodeString = [](std::string_view Input) -> std::string { + std::string Result; + Result.resize(Base64::GetMaxDecodedDataSize(uint32_t(Input.size()))); + uint32_t DecodedLength = 0; + bool Success = Base64::Decode(Input.data(), uint32_t(Input.size()), reinterpret_cast<uint8_t*>(Result.data()), DecodedLength); + CHECK(Success); + Result.resize(DecodedLength); + return Result; + }; + + SUBCASE("Encode") + { + CHECK(EncodeString("") == ""s); + CHECK(EncodeString("f") == "Zg=="s); + CHECK(EncodeString("fo") == "Zm8="s); + CHECK(EncodeString("foo") == "Zm9v"s); + CHECK(EncodeString("foob") == "Zm9vYg=="s); + CHECK(EncodeString("fooba") == "Zm9vYmE="s); + CHECK(EncodeString("foobar") == "Zm9vYmFy"s); + } + + SUBCASE("Decode") + { + CHECK(DecodeString("") == ""s); + CHECK(DecodeString("Zg==") == "f"s); + CHECK(DecodeString("Zm8=") == "fo"s); + CHECK(DecodeString("Zm9v") == "foo"s); + CHECK(DecodeString("Zm9vYg==") == "foob"s); + CHECK(DecodeString("Zm9vYmE=") == "fooba"s); + CHECK(DecodeString("Zm9vYmFy") == "foobar"s); + } + + SUBCASE("RoundTrip") + { + auto RoundTrip = [&](const std::string& Input) { + std::string Encoded = EncodeString(Input); + std::string Decoded = DecodeString(Encoded); + CHECK(Decoded == Input); + }; + + RoundTrip("Hello, World!"); + RoundTrip("Base64 encoding test with various lengths"); + RoundTrip("A"); + RoundTrip("AB"); + RoundTrip("ABC"); + RoundTrip("ABCD"); + RoundTrip("\x00\x01\x02\xff\xfe\xfd"s); + } + + SUBCASE("BinaryRoundTrip") + { + // Test with all byte values 0-255 + uint8_t AllBytes[256]; + for (int i = 0; i < 256; ++i) + { + AllBytes[i] = static_cast<uint8_t>(i); + } + + char Encoded[Base64::GetEncodedDataSize(256) + 1]; + Base64::Encode(AllBytes, 256, Encoded); + + uint8_t Decoded[256]; + uint32_t DecodedLength = 0; + bool Success = Base64::Decode(Encoded, uint32_t(strlen(Encoded)), Decoded, DecodedLength); + CHECK(Success); + CHECK(DecodedLength == 256); + CHECK(memcmp(AllBytes, Decoded, 256) == 0); + } + + SUBCASE("DecodeInvalidInput") + { + uint8_t Dest[64]; + uint32_t OutLength = 0; + + // Length not a multiple of 4 + CHECK_FALSE(Base64::Decode("abc", 3u, Dest, OutLength)); + + // Invalid character + CHECK_FALSE(Base64::Decode("ab!d", 4u, Dest, OutLength)); + } + + SUBCASE("EncodedDataSize") + { + CHECK(Base64::GetEncodedDataSize(0) == 0); + CHECK(Base64::GetEncodedDataSize(1) == 4); + CHECK(Base64::GetEncodedDataSize(2) == 4); + CHECK(Base64::GetEncodedDataSize(3) == 4); + CHECK(Base64::GetEncodedDataSize(4) == 8); + CHECK(Base64::GetEncodedDataSize(5) == 8); + CHECK(Base64::GetEncodedDataSize(6) == 8); + } + + SUBCASE("MaxDecodedDataSize") + { + CHECK(Base64::GetMaxDecodedDataSize(0) == 0); + CHECK(Base64::GetMaxDecodedDataSize(4) == 3); + CHECK(Base64::GetMaxDecodedDataSize(8) == 6); + CHECK(Base64::GetMaxDecodedDataSize(12) == 9); + } +} + +TEST_SUITE_END(); + +#endif + } // namespace zen diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index bd4d119fb..9dcf7663a 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -888,6 +888,8 @@ WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.basicfile"); + TEST_CASE("BasicFile") { ScopedCurrentDirectoryChange _; @@ -1081,6 +1083,8 @@ TEST_CASE("BasicFileBuffer") } } +TEST_SUITE_END(); + void basicfile_forcelink() { diff --git a/src/zencore/blake3.cpp b/src/zencore/blake3.cpp index 054f0d3a0..55f9b74af 100644 --- a/src/zencore/blake3.cpp +++ b/src/zencore/blake3.cpp @@ -123,7 +123,7 @@ BLAKE3::ToHexString(StringBuilderBase& outBuilder) const char str[65]; ToHexString(str); - outBuilder.AppendRange(str, &str[65]); + outBuilder.AppendRange(str, &str[StringLength]); return outBuilder; } @@ -200,6 +200,8 @@ BLAKE3Stream::GetHash() // return text; // } +TEST_SUITE_BEGIN("core.blake3"); + TEST_CASE("BLAKE3") { SUBCASE("Basics") @@ -237,6 +239,8 @@ TEST_CASE("BLAKE3") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/callstack.cpp b/src/zencore/callstack.cpp index 8aa1111bf..ee0b0625a 100644 --- a/src/zencore/callstack.cpp +++ b/src/zencore/callstack.cpp @@ -260,6 +260,8 @@ GetCallstackRaw(void* CaptureBuffer, int FramesToSkip, int FramesToCapture) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.callstack"); + TEST_CASE("Callstack.Basic") { void* Addresses[4]; @@ -272,6 +274,8 @@ TEST_CASE("Callstack.Basic") } } +TEST_SUITE_END(); + void callstack_forcelink() { diff --git a/src/zencore/commandline.cpp b/src/zencore/commandline.cpp index 426cf23d6..718ef9678 100644 --- a/src/zencore/commandline.cpp +++ b/src/zencore/commandline.cpp @@ -14,6 +14,7 @@ ZEN_THIRD_PARTY_INCLUDES_END # include <crt_externs.h> #endif +#include <locale.h> #include <functional> namespace zen { diff --git a/src/zencore/compactbinary.cpp b/src/zencore/compactbinary.cpp index b43cc18f1..9c81305d0 100644 --- a/src/zencore/compactbinary.cpp +++ b/src/zencore/compactbinary.cpp @@ -1512,6 +1512,8 @@ uson_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinary"); + TEST_CASE("guid") { using namespace std::literals; @@ -1704,8 +1706,6 @@ TEST_CASE("uson.datetime") ////////////////////////////////////////////////////////////////////////// -TEST_SUITE_BEGIN("core.datetime"); - TEST_CASE("core.datetime.compare") { DateTime T1(2000, 12, 13); @@ -1732,10 +1732,6 @@ TEST_CASE("core.datetime.add") CHECK(dT + T1 - T2 == dT1); } -TEST_SUITE_END(); - -TEST_SUITE_BEGIN("core.timespan"); - TEST_CASE("core.timespan.compare") { TimeSpan T1(1000); diff --git a/src/zencore/compactbinarybuilder.cpp b/src/zencore/compactbinarybuilder.cpp index 63c0b9c5c..a9ba30750 100644 --- a/src/zencore/compactbinarybuilder.cpp +++ b/src/zencore/compactbinarybuilder.cpp @@ -710,6 +710,8 @@ usonbuilder_forcelink() // return ""; // } +TEST_SUITE_BEGIN("core.compactbinarybuilder"); + TEST_CASE("usonbuilder.object") { using namespace std::literals; @@ -1530,6 +1532,8 @@ TEST_CASE("usonbuilder.stream") CHECK(ValidateCompactBinary(Object.GetBuffer(), CbValidateMode::All) == CbValidateError::None); } } + +TEST_SUITE_END(); #endif } // namespace zen diff --git a/src/zencore/compactbinaryjson.cpp b/src/zencore/compactbinaryjson.cpp index abbec360a..da560a449 100644 --- a/src/zencore/compactbinaryjson.cpp +++ b/src/zencore/compactbinaryjson.cpp @@ -654,6 +654,8 @@ cbjson_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinaryjson"); + TEST_CASE("uson.json") { using namespace std::literals; @@ -872,6 +874,8 @@ TEST_CASE("json.uson") } } +TEST_SUITE_END(); + #endif // ZEN_WITH_TESTS } // namespace zen diff --git a/src/zencore/compactbinarypackage.cpp b/src/zencore/compactbinarypackage.cpp index ffe64f2e9..56a292ca6 100644 --- a/src/zencore/compactbinarypackage.cpp +++ b/src/zencore/compactbinarypackage.cpp @@ -805,6 +805,8 @@ usonpackage_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinarypackage"); + TEST_CASE("usonpackage") { using namespace std::literals; @@ -1343,6 +1345,8 @@ TEST_CASE("usonpackage.invalidpackage") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp index d7292f405..3e78f8ef1 100644 --- a/src/zencore/compactbinaryvalidation.cpp +++ b/src/zencore/compactbinaryvalidation.cpp @@ -753,10 +753,14 @@ usonvalidation_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinaryvalidation"); + TEST_CASE("usonvalidation") { SUBCASE("Basic") {} } + +TEST_SUITE_END(); #endif } // namespace zen diff --git a/src/zencore/compactbinaryyaml.cpp b/src/zencore/compactbinaryyaml.cpp index 5122e952a..b7f2c55df 100644 --- a/src/zencore/compactbinaryyaml.cpp +++ b/src/zencore/compactbinaryyaml.cpp @@ -14,11 +14,6 @@ #include <string_view> #include <vector> -ZEN_THIRD_PARTY_INCLUDES_START -#include <ryml.hpp> -#include <ryml_std.hpp> -ZEN_THIRD_PARTY_INCLUDES_END - namespace zen { ////////////////////////////////////////////////////////////////////////// @@ -26,193 +21,349 @@ namespace zen { class CbYamlWriter { public: - explicit CbYamlWriter(StringBuilderBase& InBuilder) : m_StrBuilder(InBuilder) { m_NodeStack.push_back(m_Tree.rootref()); } + explicit CbYamlWriter(StringBuilderBase& InBuilder) : m_Builder(InBuilder) {} void WriteField(CbFieldView Field) { - ryml::NodeRef Node; + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); - if (m_IsFirst) + switch (Type) { - Node = Top(); + case CbFieldType::Object: + case CbFieldType::UniformObject: + WriteMapEntries(Field, 0); + break; + case CbFieldType::Array: + case CbFieldType::UniformArray: + WriteSeqEntries(Field, 0); + break; + default: + WriteScalarValue(Field); + m_Builder << '\n'; + break; + } + } + + void WriteMapEntry(CbFieldView Field, int32_t Indent) + { + WriteIndent(Indent); + WriteMapEntryContent(Field, Indent); + } + + void WriteSeqEntry(CbFieldView Field, int32_t Indent) + { + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); - m_IsFirst = false; + if (Type == CbFieldType::Object || Type == CbFieldType::UniformObject) + { + bool First = true; + for (CbFieldView MapChild : Field) + { + if (First) + { + WriteIndent(Indent); + m_Builder << "- "; + First = false; + } + else + { + WriteIndent(Indent + 1); + } + WriteMapEntryContent(MapChild, Indent + 1); + } + } + else if (Type == CbFieldType::Array || Type == CbFieldType::UniformArray) + { + WriteIndent(Indent); + m_Builder << "-\n"; + WriteSeqEntries(Field, Indent + 1); } else { - Node = Top().append_child(); + WriteIndent(Indent); + m_Builder << "- "; + WriteScalarValue(Field); + m_Builder << '\n'; } + } - if (std::u8string_view Name = Field.GetU8Name(); !Name.empty()) +private: + void WriteMapEntries(CbFieldView MapField, int32_t Indent) + { + for (CbFieldView Child : MapField) { - Node.set_key_serialized(ryml::csubstr((const char*)Name.data(), Name.size())); + WriteIndent(Indent); + WriteMapEntryContent(Child, Indent); } + } + + void WriteMapEntryContent(CbFieldView Field, int32_t Indent) + { + std::u8string_view Name = Field.GetU8Name(); + m_Builder << std::string_view(reinterpret_cast<const char*>(Name.data()), Name.size()); - switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); + + if (IsContainer(Type)) { - case CbFieldType::Null: - Node.set_val("null"); - break; - case CbFieldType::Object: - case CbFieldType::UniformObject: - Node |= ryml::MAP; - m_NodeStack.push_back(Node); - for (CbFieldView It : Field) + m_Builder << ":\n"; + WriteFieldValue(Field, Indent + 1); + } + else + { + m_Builder << ": "; + WriteScalarValue(Field); + m_Builder << '\n'; + } + } + + void WriteSeqEntries(CbFieldView SeqField, int32_t Indent) + { + for (CbFieldView Child : SeqField) + { + CbValue Accessor = Child.GetValue(); + CbFieldType Type = Accessor.GetType(); + + if (Type == CbFieldType::Object || Type == CbFieldType::UniformObject) + { + bool First = true; + for (CbFieldView MapChild : Child) { - WriteField(It); + if (First) + { + WriteIndent(Indent); + m_Builder << "- "; + First = false; + } + else + { + WriteIndent(Indent + 1); + } + WriteMapEntryContent(MapChild, Indent + 1); } - m_NodeStack.pop_back(); + } + else if (Type == CbFieldType::Array || Type == CbFieldType::UniformArray) + { + WriteIndent(Indent); + m_Builder << "-\n"; + WriteSeqEntries(Child, Indent + 1); + } + else + { + WriteIndent(Indent); + m_Builder << "- "; + WriteScalarValue(Child); + m_Builder << '\n'; + } + } + } + + void WriteFieldValue(CbFieldView Field, int32_t Indent) + { + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); + + switch (Type) + { + case CbFieldType::Object: + case CbFieldType::UniformObject: + WriteMapEntries(Field, Indent); break; case CbFieldType::Array: case CbFieldType::UniformArray: - Node |= ryml::SEQ; - m_NodeStack.push_back(Node); - for (CbFieldView It : Field) - { - WriteField(It); - } - m_NodeStack.pop_back(); + WriteSeqEntries(Field, Indent); break; - case CbFieldType::Binary: - { - ExtendableStringBuilder<256> Builder; - const MemoryView Value = Accessor.AsBinary(); - ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); - - Node.set_key_serialized(Builder.c_str()); - } + case CbFieldType::CustomById: + WriteCustomById(Field.GetValue().AsCustomById(), Indent); break; - case CbFieldType::String: - { - const std::u8string_view U8String = Accessor.AsU8String(); - Node.set_val(ryml::csubstr((const char*)U8String.data(), U8String.size())); - } + case CbFieldType::CustomByName: + WriteCustomByName(Field.GetValue().AsCustomByName(), Indent); + break; + default: + WriteScalarValue(Field); + m_Builder << '\n'; + break; + } + } + + void WriteScalarValue(CbFieldView Field) + { + CbValue Accessor = Field.GetValue(); + switch (Accessor.GetType()) + { + case CbFieldType::Null: + m_Builder << "null"; + break; + case CbFieldType::BoolFalse: + m_Builder << "false"; + break; + case CbFieldType::BoolTrue: + m_Builder << "true"; break; case CbFieldType::IntegerPositive: - Node << Accessor.AsIntegerPositive(); + m_Builder << Accessor.AsIntegerPositive(); break; case CbFieldType::IntegerNegative: - Node << Accessor.AsIntegerNegative(); + m_Builder << Accessor.AsIntegerNegative(); break; case CbFieldType::Float32: if (const float Value = Accessor.AsFloat32(); std::isfinite(Value)) - { - Node << Value; - } + m_Builder.Append(fmt::format("{}", Value)); else - { - Node << "null"; - } + m_Builder << "null"; break; case CbFieldType::Float64: if (const double Value = Accessor.AsFloat64(); std::isfinite(Value)) - { - Node << Value; - } + m_Builder.Append(fmt::format("{}", Value)); else + m_Builder << "null"; + break; + case CbFieldType::String: { - Node << "null"; + const std::u8string_view U8String = Accessor.AsU8String(); + WriteString(std::string_view(reinterpret_cast<const char*>(U8String.data()), U8String.size())); } break; - case CbFieldType::BoolFalse: - Node << "false"; - break; - case CbFieldType::BoolTrue: - Node << "true"; + case CbFieldType::Hash: + WriteString(Accessor.AsHash().ToHexString()); break; case CbFieldType::ObjectAttachment: case CbFieldType::BinaryAttachment: - Node << Accessor.AsAttachment().ToHexString(); - break; - case CbFieldType::Hash: - Node << Accessor.AsHash().ToHexString(); + WriteString(Accessor.AsAttachment().ToHexString()); break; case CbFieldType::Uuid: - Node << fmt::format("{}", Accessor.AsUuid()); + WriteString(fmt::format("{}", Accessor.AsUuid())); break; case CbFieldType::DateTime: - Node << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); + WriteString(DateTime(Accessor.AsDateTimeTicks()).ToIso8601()); break; case CbFieldType::TimeSpan: if (const TimeSpan Span(Accessor.AsTimeSpanTicks()); Span.GetDays() == 0) - { - Node << Span.ToString("%h:%m:%s.%n"); - } + WriteString(Span.ToString("%h:%m:%s.%n")); else - { - Node << Span.ToString("%d.%h:%m:%s.%n"); - } + WriteString(Span.ToString("%d.%h:%m:%s.%n")); break; case CbFieldType::ObjectId: - Node << fmt::format("{}", Accessor.AsObjectId()); + WriteString(fmt::format("{}", Accessor.AsObjectId())); break; - case CbFieldType::CustomById: - { - CbCustomById Custom = Accessor.AsCustomById(); + case CbFieldType::Binary: + WriteBase64(Accessor.AsBinary()); + break; + default: + ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType())); + break; + } + } - Node |= ryml::MAP; + void WriteCustomById(CbCustomById Custom, int32_t Indent) + { + WriteIndent(Indent); + m_Builder << "Id: "; + m_Builder.Append(fmt::format("{}", Custom.Id)); + m_Builder << '\n'; + + WriteIndent(Indent); + m_Builder << "Data: "; + WriteBase64(Custom.Data); + m_Builder << '\n'; + } - ryml::NodeRef IdNode = Node.append_child(); - IdNode.set_key("Id"); - IdNode.set_val_serialized(fmt::format("{}", Custom.Id)); + void WriteCustomByName(CbCustomByName Custom, int32_t Indent) + { + WriteIndent(Indent); + m_Builder << "Name: "; + WriteString(std::string_view(reinterpret_cast<const char*>(Custom.Name.data()), Custom.Name.size())); + m_Builder << '\n'; + + WriteIndent(Indent); + m_Builder << "Data: "; + WriteBase64(Custom.Data); + m_Builder << '\n'; + } - ryml::NodeRef DataNode = Node.append_child(); - DataNode.set_key("Data"); + void WriteBase64(MemoryView Value) + { + ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); + ExtendableStringBuilder<256> Buf; + const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); + const size_t EncodedIndex = Buf.AddUninitialized(size_t(EncodedSize)); + Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Buf.Data() + EncodedIndex); + WriteString(Buf.ToView()); + } - ExtendableStringBuilder<256> Builder; - const MemoryView& Value = Custom.Data; - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + void WriteString(std::string_view Str) + { + if (NeedsQuoting(Str)) + { + m_Builder << '\''; + for (char C : Str) + { + if (C == '\'') + m_Builder << "''"; + else + m_Builder << C; + } + m_Builder << '\''; + } + else + { + m_Builder << Str; + } + } - DataNode.set_val_serialized(Builder.c_str()); - } - break; - case CbFieldType::CustomByName: - { - CbCustomByName Custom = Accessor.AsCustomByName(); + void WriteIndent(int32_t Indent) + { + for (int32_t I = 0; I < Indent; ++I) + m_Builder << " "; + } - Node |= ryml::MAP; + static bool NeedsQuoting(std::string_view Str) + { + if (Str.empty()) + return false; - ryml::NodeRef NameNode = Node.append_child(); - NameNode.set_key("Name"); - std::string_view Name = std::string_view((const char*)Custom.Name.data(), Custom.Name.size()); - NameNode.set_val_serialized(std::string(Name)); + char First = Str[0]; + if (First == ' ' || First == '\n' || First == '\t' || First == '\r' || First == '*' || First == '&' || First == '%' || + First == '@' || First == '`') + return true; - ryml::NodeRef DataNode = Node.append_child(); - DataNode.set_key("Data"); + if (Str.size() >= 2 && Str[0] == '<' && Str[1] == '<') + return true; - ExtendableStringBuilder<256> Builder; - const MemoryView& Value = Custom.Data; - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + char Last = Str.back(); + if (Last == ' ' || Last == '\n' || Last == '\t' || Last == '\r') + return true; - DataNode.set_val_serialized(Builder.c_str()); - } - break; - default: - ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType())); - break; + for (char C : Str) + { + if (C == '#' || C == ':' || C == '-' || C == '?' || C == ',' || C == '\n' || C == '{' || C == '}' || C == '[' || C == ']' || + C == '\'' || C == '"') + return true; } - if (m_NodeStack.size() == 1) + return false; + } + + static bool IsContainer(CbFieldType Type) + { + switch (Type) { - std::string Yaml = ryml::emitrs_yaml<std::string>(m_Tree); - m_StrBuilder << Yaml; + case CbFieldType::Object: + case CbFieldType::UniformObject: + case CbFieldType::Array: + case CbFieldType::UniformArray: + case CbFieldType::CustomById: + case CbFieldType::CustomByName: + return true; + default: + return false; } } -private: - StringBuilderBase& m_StrBuilder; - bool m_IsFirst = true; - - ryml::Tree m_Tree; - std::vector<ryml::NodeRef> m_NodeStack; - ryml::NodeRef& Top() { return m_NodeStack.back(); } + StringBuilderBase& m_Builder; }; void @@ -229,12 +380,40 @@ CompactBinaryToYaml(const CbArrayView& Array, StringBuilderBase& Builder) Writer.WriteField(Array.AsFieldView()); } +void +CompactBinaryToYaml(MemoryView Data, StringBuilderBase& InBuilder) +{ + std::vector<CbFieldView> Fields = ReadCompactBinaryStream(Data); + if (Fields.empty()) + return; + + CbYamlWriter Writer(InBuilder); + if (Fields.size() == 1) + { + Writer.WriteField(Fields[0]); + return; + } + + if (Fields[0].HasName()) + { + for (const CbFieldView& Field : Fields) + Writer.WriteMapEntry(Field, 0); + } + else + { + for (const CbFieldView& Field : Fields) + Writer.WriteSeqEntry(Field, 0); + } +} + #if ZEN_WITH_TESTS void cbyaml_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinaryyaml"); + TEST_CASE("uson.yaml") { using namespace std::literals; @@ -347,6 +526,8 @@ mixed_seq: )"sv); } } + +TEST_SUITE_END(); #endif } // namespace zen diff --git a/src/zencore/compositebuffer.cpp b/src/zencore/compositebuffer.cpp index 252ac9045..ed2b16384 100644 --- a/src/zencore/compositebuffer.cpp +++ b/src/zencore/compositebuffer.cpp @@ -297,6 +297,9 @@ CompositeBuffer::IterateRange(uint64_t Offset, } #if ZEN_WITH_TESTS + +TEST_SUITE_BEGIN("core.compositebuffer"); + TEST_CASE("CompositeBuffer Null") { CompositeBuffer Buffer; @@ -462,6 +465,8 @@ TEST_CASE("CompositeBuffer Composite") TestIterateRange(8, 0, MakeMemoryView(FlatArray).Mid(8, 0), FlatView2); } +TEST_SUITE_END(); + void compositebuffer_forcelink() { diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp index 25ed0fc46..6aa0adce0 100644 --- a/src/zencore/compress.cpp +++ b/src/zencore/compress.cpp @@ -2420,6 +2420,8 @@ private: #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.compress"); + TEST_CASE("CompressedBuffer") { uint8_t Zeroes[1024]{}; @@ -2967,6 +2969,8 @@ TEST_CASE("CompressedBufferReader") } } +TEST_SUITE_END(); + void compress_forcelink() { diff --git a/src/zencore/crypto.cpp b/src/zencore/crypto.cpp index 09eebb6ae..049854b42 100644 --- a/src/zencore/crypto.cpp +++ b/src/zencore/crypto.cpp @@ -449,6 +449,8 @@ crypto_forcelink() { } +TEST_SUITE_BEGIN("core.crypto"); + TEST_CASE("crypto.bits") { using CryptoBits256Bit = CryptoBits<256>; @@ -500,6 +502,8 @@ TEST_CASE("crypto.aes") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 92a065707..8ed63565c 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -194,7 +194,7 @@ WipeDirectory(const wchar_t* DirPath, bool KeepDotFiles) FindClose(hFind); } - return true; + return Success; } bool @@ -1022,7 +1022,7 @@ TryCloneFile(const std::filesystem::path& FromPath, const std::filesystem::path& return false; } fchmod(ToFd, 0666); - ScopedFd $To = { FromFd }; + ScopedFd $To = { ToFd }; ioctl(ToFd, FICLONE, FromFd); @@ -1112,7 +1112,8 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP size_t FileSizeBytes = Stat.st_size; - fchown(ToFd, Stat.st_uid, Stat.st_gid); + int $Ignore = fchown(ToFd, Stat.st_uid, Stat.st_gid); + ZEN_UNUSED($Ignore); // What's the appropriate error handling here? // Copy impl const size_t BufferSize = Min(FileSizeBytes, 64u << 10); @@ -1326,11 +1327,6 @@ ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uin { BytesRead = size_t(dwNumberOfBytesRead); } - else if ((BytesRead != NumberOfBytesToRead)) - { - Ec = MakeErrorCode(ERROR_HANDLE_EOF); - return; - } else { Ec = MakeErrorCodeFromLastError(); @@ -1344,20 +1340,15 @@ ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uin { BytesRead = size_t(ReadResult); } - else if ((BytesRead != NumberOfBytesToRead)) - { - Ec = MakeErrorCode(EIO); - return; - } else { Ec = MakeErrorCodeFromLastError(); return; } #endif - Size -= NumberOfBytesToRead; - FileOffset += NumberOfBytesToRead; - Data = reinterpret_cast<uint8_t*>(Data) + NumberOfBytesToRead; + Size -= BytesRead; + FileOffset += BytesRead; + Data = reinterpret_cast<uint8_t*>(Data) + BytesRead; } } @@ -1408,7 +1399,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer const uint64_t ChunkSize = Min<uint64_t>(WriteSize, uint64_t(2) * 1024 * 1024 * 1024); #if ZEN_PLATFORM_WINDOWS - hRes = Outfile.Write(DataPtr, gsl::narrow_cast<uint32_t>(WriteSize)); + hRes = Outfile.Write(DataPtr, gsl::narrow_cast<uint32_t>(ChunkSize)); if (FAILED(hRes)) { Outfile.Close(); @@ -1417,7 +1408,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer ThrowSystemException(hRes, fmt::format("File write failed for '{}'", Path).c_str()); } #else - if (write(Fd, DataPtr, WriteSize) != int64_t(WriteSize)) + if (write(Fd, DataPtr, ChunkSize) != int64_t(ChunkSize)) { close(Fd); std::error_code DummyEc; @@ -3069,7 +3060,7 @@ SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) } void -MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path) +MakeSafeAbsolutePathInPlace(std::filesystem::path& Path) { if (!Path.empty()) { @@ -3091,7 +3082,7 @@ std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path) { std::filesystem::path Tmp(Path); - MakeSafeAbsolutePathÍnPlace(Tmp); + MakeSafeAbsolutePathInPlace(Tmp); return Tmp; } @@ -3319,6 +3310,8 @@ filesystem_forcelink() { } +TEST_SUITE_BEGIN("core.filesystem"); + TEST_CASE("filesystem") { using namespace std::filesystem; @@ -3543,7 +3536,6 @@ TEST_CASE("PathBuilder") Path.Reset(); Path.Append(fspath(L"/\u0119oo/")); Path /= L"bar"; - printf("%ls\n", Path.ToPath().c_str()); CHECK(Path.ToView() == L"/\u0119oo/bar"); CHECK(Path.ToPath() == L"\\\u0119oo\\bar"); # endif @@ -3614,6 +3606,8 @@ TEST_CASE("SharedMemory") CHECK(!OpenSharedMemory("SharedMemoryTest0", 482, false)); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/include/zencore/base64.h b/src/zencore/include/zencore/base64.h index 4d78b085f..08d9f3043 100644 --- a/src/zencore/include/zencore/base64.h +++ b/src/zencore/include/zencore/base64.h @@ -11,7 +11,11 @@ struct Base64 template<typename CharType> static uint32_t Encode(const uint8_t* Source, uint32_t Length, CharType* Dest); + template<typename CharType> + static bool Decode(const CharType* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength); + static inline constexpr int32_t GetEncodedDataSize(uint32_t Size) { return ((Size + 2) / 3) * 4; } + static inline constexpr int32_t GetMaxDecodedDataSize(uint32_t Length) { return (Length / 4) * 3; } }; } // namespace zen diff --git a/src/zencore/include/zencore/blockingqueue.h b/src/zencore/include/zencore/blockingqueue.h index e91fdc659..b6c93e937 100644 --- a/src/zencore/include/zencore/blockingqueue.h +++ b/src/zencore/include/zencore/blockingqueue.h @@ -2,6 +2,8 @@ #pragma once +#include <zencore/zencore.h> // For ZEN_ASSERT + #include <atomic> #include <condition_variable> #include <deque> diff --git a/src/zencore/include/zencore/compactbinaryfile.h b/src/zencore/include/zencore/compactbinaryfile.h index 00c37e941..33f3e7bea 100644 --- a/src/zencore/include/zencore/compactbinaryfile.h +++ b/src/zencore/include/zencore/compactbinaryfile.h @@ -1,4 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#pragma once #include <zencore/compactbinary.h> #include <zencore/iohash.h> diff --git a/src/zencore/include/zencore/compactbinaryvalue.h b/src/zencore/include/zencore/compactbinaryvalue.h index aa2d2821d..4ce8009b8 100644 --- a/src/zencore/include/zencore/compactbinaryvalue.h +++ b/src/zencore/include/zencore/compactbinaryvalue.h @@ -128,17 +128,21 @@ CbValue::AsString(CbFieldError* OutError, std::string_view Default) const uint32_t ValueSizeByteCount; const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount); - if (OutError) + if (ValueSize >= (uint64_t(1) << 31)) { - if (ValueSize >= (uint64_t(1) << 31)) + if (OutError) { *OutError = CbFieldError::RangeError; - return Default; } + return Default; + } + + if (OutError) + { *OutError = CbFieldError::None; } - return std::string_view(Chars + ValueSizeByteCount, int32_t(ValueSize)); + return std::string_view(Chars + ValueSizeByteCount, size_t(ValueSize)); } inline std::u8string_view @@ -148,17 +152,21 @@ CbValue::AsU8String(CbFieldError* OutError, std::u8string_view Default) const uint32_t ValueSizeByteCount; const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount); - if (OutError) + if (ValueSize >= (uint64_t(1) << 31)) { - if (ValueSize >= (uint64_t(1) << 31)) + if (OutError) { *OutError = CbFieldError::RangeError; - return Default; } + return Default; + } + + if (OutError) + { *OutError = CbFieldError::None; } - return std::u8string_view(Chars + ValueSizeByteCount, int32_t(ValueSize)); + return std::u8string_view(Chars + ValueSizeByteCount, size_t(ValueSize)); } inline uint64_t diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index f28863679..16e2b59f8 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -64,80 +64,80 @@ std::filesystem::path PathFromHandle(void* NativeHandle, std::error_code& Ec); */ std::filesystem::path CanonicalPath(std::filesystem::path InPath, std::error_code& Ec); -/** Query file size +/** Check if a path exists and is a regular file (throws) */ bool IsFile(const std::filesystem::path& Path); -/** Query file size +/** Check if a path exists and is a regular file (does not throw) */ bool IsFile(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size +/** Check if a path exists and is a directory (throws) */ bool IsDir(const std::filesystem::path& Path); -/** Query file size +/** Check if a path exists and is a directory (does not throw) */ bool IsDir(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size +/** Delete file at path, if it exists (throws) */ bool RemoveFile(const std::filesystem::path& Path); -/** Query file size +/** Delete file at path, if it exists (does not throw) */ bool RemoveFile(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size +/** Delete directory at path, if it exists (throws) */ bool RemoveDir(const std::filesystem::path& Path); -/** Query file size +/** Delete directory at path, if it exists (does not throw) */ bool RemoveDir(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size +/** Query file size (throws) */ uint64_t FileSizeFromPath(const std::filesystem::path& Path); -/** Query file size +/** Query file size (does not throw) */ uint64_t FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size from native file handle +/** Query file size from native file handle (throws) */ uint64_t FileSizeFromHandle(void* NativeHandle); -/** Query file size from native file handle +/** Query file size from native file handle (does not throw) */ uint64_t FileSizeFromHandle(void* NativeHandle, std::error_code& Ec); /** Get a native time tick of last modification time */ -uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec); +uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); /** Get a native time tick of last modification time */ -uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); +uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec); bool TryGetFileProperties(const std::filesystem::path& Path, uint64_t& OutSize, uint64_t& OutModificationTick, uint32_t& OutNativeModeOrAttributes); -/** Move a file, if the files are not on the same drive the function will fail +/** Move/rename a file, if the files are not on the same drive the function will fail (throws) */ void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); -/** Move a file, if the files are not on the same drive the function will fail +/** Move/rename a file, if the files are not on the same drive the function will fail */ void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); -/** Move a directory, if the files are not on the same drive the function will fail +/** Move/rename a directory, if the files are not on the same drive the function will fail (throws) */ void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); -/** Move a directory, if the files are not on the same drive the function will fail +/** Move/rename a directory, if the files are not on the same drive the function will fail */ void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); @@ -421,7 +421,7 @@ uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); -void MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path); +void MakeSafeAbsolutePathInPlace(std::filesystem::path& Path); [[nodiscard]] std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path); class SharedMemory diff --git a/src/zencore/include/zencore/hashutils.h b/src/zencore/include/zencore/hashutils.h index 4e877e219..8abfd4b6e 100644 --- a/src/zencore/include/zencore/hashutils.h +++ b/src/zencore/include/zencore/hashutils.h @@ -2,6 +2,10 @@ #pragma once +#include <cstddef> +#include <functional> +#include <type_traits> + namespace zen { template<typename T> diff --git a/src/zencore/include/zencore/iobuffer.h b/src/zencore/include/zencore/iobuffer.h index 182768ff6..82c201edd 100644 --- a/src/zencore/include/zencore/iobuffer.h +++ b/src/zencore/include/zencore/iobuffer.h @@ -426,22 +426,39 @@ private: class IoBufferBuilder { public: - static IoBuffer MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); - static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName); - static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); - /** Make sure buffer data is memory resident, but avoid memory mapping data from files - */ - static IoBuffer ReadFromFileMaybe(const IoBuffer& InBuffer); - inline static IoBuffer MakeFromMemory(MemoryView Memory) { return IoBuffer(IoBuffer::Wrap, Memory.GetData(), Memory.GetSize()); } - inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) + static IoBuffer MakeFromFile(const std::filesystem::path& FileName, + uint64_t Offset = 0, + uint64_t Size = ~0ull, + ZenContentType ContentType = ZenContentType::kBinary); + static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName, ZenContentType ContentType = ZenContentType::kBinary); + static IoBuffer MakeFromFileHandle(void* FileHandle, + uint64_t Offset = 0, + uint64_t Size = ~0ull, + ZenContentType ContentType = ZenContentType::kBinary); + inline static IoBuffer MakeFromMemory(MemoryView Memory, ZenContentType ContentType = ZenContentType::kBinary) + { + IoBuffer NewBuffer(IoBuffer::Wrap, Memory.GetData(), Memory.GetSize()); + NewBuffer.SetContentType(ContentType); + return NewBuffer; + } + inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz, ZenContentType ContentType = ZenContentType::kBinary) { if (Sz) { - return IoBuffer(IoBuffer::Clone, Ptr, Sz); + IoBuffer NewBuffer(IoBuffer::Clone, Ptr, Sz); + NewBuffer.SetContentType(ContentType); + return NewBuffer; } return {}; } - inline static IoBuffer MakeCloneFromMemory(MemoryView Memory) { return MakeCloneFromMemory(Memory.GetData(), Memory.GetSize()); } + inline static IoBuffer MakeCloneFromMemory(MemoryView Memory, ZenContentType ContentType = ZenContentType::kBinary) + { + return MakeCloneFromMemory(Memory.GetData(), Memory.GetSize(), ContentType); + } + + /** Make sure buffer data is memory resident, but avoid memory mapping data from files + */ + static IoBuffer ReadFromFileMaybe(const IoBuffer& InBuffer); }; void iobuffer_forcelink(); diff --git a/src/zencore/include/zencore/logbase.h b/src/zencore/include/zencore/logbase.h index 00af68b0a..ece17a85e 100644 --- a/src/zencore/include/zencore/logbase.h +++ b/src/zencore/include/zencore/logbase.h @@ -4,96 +4,85 @@ #include <string_view> -#define ZEN_LOG_LEVEL_TRACE 0 -#define ZEN_LOG_LEVEL_DEBUG 1 -#define ZEN_LOG_LEVEL_INFO 2 -#define ZEN_LOG_LEVEL_WARN 3 -#define ZEN_LOG_LEVEL_ERROR 4 -#define ZEN_LOG_LEVEL_CRITICAL 5 -#define ZEN_LOG_LEVEL_OFF 6 - -#define ZEN_LEVEL_NAME_TRACE std::string_view("trace", 5) -#define ZEN_LEVEL_NAME_DEBUG std::string_view("debug", 5) -#define ZEN_LEVEL_NAME_INFO std::string_view("info", 4) -#define ZEN_LEVEL_NAME_WARNING std::string_view("warning", 7) -#define ZEN_LEVEL_NAME_ERROR std::string_view("error", 5) -#define ZEN_LEVEL_NAME_CRITICAL std::string_view("critical", 8) -#define ZEN_LEVEL_NAME_OFF std::string_view("off", 3) - -namespace zen::logging::level { +namespace zen::logging { enum LogLevel : int { - Trace = ZEN_LOG_LEVEL_TRACE, - Debug = ZEN_LOG_LEVEL_DEBUG, - Info = ZEN_LOG_LEVEL_INFO, - Warn = ZEN_LOG_LEVEL_WARN, - Err = ZEN_LOG_LEVEL_ERROR, - Critical = ZEN_LOG_LEVEL_CRITICAL, - Off = ZEN_LOG_LEVEL_OFF, + Trace, + Debug, + Info, + Warn, + Err, + Critical, + Off, LogLevelCount }; LogLevel ParseLogLevelString(std::string_view String); std::string_view ToStringView(LogLevel Level); -} // namespace zen::logging::level - -namespace zen::logging { - -void SetLogLevel(level::LogLevel NewLogLevel); -level::LogLevel GetLogLevel(); +void SetLogLevel(LogLevel NewLogLevel); +LogLevel GetLogLevel(); -} // namespace zen::logging +struct SourceLocation +{ + constexpr SourceLocation() = default; + constexpr SourceLocation(const char* InFilename, int InLine) : Filename(InFilename), Line(InLine) {} -namespace spdlog { -class logger; -} + constexpr operator bool() const noexcept { return Line != 0; } -namespace zen::logging { + const char* Filename{nullptr}; + int Line{0}; +}; -struct SourceLocation +/** This encodes the constant parts of a log message which can be emitted once + * and then referred to by log events. + * + * It's *critical* that instances of this struct are permanent and never + * destroyed, as log messages will refer to them by pointer. The easiest way + * to ensure this is to create them as function-local statics. + * + * The logging macros already do this for you so this should not be something + * you normally would need to worry about. + */ +struct LogPoint { - constexpr SourceLocation() = default; - constexpr SourceLocation(const char* filename_in, int line_in, const char* funcname_in) - : filename(filename_in) - , line(line_in) - , funcname(funcname_in) - { - } - - constexpr bool empty() const noexcept { return line == 0; } - - // IMPORTANT NOTE: the layout of this class must match the spdlog::source_loc class - // since we currently pass a pointer to it into spdlog after casting it to - // spdlog::source_loc* - // - // This is intended to be an intermediate state, before we (probably) transition off - // spdlog entirely - - const char* filename{nullptr}; - int line{0}; - const char* funcname{nullptr}; + SourceLocation Location; + LogLevel Level; + std::string_view FormatString; }; +class Logger; + } // namespace zen::logging namespace zen { +// Lightweight non-owning handle to a Logger. Loggers are owned by the Registry +// via Ref<Logger>; LoggerRef exists as a cheap (raw pointer) handle that can be +// stored in members and passed through logging macros without requiring the +// complete Logger type or incurring refcount overhead on every log call. struct LoggerRef { LoggerRef() = default; - LoggerRef(spdlog::logger& InLogger) : SpdLogger(&InLogger) {} + LoggerRef(logging::Logger& InLogger) : m_Logger(&InLogger) {} + // This exists so that logging macros can pass LoggerRef or LogCategory + // to ZEN_LOG without needing to know which one it is LoggerRef Logger() { return *this; } - bool ShouldLog(int Level) const; - inline operator bool() const { return SpdLogger != nullptr; } + bool ShouldLog(logging::LogLevel Level) const; + inline operator bool() const { return m_Logger != nullptr; } + + inline logging::Logger* operator->() const { return m_Logger; } + inline logging::Logger& operator*() const { return *m_Logger; } - void SetLogLevel(logging::level::LogLevel NewLogLevel); - logging::level::LogLevel GetLogLevel(); + void SetLogLevel(logging::LogLevel NewLogLevel); + logging::LogLevel GetLogLevel(); + void Flush(); - spdlog::logger* SpdLogger = nullptr; +private: + logging::Logger* m_Logger = nullptr; }; } // namespace zen diff --git a/src/zencore/include/zencore/logging.h b/src/zencore/include/zencore/logging.h index afbbbd3ee..4b593c19e 100644 --- a/src/zencore/include/zencore/logging.h +++ b/src/zencore/include/zencore/logging.h @@ -9,16 +9,9 @@ #if ZEN_PLATFORM_WINDOWS # define ZEN_LOG_SECTION(Id) ZEN_DATA_SECTION(Id) -# pragma section(".zlog$f", read) # pragma section(".zlog$l", read) -# pragma section(".zlog$m", read) -# pragma section(".zlog$s", read) -# define ZEN_DECLARE_FUNCTION static constinit ZEN_LOG_SECTION(".zlog$f") char FuncName[] = __FUNCTION__; -# define ZEN_LOG_FUNCNAME FuncName #else # define ZEN_LOG_SECTION(Id) -# define ZEN_DECLARE_FUNCTION -# define ZEN_LOG_FUNCNAME static_cast<const char*>(__func__) #endif namespace zen::logging { @@ -31,39 +24,35 @@ void FlushLogging(); LoggerRef Default(); void SetDefault(std::string_view NewDefaultLoggerId); LoggerRef ConsoleLog(); +void ResetConsoleLog(); void SuppressConsoleLog(); LoggerRef ErrorLog(); void SetErrorLog(std::string_view LoggerId); LoggerRef Get(std::string_view Name); -void ConfigureLogLevels(level::LogLevel Level, std::string_view Loggers); +void ConfigureLogLevels(LogLevel Level, std::string_view Loggers); void RefreshLogLevels(); -void RefreshLogLevels(level::LogLevel DefaultLevel); - +void RefreshLogLevels(LogLevel DefaultLevel); + +/** LogCategory allows for the creation of log categories that can be used with + * the logging macros just like a logger reference. The main purpose of this is + * to allow for static log categories in global scope where we can't actually + * go ahead and instantiate a logger immediately because the logging system may + * not be initialized yet. + */ struct LogCategory { - inline LogCategory(std::string_view InCategory) : CategoryName(InCategory) {} - - inline zen::LoggerRef Logger() - { - if (LoggerRef) - { - return LoggerRef; - } + inline LogCategory(std::string_view InCategory) : m_CategoryName(InCategory) {} - LoggerRef = zen::logging::Get(CategoryName); - return LoggerRef; - } + LoggerRef Logger(); - std::string CategoryName; - zen::LoggerRef LoggerRef; +private: + std::string m_CategoryName; + LoggerRef m_LoggerRef; }; -void EmitConsoleLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args); -void EmitLogMessage(LoggerRef& Logger, int LogLevel, std::string_view Message); -void EmitLogMessage(LoggerRef& Logger, const SourceLocation& Location, int LogLevel, std::string_view Message); -void EmitLogMessage(LoggerRef& Logger, int LogLevel, std::string_view Format, fmt::format_args Args); -void EmitLogMessage(LoggerRef& Logger, const SourceLocation& Location, int LogLevel, std::string_view Format, fmt::format_args Args); +void EmitConsoleLogMessage(const LogPoint& Lp, fmt::format_args Args); +void EmitLogMessage(LoggerRef& Logger, const LogPoint& Lp, fmt::format_args Args); template<typename... T> auto @@ -78,15 +67,14 @@ namespace zen { extern LoggerRef TheDefaultLogger; -inline LoggerRef -Log() -{ - if (TheDefaultLogger) - { - return TheDefaultLogger; - } - return zen::logging::ConsoleLog(); -} +/** + * This is the default logger, which any ZEN_INFO et al will get if there's + * no Log() function declared in the current scope. + * + * Typically, classes which want to log to its own channel will declare a Log() + * member function which returns a LoggerRef created at construction time. + */ +LoggerRef Log(); using logging::ConsoleLog; using logging::ErrorLog; @@ -97,12 +85,6 @@ using zen::ConsoleLog; using zen::ErrorLog; using zen::Log; -inline consteval bool -LogIsErrorLevel(int LogLevel) -{ - return (LogLevel == zen::logging::level::Err || LogLevel == zen::logging::level::Critical); -}; - #if ZEN_BUILD_DEBUG # define ZEN_CHECK_FORMAT_STRING(fmtstr, ...) \ while (false) \ @@ -116,75 +98,66 @@ LogIsErrorLevel(int LogLevel) } #endif -#define ZEN_LOG_WITH_LOCATION(InLogger, InLevel, fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - ZEN_DECLARE_FUNCTION \ - static constinit ZEN_LOG_SECTION(".zlog$s") char FileName[] = __FILE__; \ - static constinit ZEN_LOG_SECTION(".zlog$m") char FormatString[] = fmtstr; \ - static constinit ZEN_LOG_SECTION(".zlog$l") zen::logging::SourceLocation Location{FileName, __LINE__, ZEN_LOG_FUNCNAME}; \ - zen::LoggerRef Logger = InLogger; \ - ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ - if (Logger.ShouldLog(InLevel)) \ - { \ - zen::logging::EmitLogMessage(Logger, \ - Location, \ - InLevel, \ - std::string_view(FormatString, sizeof FormatString - 1), \ - zen::logging::LogCaptureArguments(__VA_ARGS__)); \ - } \ +#define ZEN_LOG_WITH_LOCATION(InLogger, InLevel, fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + static constinit ZEN_LOG_SECTION(".zlog$l") \ + zen::logging::LogPoint LogPoint{zen::logging::SourceLocation{__FILE__, __LINE__}, InLevel, std::string_view(fmtstr)}; \ + zen::LoggerRef Logger = InLogger; \ + ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ + if (Logger.ShouldLog(InLevel)) \ + { \ + zen::logging::EmitLogMessage(Logger, LogPoint, zen::logging::LogCaptureArguments(__VA_ARGS__)); \ + } \ } while (false); -#define ZEN_LOG(InLogger, InLevel, fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - static constinit ZEN_LOG_SECTION(".zlog$m") char FormatString[] = fmtstr; \ - zen::LoggerRef Logger = InLogger; \ - ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ - if (Logger.ShouldLog(InLevel)) \ - { \ - zen::logging::EmitLogMessage(Logger, \ - InLevel, \ - std::string_view(FormatString, sizeof FormatString - 1), \ - zen::logging::LogCaptureArguments(__VA_ARGS__)); \ - } \ +#define ZEN_LOG(InLogger, InLevel, fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + static constinit ZEN_LOG_SECTION(".zlog$l") zen::logging::LogPoint LogPoint{{}, InLevel, std::string_view(fmtstr)}; \ + zen::LoggerRef Logger = InLogger; \ + ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ + if (Logger.ShouldLog(InLevel)) \ + { \ + zen::logging::EmitLogMessage(Logger, LogPoint, zen::logging::LogCaptureArguments(__VA_ARGS__)); \ + } \ } while (false); #define ZEN_DEFINE_LOG_CATEGORY_STATIC(Category, Name) \ static zen::logging::LogCategory Category { Name } -#define ZEN_LOG_TRACE(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::level::Trace, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_DEBUG(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::level::Debug, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_INFO(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::level::Info, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_WARN(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::level::Warn, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_ERROR(Category, fmtstr, ...) ZEN_LOG_WITH_LOCATION(Category.Logger(), zen::logging::level::Err, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_CRITICAL(Category, fmtstr, ...) \ - ZEN_LOG_WITH_LOCATION(Category.Logger(), zen::logging::level::Critical, fmtstr, ##__VA_ARGS__) - -#define ZEN_TRACE(fmtstr, ...) ZEN_LOG(Log(), zen::logging::level::Trace, fmtstr, ##__VA_ARGS__) -#define ZEN_DEBUG(fmtstr, ...) ZEN_LOG(Log(), zen::logging::level::Debug, fmtstr, ##__VA_ARGS__) -#define ZEN_INFO(fmtstr, ...) ZEN_LOG(Log(), zen::logging::level::Info, fmtstr, ##__VA_ARGS__) -#define ZEN_WARN(fmtstr, ...) ZEN_LOG(Log(), zen::logging::level::Warn, fmtstr, ##__VA_ARGS__) -#define ZEN_ERROR(fmtstr, ...) ZEN_LOG_WITH_LOCATION(Log(), zen::logging::level::Err, fmtstr, ##__VA_ARGS__) -#define ZEN_CRITICAL(fmtstr, ...) ZEN_LOG_WITH_LOCATION(Log(), zen::logging::level::Critical, fmtstr, ##__VA_ARGS__) - -#define ZEN_CONSOLE_LOG(InLevel, fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ - zen::logging::EmitConsoleLogMessage(InLevel, fmtstr, zen::logging::LogCaptureArguments(__VA_ARGS__)); \ +#define ZEN_LOG_TRACE(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::Trace, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_DEBUG(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::Debug, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_INFO(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::Info, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_WARN(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::Warn, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_ERROR(Category, fmtstr, ...) ZEN_LOG_WITH_LOCATION(Category.Logger(), zen::logging::Err, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_CRITICAL(Category, fmtstr, ...) ZEN_LOG_WITH_LOCATION(Category.Logger(), zen::logging::Critical, fmtstr, ##__VA_ARGS__) + +#define ZEN_TRACE(fmtstr, ...) ZEN_LOG(Log(), zen::logging::Trace, fmtstr, ##__VA_ARGS__) +#define ZEN_DEBUG(fmtstr, ...) ZEN_LOG(Log(), zen::logging::Debug, fmtstr, ##__VA_ARGS__) +#define ZEN_INFO(fmtstr, ...) ZEN_LOG(Log(), zen::logging::Info, fmtstr, ##__VA_ARGS__) +#define ZEN_WARN(fmtstr, ...) ZEN_LOG(Log(), zen::logging::Warn, fmtstr, ##__VA_ARGS__) +#define ZEN_ERROR(fmtstr, ...) ZEN_LOG_WITH_LOCATION(Log(), zen::logging::Err, fmtstr, ##__VA_ARGS__) +#define ZEN_CRITICAL(fmtstr, ...) ZEN_LOG_WITH_LOCATION(Log(), zen::logging::Critical, fmtstr, ##__VA_ARGS__) + +#define ZEN_CONSOLE_LOG(InLevel, fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + static constinit ZEN_LOG_SECTION(".zlog$l") zen::logging::LogPoint LogPoint{{}, InLevel, std::string_view(fmtstr)}; \ + ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ + zen::logging::EmitConsoleLogMessage(LogPoint, zen::logging::LogCaptureArguments(__VA_ARGS__)); \ } while (false) -#define ZEN_CONSOLE(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_TRACE(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Trace, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_DEBUG(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Debug, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_INFO(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_WARN(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Warn, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_ERROR(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Err, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_CRITICAL(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Critical, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_TRACE(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Trace, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_DEBUG(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Debug, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_INFO(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_WARN(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Warn, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_ERROR(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Err, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_CRITICAL(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Critical, fmtstr, ##__VA_ARGS__) ////////////////////////////////////////////////////////////////////////// @@ -239,28 +212,28 @@ std::string_view EmitActivitiesForLogging(StringBuilderBase& OutString); #define ZEN_LOG_SCOPE(...) ScopedLazyActivity $Activity##__LINE__([&](StringBuilderBase& Out) { Out << fmt::format(__VA_ARGS__); }) -#define ZEN_SCOPED_WARN(fmtstr, ...) \ - do \ - { \ - ExtendableStringBuilder<256> ScopeString; \ - const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ - ZEN_LOG(Log(), zen::logging::level::Warn, fmtstr "{}", ##__VA_ARGS__, Scopes); \ +#define ZEN_SCOPED_WARN(fmtstr, ...) \ + do \ + { \ + ExtendableStringBuilder<256> ScopeString; \ + const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ + ZEN_LOG(Log(), zen::logging::Warn, fmtstr "{}", ##__VA_ARGS__, Scopes); \ } while (false) -#define ZEN_SCOPED_ERROR(fmtstr, ...) \ - do \ - { \ - ExtendableStringBuilder<256> ScopeString; \ - const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ - ZEN_LOG_WITH_LOCATION(Log(), zen::logging::level::Err, fmtstr "{}", ##__VA_ARGS__, Scopes); \ +#define ZEN_SCOPED_ERROR(fmtstr, ...) \ + do \ + { \ + ExtendableStringBuilder<256> ScopeString; \ + const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ + ZEN_LOG_WITH_LOCATION(Log(), zen::logging::Err, fmtstr "{}", ##__VA_ARGS__, Scopes); \ } while (false) -#define ZEN_SCOPED_CRITICAL(fmtstr, ...) \ - do \ - { \ - ExtendableStringBuilder<256> ScopeString; \ - const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ - ZEN_LOG_WITH_LOCATION(Log(), zen::logging::level::Critical, fmtstr "{}", ##__VA_ARGS__, Scopes); \ +#define ZEN_SCOPED_CRITICAL(fmtstr, ...) \ + do \ + { \ + ExtendableStringBuilder<256> ScopeString; \ + const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ + ZEN_LOG_WITH_LOCATION(Log(), zen::logging::Critical, fmtstr "{}", ##__VA_ARGS__, Scopes); \ } while (false) ScopedActivityBase* GetThreadActivity(); diff --git a/src/zencore/include/zencore/logging/ansicolorsink.h b/src/zencore/include/zencore/logging/ansicolorsink.h new file mode 100644 index 000000000..5060a8393 --- /dev/null +++ b/src/zencore/include/zencore/logging/ansicolorsink.h @@ -0,0 +1,33 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/logging/sink.h> + +#include <memory> + +namespace zen::logging { + +enum class ColorMode +{ + On, + Off, + Auto +}; + +class AnsiColorStdoutSink : public Sink +{ +public: + explicit AnsiColorStdoutSink(ColorMode Mode = ColorMode::Auto); + ~AnsiColorStdoutSink() override; + + void Log(const LogMessage& Msg) override; + void Flush() override; + void SetFormatter(std::unique_ptr<Formatter> InFormatter) override; + +private: + struct Impl; + std::unique_ptr<Impl> m_Impl; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/asyncsink.h b/src/zencore/include/zencore/logging/asyncsink.h new file mode 100644 index 000000000..c49a1ccce --- /dev/null +++ b/src/zencore/include/zencore/logging/asyncsink.h @@ -0,0 +1,30 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/logging/sink.h> + +#include <memory> +#include <vector> + +namespace zen::logging { + +class AsyncSink : public Sink +{ +public: + explicit AsyncSink(std::vector<SinkPtr> InSinks); + ~AsyncSink() override; + + AsyncSink(const AsyncSink&) = delete; + AsyncSink& operator=(const AsyncSink&) = delete; + + void Log(const LogMessage& Msg) override; + void Flush() override; + void SetFormatter(std::unique_ptr<Formatter> InFormatter) override; + +private: + struct Impl; + std::unique_ptr<Impl> m_Impl; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/formatter.h b/src/zencore/include/zencore/logging/formatter.h new file mode 100644 index 000000000..11904d71d --- /dev/null +++ b/src/zencore/include/zencore/logging/formatter.h @@ -0,0 +1,20 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/logging/logmsg.h> +#include <zencore/logging/memorybuffer.h> + +#include <memory> + +namespace zen::logging { + +class Formatter +{ +public: + virtual ~Formatter() = default; + virtual void Format(const LogMessage& Msg, MemoryBuffer& Dest) = 0; + virtual std::unique_ptr<Formatter> Clone() const = 0; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/helpers.h b/src/zencore/include/zencore/logging/helpers.h new file mode 100644 index 000000000..ce021e1a5 --- /dev/null +++ b/src/zencore/include/zencore/logging/helpers.h @@ -0,0 +1,122 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/logbase.h> +#include <zencore/logging/memorybuffer.h> + +#include <chrono> +#include <ctime> +#include <string_view> + +namespace zen::logging::helpers { + +inline void +AppendStringView(std::string_view Sv, MemoryBuffer& Dest) +{ + Dest.append(Sv.data(), Sv.data() + Sv.size()); +} + +inline void +AppendInt(int N, MemoryBuffer& Dest) +{ + fmt::format_int Formatted(N); + Dest.append(Formatted.data(), Formatted.data() + Formatted.size()); +} + +inline void +Pad2(int N, MemoryBuffer& Dest) +{ + if (N >= 0 && N < 100) + { + Dest.push_back(static_cast<char>('0' + N / 10)); + Dest.push_back(static_cast<char>('0' + N % 10)); + } + else + { + fmt::format_int Formatted(N); + Dest.append(Formatted.data(), Formatted.data() + Formatted.size()); + } +} + +inline void +Pad3(uint32_t N, MemoryBuffer& Dest) +{ + if (N < 1000) + { + Dest.push_back(static_cast<char>('0' + N / 100)); + Dest.push_back(static_cast<char>('0' + (N / 10) % 10)); + Dest.push_back(static_cast<char>('0' + N % 10)); + } + else + { + AppendInt(static_cast<int>(N), Dest); + } +} + +inline void +PadUint(size_t N, unsigned int Width, MemoryBuffer& Dest) +{ + fmt::format_int Formatted(N); + auto StrLen = static_cast<unsigned int>(Formatted.size()); + if (Width > StrLen) + { + for (unsigned int Pad = 0; Pad < Width - StrLen; ++Pad) + { + Dest.push_back('0'); + } + } + Dest.append(Formatted.data(), Formatted.data() + Formatted.size()); +} + +template<typename ToDuration> +inline ToDuration +TimeFraction(std::chrono::system_clock::time_point Tp) +{ + using std::chrono::duration_cast; + using std::chrono::seconds; + auto Duration = Tp.time_since_epoch(); + auto Secs = duration_cast<seconds>(Duration); + return duration_cast<ToDuration>(Duration) - duration_cast<ToDuration>(Secs); +} + +inline std::tm +SafeLocaltime(std::time_t Time) +{ + std::tm Result{}; +#if defined(_WIN32) + localtime_s(&Result, &Time); +#else + localtime_r(&Time, &Result); +#endif + return Result; +} + +inline const char* +ShortFilename(const char* Path) +{ + if (Path == nullptr) + { + return Path; + } + + const char* It = Path; + const char* LastSep = Path; + while (*It) + { + if (*It == '/' || *It == '\\') + { + LastSep = It + 1; + } + ++It; + } + return LastSep; +} + +inline std::string_view +LevelToShortString(LogLevel Level) +{ + return ToStringView(Level); +} + +} // namespace zen::logging::helpers diff --git a/src/zencore/include/zencore/logging/logger.h b/src/zencore/include/zencore/logging/logger.h new file mode 100644 index 000000000..39d1139a5 --- /dev/null +++ b/src/zencore/include/zencore/logging/logger.h @@ -0,0 +1,63 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/logging/sink.h> + +#include <atomic> +#include <memory> +#include <string_view> + +namespace zen::logging { + +class ErrorHandler +{ +public: + virtual ~ErrorHandler() = default; + virtual void HandleError(const std::string_view& Msg) = 0; +}; + +class Logger : public RefCounted +{ +public: + Logger(std::string_view InName, SinkPtr InSink); + Logger(std::string_view InName, std::span<const SinkPtr> InSinks); + ~Logger(); + + Logger(const Logger&) = delete; + Logger& operator=(const Logger&) = delete; + + void Log(const LogPoint& Point, fmt::format_args Args); + + bool ShouldLog(LogLevel InLevel) const { return InLevel >= m_Level.load(std::memory_order_relaxed); } + + void SetLevel(LogLevel InLevel) { m_Level.store(InLevel, std::memory_order_relaxed); } + LogLevel GetLevel() const { return m_Level.load(std::memory_order_relaxed); } + + void SetFlushLevel(LogLevel InLevel) { m_FlushLevel.store(InLevel, std::memory_order_relaxed); } + LogLevel GetFlushLevel() const { return m_FlushLevel.load(std::memory_order_relaxed); } + + std::string_view Name() const; + + void SetSinks(std::vector<SinkPtr> InSinks); + void AddSink(SinkPtr InSink); + + void SetFormatter(std::unique_ptr<Formatter> InFormatter); + + void SetErrorHandler(ErrorHandler* Handler); + + void Flush(); + + Ref<Logger> Clone(std::string_view NewName) const; + +private: + void SinkIt(const LogMessage& Msg); + void FlushIfNeeded(LogLevel InLevel); + + struct Impl; + std::unique_ptr<Impl> m_Impl; + std::atomic<LogLevel> m_Level{Info}; + std::atomic<LogLevel> m_FlushLevel{Off}; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/logmsg.h b/src/zencore/include/zencore/logging/logmsg.h new file mode 100644 index 000000000..1d8b6b1b7 --- /dev/null +++ b/src/zencore/include/zencore/logging/logmsg.h @@ -0,0 +1,66 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/logbase.h> + +#include <chrono> +#include <string_view> + +namespace zen::logging { + +using LogClock = std::chrono::system_clock; + +struct LogMessage +{ + LogMessage() = default; + + LogMessage(const LogPoint& InPoint, std::string_view InLoggerName, std::string_view InPayload) + : m_LoggerName(InLoggerName) + , m_Level(InPoint.Level) + , m_Time(LogClock::now()) + , m_Source(InPoint.Location) + , m_Payload(InPayload) + , m_Point(&InPoint) + { + } + + std::string_view GetPayload() const { return m_Payload; } + int GetThreadId() const { return m_ThreadId; } + LogClock::time_point GetTime() const { return m_Time; } + LogLevel GetLevel() const { return m_Level; } + std::string_view GetLoggerName() const { return m_LoggerName; } + const SourceLocation& GetSource() const { return m_Source; } + const LogPoint& GetLogPoint() const { return *m_Point; } + + void SetThreadId(int InThreadId) { m_ThreadId = InThreadId; } + void SetPayload(std::string_view InPayload) { m_Payload = InPayload; } + void SetLoggerName(std::string_view InName) { m_LoggerName = InName; } + void SetLevel(LogLevel InLevel) { m_Level = InLevel; } + void SetTime(LogClock::time_point InTime) { m_Time = InTime; } + void SetSource(const SourceLocation& InSource) { m_Source = InSource; } + + mutable size_t ColorRangeStart = 0; + mutable size_t ColorRangeEnd = 0; + +private: + static constexpr LogPoint s_DefaultPoints[LogLevelCount] = { + {{}, Trace, {}}, + {{}, Debug, {}}, + {{}, Info, {}}, + {{}, Warn, {}}, + {{}, Err, {}}, + {{}, Critical, {}}, + {{}, Off, {}}, + }; + + std::string_view m_LoggerName; + LogLevel m_Level = Off; + std::chrono::system_clock::time_point m_Time; + SourceLocation m_Source; + std::string_view m_Payload; + const LogPoint* m_Point = &s_DefaultPoints[Off]; + int m_ThreadId = 0; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/memorybuffer.h b/src/zencore/include/zencore/logging/memorybuffer.h new file mode 100644 index 000000000..cd0ff324f --- /dev/null +++ b/src/zencore/include/zencore/logging/memorybuffer.h @@ -0,0 +1,11 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <fmt/format.h> + +namespace zen::logging { + +using MemoryBuffer = fmt::basic_memory_buffer<char, 250>; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/messageonlyformatter.h b/src/zencore/include/zencore/logging/messageonlyformatter.h new file mode 100644 index 000000000..ce25fe9a6 --- /dev/null +++ b/src/zencore/include/zencore/logging/messageonlyformatter.h @@ -0,0 +1,22 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/logging/formatter.h> +#include <zencore/logging/helpers.h> + +namespace zen::logging { + +class MessageOnlyFormatter : public Formatter +{ +public: + void Format(const LogMessage& Msg, MemoryBuffer& Dest) override + { + helpers::AppendStringView(Msg.GetPayload(), Dest); + Dest.push_back('\n'); + } + + std::unique_ptr<Formatter> Clone() const override { return std::make_unique<MessageOnlyFormatter>(); } +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/msvcsink.h b/src/zencore/include/zencore/logging/msvcsink.h new file mode 100644 index 000000000..48ea1b915 --- /dev/null +++ b/src/zencore/include/zencore/logging/msvcsink.h @@ -0,0 +1,30 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/logging/sink.h> + +#if ZEN_PLATFORM_WINDOWS + +# include <mutex> + +namespace zen::logging { + +class MsvcSink : public Sink +{ +public: + MsvcSink(); + ~MsvcSink() override = default; + + void Log(const LogMessage& Msg) override; + void Flush() override; + void SetFormatter(std::unique_ptr<Formatter> InFormatter) override; + +private: + std::mutex m_Mutex; + std::unique_ptr<Formatter> m_Formatter; +}; + +} // namespace zen::logging + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/src/zencore/include/zencore/logging/nullsink.h b/src/zencore/include/zencore/logging/nullsink.h new file mode 100644 index 000000000..7ac5677c6 --- /dev/null +++ b/src/zencore/include/zencore/logging/nullsink.h @@ -0,0 +1,17 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/logging/sink.h> + +namespace zen::logging { + +class NullSink : public Sink +{ +public: + void Log(const LogMessage& /*Msg*/) override {} + void Flush() override {} + void SetFormatter(std::unique_ptr<Formatter> /*InFormatter*/) override {} +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/registry.h b/src/zencore/include/zencore/logging/registry.h new file mode 100644 index 000000000..a4d3692d2 --- /dev/null +++ b/src/zencore/include/zencore/logging/registry.h @@ -0,0 +1,70 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/logging/logger.h> + +#include <chrono> +#include <memory> +#include <span> +#include <string> +#include <type_traits> +#include <utility> + +namespace zen::logging { + +class Registry +{ +public: + using LogLevels = std::span<const std::pair<std::string, LogLevel>>; + + static Registry& Instance(); + void Shutdown(); + + void Register(Ref<Logger> InLogger); + void Drop(const std::string& Name); + Ref<Logger> Get(const std::string& Name); + + void SetDefaultLogger(Ref<Logger> InLogger); + Logger* DefaultLoggerRaw(); + Ref<Logger> DefaultLogger(); + + void SetGlobalLevel(LogLevel Level); + LogLevel GetGlobalLevel() const; + void SetLevels(LogLevels Levels, LogLevel* DefaultLevel); + + void FlushAll(); + void FlushOn(LogLevel Level); + void FlushEvery(std::chrono::seconds Interval); + + // Change formatter on all registered loggers + void SetFormatter(std::unique_ptr<Formatter> InFormatter); + + // Apply function to all registered loggers. Note that the function will + // be called while the registry mutex is held, so it should be fast and + // not attempt to call back into the registry. + template<typename Func> + void ApplyAll(Func&& F) + { + ApplyAllImpl([](void* Ctx, Ref<Logger> L) { (*static_cast<std::decay_t<Func>*>(Ctx))(std::move(L)); }, &F); + } + + // Set error handler for all loggers in the registry. The handler is called + // if any logger encounters an error during logging or flushing. + // The caller must ensure the handler outlives the registry. + void SetErrorHandler(ErrorHandler* Handler); + +private: + void ApplyAllImpl(void (*Func)(void*, Ref<Logger>), void* Context); + + Registry(); + ~Registry(); + + Registry(const Registry&) = delete; + Registry& operator=(const Registry&) = delete; + + struct Impl; + std::unique_ptr<Impl> m_Impl; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/sink.h b/src/zencore/include/zencore/logging/sink.h new file mode 100644 index 000000000..172176a4e --- /dev/null +++ b/src/zencore/include/zencore/logging/sink.h @@ -0,0 +1,34 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenbase/refcount.h> +#include <zencore/logging/formatter.h> +#include <zencore/logging/logmsg.h> + +#include <atomic> +#include <memory> + +namespace zen::logging { + +class Sink : public RefCounted +{ +public: + virtual ~Sink() = default; + + virtual void Log(const LogMessage& Msg) = 0; + virtual void Flush() = 0; + + virtual void SetFormatter(std::unique_ptr<Formatter> InFormatter) = 0; + + bool ShouldLog(LogLevel InLevel) const { return InLevel >= m_Level.load(std::memory_order_relaxed); } + void SetLevel(LogLevel InLevel) { m_Level.store(InLevel, std::memory_order_relaxed); } + LogLevel GetLevel() const { return m_Level.load(std::memory_order_relaxed); } + +protected: + std::atomic<LogLevel> m_Level{Trace}; +}; + +using SinkPtr = Ref<Sink>; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/tracesink.h b/src/zencore/include/zencore/logging/tracesink.h new file mode 100644 index 000000000..785c51e10 --- /dev/null +++ b/src/zencore/include/zencore/logging/tracesink.h @@ -0,0 +1,27 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/logging/sink.h> + +namespace zen::logging { + +#if ZEN_WITH_TRACE + +/** + * A logging sink that forwards log messages to the trace system. + * + * Work-in-progress, not fully implemented. + */ + +class TraceSink : public Sink +{ +public: + void Log(const LogMessage& Msg) override; + void Flush() override; + void SetFormatter(std::unique_ptr<Formatter> InFormatter) override; +}; + +#endif + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/md5.h b/src/zencore/include/zencore/md5.h index d934dd86b..3b0b7cae6 100644 --- a/src/zencore/include/zencore/md5.h +++ b/src/zencore/include/zencore/md5.h @@ -43,6 +43,8 @@ public: MD5 GetHash(); private: + // Opaque storage for MD5_CTX (104 bytes, aligned to uint32_t) + alignas(4) uint8_t m_Context[104]; }; void md5_forcelink(); // internal diff --git a/src/zencore/include/zencore/meta.h b/src/zencore/include/zencore/meta.h index 82eb5cc30..20ec4ac6f 100644 --- a/src/zencore/include/zencore/meta.h +++ b/src/zencore/include/zencore/meta.h @@ -1,4 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#pragma once /* This file contains utility functions for meta programming * diff --git a/src/zencore/include/zencore/mpscqueue.h b/src/zencore/include/zencore/mpscqueue.h index 19e410d85..d97c433fd 100644 --- a/src/zencore/include/zencore/mpscqueue.h +++ b/src/zencore/include/zencore/mpscqueue.h @@ -22,10 +22,10 @@ namespace zen { template<typename ElementType> struct TypeCompatibleStorage { - ElementType* Data() { return (ElementType*)this; } - const ElementType* Data() const { return (const ElementType*)this; } + ElementType* Data() { return reinterpret_cast<ElementType*>(&Storage); } + const ElementType* Data() const { return reinterpret_cast<const ElementType*>(&Storage); } - alignas(ElementType) char DataMember; + alignas(ElementType) char Storage[sizeof(ElementType)]; }; /** Fast multi-producer/single-consumer unbounded concurrent queue. @@ -58,7 +58,7 @@ public: Tail = Next; Next = Tail->Next.load(std::memory_order_relaxed); - std::destroy_at((ElementType*)&Tail->Value); + std::destroy_at(Tail->Value.Data()); delete Tail; } } @@ -67,7 +67,7 @@ public: void Enqueue(ArgTypes&&... Args) { Node* New = new Node; - new (&New->Value) ElementType(std::forward<ArgTypes>(Args)...); + new (New->Value.Data()) ElementType(std::forward<ArgTypes>(Args)...); Node* Prev = Head.exchange(New, std::memory_order_acq_rel); Prev->Next.store(New, std::memory_order_release); @@ -82,7 +82,7 @@ public: return {}; } - ElementType* ValuePtr = (ElementType*)&Next->Value; + ElementType* ValuePtr = Next->Value.Data(); std::optional<ElementType> Res{std::move(*ValuePtr)}; std::destroy_at(ValuePtr); @@ -100,9 +100,11 @@ private: }; private: - std::atomic<Node*> Head; // accessed only by producers - alignas(hardware_constructive_interference_size) - Node* Tail; // accessed only by consumer, hence should be on a different cache line than `Head` + // Use a fixed constant to avoid GCC's -Winterference-size warning with std::hardware_destructive_interference_size + static constexpr std::size_t CacheLineSize = 64; + + alignas(CacheLineSize) std::atomic<Node*> Head; // accessed only by producers + alignas(CacheLineSize) Node* Tail; // accessed only by consumer, separate cache line from Head }; void mpscqueue_forcelink(); diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index e3b7a70d7..809312c7b 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -9,6 +9,10 @@ namespace zen { +#if ZEN_PLATFORM_WINDOWS +class JobObject; +#endif + /** Basic process abstraction */ class ProcessHandle @@ -46,6 +50,7 @@ private: /** Basic process creation */ + struct CreateProcOptions { enum @@ -63,6 +68,9 @@ struct CreateProcOptions const std::filesystem::path* WorkingDirectory = nullptr; uint32_t Flags = 0; std::filesystem::path StdoutFile; +#if ZEN_PLATFORM_WINDOWS + JobObject* AssignToJob = nullptr; // When set, the process is created suspended, assigned to the job, then resumed +#endif }; #if ZEN_PLATFORM_WINDOWS @@ -99,12 +107,38 @@ private: std::vector<HandleType> m_ProcessHandles; }; +#if ZEN_PLATFORM_WINDOWS +/** Windows Job Object wrapper + * + * When configured with JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, the OS will + * terminate all assigned child processes when the job handle is closed + * (including abnormal termination of the owning process). This provides + * an OS-level guarantee against orphaned child processes. + */ +class JobObject +{ +public: + JobObject(); + ~JobObject(); + JobObject(const JobObject&) = delete; + JobObject& operator=(const JobObject&) = delete; + + void Initialize(); + bool AssignProcess(void* ProcessHandle); + [[nodiscard]] bool IsValid() const; + +private: + void* m_JobHandle = nullptr; +}; +#endif // ZEN_PLATFORM_WINDOWS + bool IsProcessRunning(int pid); bool IsProcessRunning(int pid, std::error_code& OutEc); int GetCurrentProcessId(); int GetProcessId(CreateProcResult ProcId); std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc); +std::string GetProcessCommandLine(int Pid, std::error_code& OutEc); std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle, bool IncludeSelf = true); /** Wait for all threads in the current process to exit (except the calling thread) diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h index faf1238b7..27e5a8a82 100644 --- a/src/zencore/include/zencore/sentryintegration.h +++ b/src/zencore/include/zencore/sentryintegration.h @@ -11,11 +11,9 @@ #if ZEN_USE_SENTRY -# include <memory> +# include <zencore/logging/logger.h> -ZEN_THIRD_PARTY_INCLUDES_START -# include <spdlog/logger.h> -ZEN_THIRD_PARTY_INCLUDES_END +# include <memory> namespace sentry { @@ -42,6 +40,7 @@ public: }; void Initialize(const Config& Conf, const std::string& CommandLine); + void Close(); void LogStartupInformation(); static void ClearCaches(); @@ -53,7 +52,7 @@ private: std::string m_SentryUserName; std::string m_SentryHostName; std::string m_SentryId; - std::shared_ptr<spdlog::logger> m_SentryLogger; + Ref<logging::Logger> m_SentryLogger; }; } // namespace zen diff --git a/src/zencore/include/zencore/sharedbuffer.h b/src/zencore/include/zencore/sharedbuffer.h index c57e9f568..3d4c19282 100644 --- a/src/zencore/include/zencore/sharedbuffer.h +++ b/src/zencore/include/zencore/sharedbuffer.h @@ -116,14 +116,15 @@ public: inline void Reset() { m_Buffer = nullptr; } inline bool GetFileReference(IoBufferFileReference& OutRef) const { - if (const IoBufferExtendedCore* Core = m_Buffer->ExtendedCore()) + if (!IsNull()) { - return Core->GetFileReference(OutRef); - } - else - { - return false; + if (const IoBufferExtendedCore* Core = m_Buffer->ExtendedCore()) + { + return Core->GetFileReference(OutRef); + } } + + return false; } [[nodiscard]] MemoryView GetView() const diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h index cbff6454f..4deca63ed 100644 --- a/src/zencore/include/zencore/string.h +++ b/src/zencore/include/zencore/string.h @@ -8,7 +8,6 @@ #include <stdint.h> #include <string.h> #include <charconv> -#include <codecvt> #include <compare> #include <concepts> #include <optional> @@ -51,7 +50,7 @@ StringLength(const wchar_t* str) return wcslen(str); } -inline bool +inline int StringCompare(const char16_t* s1, const char16_t* s2) { char16_t c1, c2; @@ -66,7 +65,7 @@ StringCompare(const char16_t* s1, const char16_t* s2) ++s1; ++s2; } - return uint16_t(c1) - uint16_t(c2); + return int(uint16_t(c1)) - int(uint16_t(c2)); } inline bool @@ -122,10 +121,10 @@ public: StringBuilderImpl() = default; ~StringBuilderImpl(); - StringBuilderImpl(const StringBuilderImpl&) = delete; - StringBuilderImpl(const StringBuilderImpl&&) = delete; + StringBuilderImpl(const StringBuilderImpl&) = delete; + StringBuilderImpl(StringBuilderImpl&&) = delete; const StringBuilderImpl& operator=(const StringBuilderImpl&) = delete; - const StringBuilderImpl& operator=(const StringBuilderImpl&&) = delete; + StringBuilderImpl& operator=(StringBuilderImpl&&) = delete; inline size_t AddUninitialized(size_t Count) { @@ -374,9 +373,9 @@ protected: [[noreturn]] void Fail(const char* FailReason); // note: throws exception - C* m_Base; - C* m_CurPos; - C* m_End; + C* m_Base = nullptr; + C* m_CurPos = nullptr; + C* m_End = nullptr; bool m_IsDynamic = false; bool m_IsExtendable = false; }; @@ -773,8 +772,9 @@ std::optional<T> ParseInt(const std::string_view& Input) { T Out = 0; - const std::from_chars_result Result = std::from_chars(Input.data(), Input.data() + Input.size(), Out); - if (Result.ec == std::errc::invalid_argument || Result.ec == std::errc::result_out_of_range) + const char* End = Input.data() + Input.size(); + const std::from_chars_result Result = std::from_chars(Input.data(), End, Out); + if (Result.ec == std::errc::invalid_argument || Result.ec == std::errc::result_out_of_range || Result.ptr != End) { return std::nullopt; } @@ -797,6 +797,22 @@ HashStringDjb2(const std::string_view& InString) } constexpr uint32_t +HashStringDjb2(const std::span<const std::string_view> InStrings) +{ + uint32_t HashValue = 5381; + + for (const std::string_view& String : InStrings) + { + for (int CurChar : String) + { + HashValue = HashValue * 33 + CurChar; + } + } + + return HashValue; +} + +constexpr uint32_t HashStringAsLowerDjb2(const std::string_view& InString) { uint32_t HashValue = 5381; @@ -1249,6 +1265,8 @@ private: uint64_t LoMask, HiMask; }; +std::string HideSensitiveString(std::string_view String); + ////////////////////////////////////////////////////////////////////////// void string_forcelink(); // internal diff --git a/src/zencore/include/zencore/system.h b/src/zencore/include/zencore/system.h index aec2e0ce4..a67999e52 100644 --- a/src/zencore/include/zencore/system.h +++ b/src/zencore/include/zencore/system.h @@ -4,6 +4,8 @@ #include <zencore/zencore.h> +#include <chrono> +#include <memory> #include <string> namespace zen { @@ -12,6 +14,8 @@ class CbWriter; std::string GetMachineName(); std::string_view GetOperatingSystemName(); +std::string GetOperatingSystemVersion(); +std::string_view GetRuntimePlatformName(); // "windows", "wine", "linux", or "macos" std::string_view GetCpuName(); struct SystemMetrics @@ -25,6 +29,14 @@ struct SystemMetrics uint64_t AvailVirtualMemoryMiB = 0; uint64_t PageFileMiB = 0; uint64_t AvailPageFileMiB = 0; + uint64_t UptimeSeconds = 0; +}; + +/// Extended metrics that include CPU usage percentage, which requires +/// stateful delta tracking via SystemMetricsTracker. +struct ExtendedSystemMetrics : SystemMetrics +{ + float CpuUsagePercent = 0.0f; }; SystemMetrics GetSystemMetrics(); @@ -32,6 +44,31 @@ SystemMetrics GetSystemMetrics(); void SetCpuCountForReporting(int FakeCpuCount); SystemMetrics GetSystemMetricsForReporting(); +ExtendedSystemMetrics ApplyReportingOverrides(ExtendedSystemMetrics Metrics); + void Describe(const SystemMetrics& Metrics, CbWriter& Writer); +void Describe(const ExtendedSystemMetrics& Metrics, CbWriter& Writer); + +/// Stateful tracker that computes CPU usage as a delta between consecutive +/// Query() calls. The first call returns CpuUsagePercent = 0 (no previous +/// sample). Thread-safe: concurrent calls are serialised internally. +/// CPU sampling is rate-limited to MinInterval (default 1 s); calls that +/// arrive sooner return the previously cached value. +class SystemMetricsTracker +{ +public: + explicit SystemMetricsTracker(std::chrono::milliseconds MinInterval = std::chrono::seconds(1)); + ~SystemMetricsTracker(); + + SystemMetricsTracker(const SystemMetricsTracker&) = delete; + SystemMetricsTracker& operator=(const SystemMetricsTracker&) = delete; + + /// Collect current metrics. CPU usage is computed as delta since last Query(). + ExtendedSystemMetrics Query(); + +private: + struct Impl; + std::unique_ptr<Impl> m_Impl; +}; } // namespace zen diff --git a/src/zencore/include/zencore/testing.h b/src/zencore/include/zencore/testing.h index a00ee3166..8410216c4 100644 --- a/src/zencore/include/zencore/testing.h +++ b/src/zencore/include/zencore/testing.h @@ -43,8 +43,9 @@ public: TestRunner(); ~TestRunner(); - int ApplyCommandLine(int argc, char const* const* argv); - int Run(); + void SetDefaultSuiteFilter(const char* Pattern); + int ApplyCommandLine(int Argc, char const* const* Argv); + int Run(); private: struct Impl; @@ -59,6 +60,8 @@ private: return Runner.Run(); \ }() +int RunTestMain(int Argc, char* Argv[], const char* ExecutableName, void (*ForceLink)()); + } // namespace zen::testing #endif diff --git a/src/zencore/include/zencore/testutils.h b/src/zencore/include/zencore/testutils.h index e2a4f8346..2a789d18f 100644 --- a/src/zencore/include/zencore/testutils.h +++ b/src/zencore/include/zencore/testutils.h @@ -59,6 +59,33 @@ struct TrueType static const bool Enabled = true; }; +namespace utf8test { + + // 2-byte UTF-8 (Latin extended) + static constexpr const char kLatin[] = u8"café_résumé"; + static constexpr const wchar_t kLatinW[] = L"café_résumé"; + + // 2-byte UTF-8 (Cyrillic) + static constexpr const char kCyrillic[] = u8"данные"; + static constexpr const wchar_t kCyrillicW[] = L"данные"; + + // 3-byte UTF-8 (CJK) + static constexpr const char kCJK[] = u8"日本語"; + static constexpr const wchar_t kCJKW[] = L"日本語"; + + // Mixed scripts + static constexpr const char kMixed[] = u8"zen_éд日"; + static constexpr const wchar_t kMixedW[] = L"zen_éд日"; + + // 4-byte UTF-8 (supplementary plane) — string tests only, NOT filesystem + static constexpr const char kEmoji[] = u8"📦"; + static constexpr const wchar_t kEmojiW[] = L"📦"; + + // BMP-only test strings suitable for filesystem use + static constexpr const char* kFilenameSafe[] = {kLatin, kCyrillic, kCJK, kMixed}; + +} // namespace utf8test + } // namespace zen #endif // ZEN_WITH_TESTS diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h index de8f9399c..d0d710ee8 100644 --- a/src/zencore/include/zencore/thread.h +++ b/src/zencore/include/zencore/thread.h @@ -58,17 +58,27 @@ public: } private: - RwLock* m_Lock; + RwLock* m_Lock = nullptr; }; - inline void WithSharedLock(auto&& Fun) + inline auto WithSharedLock(auto&& Fun) { SharedLockScope $(*this); - Fun(); + return Fun(); } struct ExclusiveLockScope { + ExclusiveLockScope(const ExclusiveLockScope& Rhs) = delete; + ExclusiveLockScope(ExclusiveLockScope&& Rhs) : m_Lock(Rhs.m_Lock) { Rhs.m_Lock = nullptr; } + ExclusiveLockScope& operator=(ExclusiveLockScope&& Rhs) + { + ReleaseNow(); + m_Lock = Rhs.m_Lock; + Rhs.m_Lock = nullptr; + return *this; + } + ExclusiveLockScope& operator=(const ExclusiveLockScope& Rhs) = delete; ExclusiveLockScope(RwLock& Lock) : m_Lock(&Lock) { Lock.AcquireExclusive(); } ~ExclusiveLockScope() { ReleaseNow(); } @@ -82,13 +92,13 @@ public: } private: - RwLock* m_Lock; + RwLock* m_Lock = nullptr; }; - inline void WithExclusiveLock(auto&& Fun) + inline auto WithExclusiveLock(auto&& Fun) { ExclusiveLockScope $(*this); - Fun(); + return Fun(); } private: @@ -195,7 +205,7 @@ public: // false positive completion results. void AddCount(std::ptrdiff_t Count) { - std::atomic_ptrdiff_t Old = Counter.fetch_add(Count); + std::ptrdiff_t Old = Counter.fetch_add(Count); ZEN_ASSERT(Old > 0); } diff --git a/src/zencore/include/zencore/trace.h b/src/zencore/include/zencore/trace.h index 99a565151..d17e018ea 100644 --- a/src/zencore/include/zencore/trace.h +++ b/src/zencore/include/zencore/trace.h @@ -13,6 +13,7 @@ ZEN_THIRD_PARTY_INCLUDES_START # define TRACE_IMPLEMENT 0 #endif #include <trace.h> +#include <lane_trace.h> #undef TRACE_IMPLEMENT ZEN_THIRD_PARTY_INCLUDES_END diff --git a/src/zencore/include/zencore/varint.h b/src/zencore/include/zencore/varint.h index 9fe905f25..43ca14d38 100644 --- a/src/zencore/include/zencore/varint.h +++ b/src/zencore/include/zencore/varint.h @@ -1,4 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#pragma once #include "intmath.h" diff --git a/src/zencore/include/zencore/xxhash.h b/src/zencore/include/zencore/xxhash.h index fc55b513b..f79d39b61 100644 --- a/src/zencore/include/zencore/xxhash.h +++ b/src/zencore/include/zencore/xxhash.h @@ -87,7 +87,7 @@ struct XXH3_128Stream } private: - XXH3_state_s m_State; + XXH3_state_s m_State{}; }; struct XXH3_128Stream_deprecated diff --git a/src/zencore/include/zencore/zencore.h b/src/zencore/include/zencore/zencore.h index 177a19fff..a31950b0b 100644 --- a/src/zencore/include/zencore/zencore.h +++ b/src/zencore/include/zencore/zencore.h @@ -70,26 +70,36 @@ protected: } // namespace zen -#define ZEN_ASSERT(x, ...) \ - do \ - { \ - if (x) [[unlikely]] \ - break; \ - zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, #x); \ +#define ZEN_ASSERT(x, ...) \ + do \ + { \ + if (x) [[unlikely]] \ + break; \ + zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, ZEN_ASSERT_MSG_(#x, ##__VA_ARGS__)); \ } while (false) #ifndef NDEBUG -# define ZEN_ASSERT_SLOW(x, ...) \ - do \ - { \ - if (x) [[unlikely]] \ - break; \ - zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, #x); \ +# define ZEN_ASSERT_SLOW(x, ...) \ + do \ + { \ + if (x) [[unlikely]] \ + break; \ + zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, ZEN_ASSERT_MSG_(#x, ##__VA_ARGS__)); \ } while (false) #else # define ZEN_ASSERT_SLOW(x, ...) #endif +// Internal: select between "expr" and "expr: message" forms. +// With no extra args: ZEN_ASSERT_MSG_("expr") -> "expr" +// With a message arg: ZEN_ASSERT_MSG_("expr", "msg") -> "expr" ": " "msg" +// With fmt-style args: ZEN_ASSERT_MSG_("expr", "msg", args...) -> "expr" ": " "msg" +// The extra fmt args are silently discarded here — use ZEN_ASSERT_FORMAT for those. +#define ZEN_ASSERT_MSG_SELECT_(_1, _2, N, ...) N +#define ZEN_ASSERT_MSG_1_(expr) expr +#define ZEN_ASSERT_MSG_2_(expr, msg, ...) expr ": " msg +#define ZEN_ASSERT_MSG_(expr, ...) ZEN_ASSERT_MSG_SELECT_(unused, ##__VA_ARGS__, ZEN_ASSERT_MSG_2_, ZEN_ASSERT_MSG_1_)(expr, ##__VA_ARGS__) + ////////////////////////////////////////////////////////////////////////// #define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false, __VA_ARGS__) diff --git a/src/zencore/intmath.cpp b/src/zencore/intmath.cpp index 5a686dc8e..fedf76edc 100644 --- a/src/zencore/intmath.cpp +++ b/src/zencore/intmath.cpp @@ -19,6 +19,8 @@ intmath_forcelink() { } +TEST_SUITE_BEGIN("core.intmath"); + TEST_CASE("intmath") { CHECK(FloorLog2(0x00) == 0); @@ -43,6 +45,12 @@ TEST_CASE("intmath") CHECK(FloorLog2_64(0x0000'0001'0000'0000ull) == 32); CHECK(FloorLog2_64(0x8000'0000'0000'0000ull) == 63); + CHECK(CountLeadingZeros(0x8000'0000u) == 0); + CHECK(CountLeadingZeros(0x0000'0000u) == 32); + CHECK(CountLeadingZeros(0x0000'0001u) == 31); + CHECK(CountLeadingZeros(0x0000'8000u) == 16); + CHECK(CountLeadingZeros(0x0001'0000u) == 15); + CHECK(CountLeadingZeros64(0x8000'0000'0000'0000ull) == 0); CHECK(CountLeadingZeros64(0x0000'0000'0000'0000ull) == 64); CHECK(CountLeadingZeros64(0x0000'0000'0000'0001ull) == 63); @@ -60,6 +68,8 @@ TEST_CASE("intmath") CHECK(ByteSwap(uint64_t(0x214d'6172'7469'6e21ull)) == 0x216e'6974'7261'4d21ull); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp index be9b39e7a..c47c54981 100644 --- a/src/zencore/iobuffer.cpp +++ b/src/zencore/iobuffer.cpp @@ -592,15 +592,17 @@ IoBufferBuilder::ReadFromFileMaybe(const IoBuffer& InBuffer) } IoBuffer -IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t Size) +IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t Size, ZenContentType ContentType) { ZEN_TRACE_CPU("IoBufferBuilder::MakeFromFileHandle"); - return IoBuffer(IoBuffer::BorrowedFile, FileHandle, Offset, Size); + IoBuffer Buffer(IoBuffer::BorrowedFile, FileHandle, Offset, Size); + Buffer.SetContentType(ContentType); + return Buffer; } IoBuffer -IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size) +IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size, ZenContentType ContentType) { ZEN_TRACE_CPU("IoBufferBuilder::MakeFromFile"); @@ -632,8 +634,6 @@ IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Of FileSize = Stat.st_size; #endif // ZEN_PLATFORM_WINDOWS - // TODO: should validate that offset is in range - if (Size == ~0ull) { Size = FileSize - Offset; @@ -652,7 +652,9 @@ IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Of #if ZEN_PLATFORM_WINDOWS void* Fd = DataFile.Detach(); #endif - return IoBuffer(IoBuffer::File, (void*)uintptr_t(Fd), Offset, Size, Offset == 0 && Size == FileSize); + IoBuffer NewBuffer(IoBuffer::File, (void*)uintptr_t(Fd), Offset, Size, Offset == 0 && Size == FileSize); + NewBuffer.SetContentType(ContentType); + return NewBuffer; } #if !ZEN_PLATFORM_WINDOWS @@ -664,7 +666,7 @@ IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Of } IoBuffer -IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName) +IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName, ZenContentType ContentType) { ZEN_TRACE_CPU("IoBufferBuilder::MakeFromTemporaryFile"); @@ -703,7 +705,9 @@ IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName) Handle = (void*)uintptr_t(Fd); #endif // ZEN_PLATFORM_WINDOWS - return IoBuffer(IoBuffer::File, Handle, 0, FileSize, /*IsWholeFile*/ true); + IoBuffer NewBuffer(IoBuffer::File, Handle, 0, FileSize, /*IsWholeFile*/ true); + NewBuffer.SetContentType(ContentType); + return NewBuffer; } ////////////////////////////////////////////////////////////////////////// @@ -715,6 +719,8 @@ iobuffer_forcelink() { } +TEST_SUITE_BEGIN("core.iobuffer"); + TEST_CASE("IoBuffer") { zen::IoBuffer buffer1; @@ -752,6 +758,8 @@ TEST_CASE("IoBuffer.mmap") # endif } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/jobqueue.cpp b/src/zencore/jobqueue.cpp index 75c1be42b..d6a8a6479 100644 --- a/src/zencore/jobqueue.cpp +++ b/src/zencore/jobqueue.cpp @@ -90,7 +90,7 @@ public: uint64_t NewJobId = IdGenerator.fetch_add(1); if (NewJobId == 0) { - IdGenerator.fetch_add(1); + NewJobId = IdGenerator.fetch_add(1); } RefPtr<Job> NewJob(new Job()); NewJob->Queue = this; @@ -129,7 +129,7 @@ public: QueuedJobs.erase(It); } }); - ZEN_ERROR("Failed to schedule job {}:'{}' to job queue. Reason: ''", NewJob->Id.Id, NewJob->Name, Ex.what()); + ZEN_ERROR("Failed to schedule job {}:'{}' to job queue. Reason: '{}'", NewJob->Id.Id, NewJob->Name, Ex.what()); throw; } } @@ -221,11 +221,11 @@ public: std::vector<JobInfo> Jobs; QueueLock.WithSharedLock([&]() { - for (auto It : RunningJobs) + for (const auto& It : RunningJobs) { Jobs.push_back({.Id = JobId{It.first}, .Status = JobStatus::Running}); } - for (auto It : CompletedJobs) + for (const auto& It : CompletedJobs) { if (IsStale(It.second->EndTick)) { @@ -234,7 +234,7 @@ public: } Jobs.push_back({.Id = JobId{It.first}, .Status = JobStatus::Completed}); } - for (auto It : AbortedJobs) + for (const auto& It : AbortedJobs) { if (IsStale(It.second->EndTick)) { @@ -243,7 +243,7 @@ public: } Jobs.push_back({.Id = JobId{It.first}, .Status = JobStatus::Aborted}); } - for (auto It : QueuedJobs) + for (const auto& It : QueuedJobs) { Jobs.push_back({.Id = It->Id, .Status = JobStatus::Queued}); } @@ -337,7 +337,7 @@ public: std::atomic_bool InitializedFlag = false; RwLock QueueLock; std::deque<RefPtr<Job>> QueuedJobs; - std::unordered_map<uint64_t, Job*> RunningJobs; + std::unordered_map<uint64_t, RefPtr<Job>> RunningJobs; std::unordered_map<uint64_t, RefPtr<Job>> CompletedJobs; std::unordered_map<uint64_t, RefPtr<Job>> AbortedJobs; @@ -429,20 +429,16 @@ JobQueue::ToString(JobStatus Status) { case JobQueue::JobStatus::Queued: return "Queued"sv; - break; case JobQueue::JobStatus::Running: return "Running"sv; - break; case JobQueue::JobStatus::Aborted: return "Aborted"sv; - break; case JobQueue::JobStatus::Completed: return "Completed"sv; - break; default: ZEN_ASSERT(false); + return ""sv; } - return ""sv; } std::unique_ptr<JobQueue> @@ -460,6 +456,8 @@ jobqueue_forcelink() { } +TEST_SUITE_BEGIN("core.jobqueue"); + TEST_CASE("JobQueue") { std::unique_ptr<JobQueue> Queue(MakeJobQueue(2, "queue")); @@ -580,6 +578,8 @@ TEST_CASE("JobQueue") } JobsLatch.Wait(); } + +TEST_SUITE_END(); #endif } // namespace zen diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp index a6697c443..099518637 100644 --- a/src/zencore/logging.cpp +++ b/src/zencore/logging.cpp @@ -2,208 +2,128 @@ #include "zencore/logging.h" +#include <zencore/logging/ansicolorsink.h> +#include <zencore/logging/logger.h> +#include <zencore/logging/messageonlyformatter.h> +#include <zencore/logging/nullsink.h> +#include <zencore/logging/registry.h> #include <zencore/string.h> #include <zencore/testing.h> #include <zencore/thread.h> #include <zencore/memory/llm.h> -ZEN_THIRD_PARTY_INCLUDES_START -#include <spdlog/details/registry.h> -#include <spdlog/sinks/null_sink.h> -#include <spdlog/sinks/stdout_color_sinks.h> -#include <spdlog/spdlog.h> -ZEN_THIRD_PARTY_INCLUDES_END +#include <mutex> #if ZEN_PLATFORM_WINDOWS # pragma section(".zlog$a", read) -# pragma section(".zlog$f", read) -# pragma section(".zlog$m", read) -# pragma section(".zlog$s", read) +# pragma section(".zlog$l", read) # pragma section(".zlog$z", read) #endif namespace zen { -// We shadow the underlying spdlog default logger, in order to avoid a bunch of overhead LoggerRef TheDefaultLogger; +LoggerRef +Log() +{ + if (TheDefaultLogger) + { + return TheDefaultLogger; + } + return zen::logging::ConsoleLog(); +} + } // namespace zen namespace zen::logging { -using MemoryBuffer_t = fmt::basic_memory_buffer<char, 250>; - -struct LoggingContext -{ - inline LoggingContext(); - inline ~LoggingContext(); - - zen::logging::MemoryBuffer_t MessageBuffer; - - inline std::string_view Message() const { return std::string_view(MessageBuffer.data(), MessageBuffer.size()); } -}; +////////////////////////////////////////////////////////////////////////// -LoggingContext::LoggingContext() +LoggerRef +LogCategory::Logger() { -} + // This should be thread safe since zen::logging::Get() will return + // the same logger instance for the same category name. Also the + // LoggerRef is simply a pointer. + if (!m_LoggerRef) + { + m_LoggerRef = zen::logging::Get(m_CategoryName); + } -LoggingContext::~LoggingContext() -{ + return m_LoggerRef; } -////////////////////////////////////////////////////////////////////////// - static inline bool -IsErrorLevel(int LogLevel) +IsErrorLevel(LogLevel InLevel) { - return (LogLevel == zen::logging::level::Err || LogLevel == zen::logging::level::Critical); + return (InLevel == Err || InLevel == Critical); }; -static_assert(sizeof(spdlog::source_loc) == sizeof(SourceLocation)); -static_assert(offsetof(spdlog::source_loc, filename) == offsetof(SourceLocation, filename)); -static_assert(offsetof(spdlog::source_loc, line) == offsetof(SourceLocation, line)); -static_assert(offsetof(spdlog::source_loc, funcname) == offsetof(SourceLocation, funcname)); - void -EmitLogMessage(LoggerRef& Logger, int LogLevel, const std::string_view Message) +EmitLogMessage(LoggerRef& Logger, const LogPoint& Lp, fmt::format_args Args) { ZEN_MEMSCOPE(ELLMTag::Logging); - const spdlog::level::level_enum InLevel = (spdlog::level::level_enum)LogLevel; - Logger.SpdLogger->log(InLevel, Message); - if (IsErrorLevel(LogLevel)) - { - if (LoggerRef ErrLogger = zen::logging::ErrorLog()) - { - ErrLogger.SpdLogger->log(InLevel, Message); - } - } -} -void -EmitLogMessage(LoggerRef& Logger, int LogLevel, std::string_view Format, fmt::format_args Args) -{ - ZEN_MEMSCOPE(ELLMTag::Logging); - zen::logging::LoggingContext LogCtx; - fmt::vformat_to(fmt::appender(LogCtx.MessageBuffer), Format, Args); - zen::logging::EmitLogMessage(Logger, LogLevel, LogCtx.Message()); -} + Logger->Log(Lp, Args); -void -EmitLogMessage(LoggerRef& Logger, const SourceLocation& InLocation, int LogLevel, const std::string_view Message) -{ - ZEN_MEMSCOPE(ELLMTag::Logging); - const spdlog::source_loc& Location = *reinterpret_cast<const spdlog::source_loc*>(&InLocation); - const spdlog::level::level_enum InLevel = (spdlog::level::level_enum)LogLevel; - Logger.SpdLogger->log(Location, InLevel, Message); - if (IsErrorLevel(LogLevel)) + if (IsErrorLevel(Lp.Level)) { if (LoggerRef ErrLogger = zen::logging::ErrorLog()) { - ErrLogger.SpdLogger->log(Location, InLevel, Message); + ErrLogger->Log(Lp, Args); } } } void -EmitLogMessage(LoggerRef& Logger, const SourceLocation& InLocation, int LogLevel, std::string_view Format, fmt::format_args Args) +EmitConsoleLogMessage(const LogPoint& Lp, fmt::format_args Args) { ZEN_MEMSCOPE(ELLMTag::Logging); - zen::logging::LoggingContext LogCtx; - fmt::vformat_to(fmt::appender(LogCtx.MessageBuffer), Format, Args); - zen::logging::EmitLogMessage(Logger, InLocation, LogLevel, LogCtx.Message()); -} - -void -EmitConsoleLogMessage(int LogLevel, const std::string_view Message) -{ - ZEN_MEMSCOPE(ELLMTag::Logging); - const spdlog::level::level_enum InLevel = (spdlog::level::level_enum)LogLevel; - ConsoleLog().SpdLogger->log(InLevel, Message); -} - -#define ZEN_COLOR_YELLOW "\033[0;33m" -#define ZEN_COLOR_RED "\033[0;31m" -#define ZEN_BRIGHT_COLOR_RED "\033[1;31m" -#define ZEN_COLOR_RESET "\033[0m" - -void -EmitConsoleLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) -{ - ZEN_MEMSCOPE(ELLMTag::Logging); - zen::logging::LoggingContext LogCtx; - - // We are not using a format option for console which include log level since it would interfere with normal console output - - const spdlog::level::level_enum InLevel = (spdlog::level::level_enum)LogLevel; - switch (InLevel) - { - case spdlog::level::level_enum::warn: - fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_COLOR_YELLOW "Warning: " ZEN_COLOR_RESET); - break; - case spdlog::level::level_enum::err: - fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_BRIGHT_COLOR_RED "Error: " ZEN_COLOR_RESET); - break; - case spdlog::level::level_enum::critical: - fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_COLOR_RED "Critical: " ZEN_COLOR_RESET); - break; - default: - break; - } - fmt::vformat_to(fmt::appender(LogCtx.MessageBuffer), Format, Args); - zen::logging::EmitConsoleLogMessage(LogLevel, LogCtx.Message()); + ConsoleLog()->Log(Lp, Args); } } // namespace zen::logging -namespace zen::logging::level { +namespace zen::logging { -spdlog::level::level_enum -to_spdlog_level(LogLevel NewLogLevel) -{ - return static_cast<spdlog::level::level_enum>((int)NewLogLevel); -} +constinit std::string_view LevelNames[] = {std::string_view("trace", 5), + std::string_view("debug", 5), + std::string_view("info", 4), + std::string_view("warning", 7), + std::string_view("error", 5), + std::string_view("critical", 8), + std::string_view("off", 3)}; LogLevel -to_logging_level(spdlog::level::level_enum NewLogLevel) -{ - return static_cast<LogLevel>((int)NewLogLevel); -} - -constinit std::string_view LevelNames[] = {ZEN_LEVEL_NAME_TRACE, - ZEN_LEVEL_NAME_DEBUG, - ZEN_LEVEL_NAME_INFO, - ZEN_LEVEL_NAME_WARNING, - ZEN_LEVEL_NAME_ERROR, - ZEN_LEVEL_NAME_CRITICAL, - ZEN_LEVEL_NAME_OFF}; - -level::LogLevel ParseLogLevelString(std::string_view Name) { - for (int Level = 0; Level < level::LogLevelCount; ++Level) + for (int Level = 0; Level < LogLevelCount; ++Level) { if (LevelNames[Level] == Name) - return static_cast<level::LogLevel>(Level); + { + return static_cast<LogLevel>(Level); + } } if (Name == "warn") { - return level::Warn; + return Warn; } if (Name == "err") { - return level::Err; + return Err; } - return level::Off; + return Off; } std::string_view -ToStringView(level::LogLevel Level) +ToStringView(LogLevel Level) { - if (int(Level) < level::LogLevelCount) + if (int(Level) < LogLevelCount) { return LevelNames[int(Level)]; } @@ -211,17 +131,17 @@ ToStringView(level::LogLevel Level) return "None"; } -} // namespace zen::logging::level +} // namespace zen::logging ////////////////////////////////////////////////////////////////////////// namespace zen::logging { RwLock LogLevelsLock; -std::string LogLevels[level::LogLevelCount]; +std::string LogLevels[LogLevelCount]; void -ConfigureLogLevels(level::LogLevel Level, std::string_view Loggers) +ConfigureLogLevels(LogLevel Level, std::string_view Loggers) { ZEN_MEMSCOPE(ELLMTag::Logging); @@ -230,18 +150,18 @@ ConfigureLogLevels(level::LogLevel Level, std::string_view Loggers) } void -RefreshLogLevels(level::LogLevel* DefaultLevel) +RefreshLogLevels(LogLevel* DefaultLevel) { ZEN_MEMSCOPE(ELLMTag::Logging); - spdlog::details::registry::log_levels Levels; + std::vector<std::pair<std::string, LogLevel>> Levels; { RwLock::SharedLockScope _(LogLevelsLock); - for (int i = 0; i < level::LogLevelCount; ++i) + for (int i = 0; i < LogLevelCount; ++i) { - level::LogLevel CurrentLevel{i}; + LogLevel CurrentLevel{i}; std::string_view Spec = LogLevels[i]; @@ -251,7 +171,7 @@ RefreshLogLevels(level::LogLevel* DefaultLevel) if (auto CommaPos = Spec.find_first_of(','); CommaPos != std::string_view::npos) { - LoggerName = Spec.substr(CommaPos + 1); + LoggerName = Spec.substr(0, CommaPos); Spec.remove_prefix(CommaPos + 1); } else @@ -260,24 +180,16 @@ RefreshLogLevels(level::LogLevel* DefaultLevel) Spec = {}; } - Levels[LoggerName] = to_spdlog_level(CurrentLevel); + Levels.emplace_back(std::move(LoggerName), CurrentLevel); } } } - if (DefaultLevel) - { - spdlog::level::level_enum SpdDefaultLevel = to_spdlog_level(*DefaultLevel); - spdlog::details::registry::instance().set_levels(Levels, &SpdDefaultLevel); - } - else - { - spdlog::details::registry::instance().set_levels(Levels, nullptr); - } + Registry::Instance().SetLevels(Levels, DefaultLevel); } void -RefreshLogLevels(level::LogLevel DefaultLevel) +RefreshLogLevels(LogLevel DefaultLevel) { RefreshLogLevels(&DefaultLevel); } @@ -289,21 +201,21 @@ RefreshLogLevels() } void -SetLogLevel(level::LogLevel NewLogLevel) +SetLogLevel(LogLevel NewLogLevel) { - spdlog::set_level(to_spdlog_level(NewLogLevel)); + Registry::Instance().SetGlobalLevel(NewLogLevel); } -level::LogLevel +LogLevel GetLogLevel() { - return level::to_logging_level(spdlog::get_level()); + return Registry::Instance().GetGlobalLevel(); } LoggerRef Default() { - ZEN_ASSERT(TheDefaultLogger); + ZEN_ASSERT(TheDefaultLogger, "logging::InitializeLogging() must be called before using the logger"); return TheDefaultLogger; } @@ -312,10 +224,10 @@ SetDefault(std::string_view NewDefaultLoggerId) { ZEN_MEMSCOPE(ELLMTag::Logging); - auto NewDefaultLogger = spdlog::get(std::string(NewDefaultLoggerId)); + Ref<Logger> NewDefaultLogger = Registry::Instance().Get(std::string(NewDefaultLoggerId)); ZEN_ASSERT(NewDefaultLogger); - spdlog::set_default_logger(NewDefaultLogger); + Registry::Instance().SetDefaultLogger(NewDefaultLogger); TheDefaultLogger = LoggerRef(*NewDefaultLogger); } @@ -338,11 +250,11 @@ SetErrorLog(std::string_view NewErrorLoggerId) } else { - auto NewErrorLogger = spdlog::get(std::string(NewErrorLoggerId)); + Ref<Logger> NewErrorLogger = Registry::Instance().Get(std::string(NewErrorLoggerId)); ZEN_ASSERT(NewErrorLogger); - TheErrorLogger = LoggerRef(*NewErrorLogger.get()); + TheErrorLogger = LoggerRef(*NewErrorLogger.Get()); } } @@ -353,39 +265,75 @@ Get(std::string_view Name) { ZEN_MEMSCOPE(ELLMTag::Logging); - std::shared_ptr<spdlog::logger> Logger = spdlog::get(std::string(Name)); + Ref<Logger> FoundLogger = Registry::Instance().Get(std::string(Name)); - if (!Logger) + if (!FoundLogger) { g_LoggerMutex.WithExclusiveLock([&] { - Logger = spdlog::get(std::string(Name)); + FoundLogger = Registry::Instance().Get(std::string(Name)); - if (!Logger) + if (!FoundLogger) { - Logger = Default().SpdLogger->clone(std::string(Name)); - spdlog::apply_logger_env_levels(Logger); - spdlog::register_logger(Logger); + FoundLogger = Default()->Clone(std::string(Name)); + Registry::Instance().Register(FoundLogger); } }); } - return *Logger; + return *FoundLogger; } -std::once_flag ConsoleInitFlag; -std::shared_ptr<spdlog::logger> ConLogger; +std::once_flag ConsoleInitFlag; +Ref<Logger> ConLogger; void SuppressConsoleLog() { + ZEN_MEMSCOPE(ELLMTag::Logging); + if (ConLogger) { - spdlog::drop("console"); + Registry::Instance().Drop("console"); ConLogger = {}; } - ConLogger = spdlog::null_logger_mt("console"); + + SinkPtr NullSinkPtr(new NullSink()); + ConLogger = Ref<Logger>(new Logger("console", std::vector<SinkPtr>{NullSinkPtr})); + Registry::Instance().Register(ConLogger); } +#define ZEN_COLOR_YELLOW "\033[0;33m" +#define ZEN_COLOR_RED "\033[0;31m" +#define ZEN_BRIGHT_COLOR_RED "\033[1;31m" +#define ZEN_COLOR_RESET "\033[0m" + +class ConsoleFormatter : public Formatter +{ +public: + void Format(const LogMessage& Msg, MemoryBuffer& Dest) override + { + switch (Msg.GetLevel()) + { + case Warn: + fmt::format_to(fmt::appender(Dest), ZEN_COLOR_YELLOW "Warning: " ZEN_COLOR_RESET); + break; + case Err: + fmt::format_to(fmt::appender(Dest), ZEN_BRIGHT_COLOR_RED "Error: " ZEN_COLOR_RESET); + break; + case Critical: + fmt::format_to(fmt::appender(Dest), ZEN_COLOR_RED "Critical: " ZEN_COLOR_RESET); + break; + default: + break; + } + + helpers::AppendStringView(Msg.GetPayload(), Dest); + Dest.push_back('\n'); + } + + std::unique_ptr<Formatter> Clone() const override { return std::make_unique<ConsoleFormatter>(); } +}; + LoggerRef ConsoleLog() { @@ -394,10 +342,10 @@ ConsoleLog() std::call_once(ConsoleInitFlag, [&] { if (!ConLogger) { - ConLogger = spdlog::stdout_color_mt("console"); - spdlog::apply_logger_env_levels(ConLogger); - - ConLogger->set_pattern("%v"); + SinkPtr ConsoleSink(new AnsiColorStdoutSink()); + ConsoleSink->SetFormatter(std::make_unique<ConsoleFormatter>()); + ConLogger = Ref<Logger>(new Logger("console", std::vector<SinkPtr>{ConsoleSink})); + Registry::Instance().Register(ConLogger); } }); @@ -405,17 +353,29 @@ ConsoleLog() } void +ResetConsoleLog() +{ + ZEN_MEMSCOPE(ELLMTag::Logging); + + LoggerRef ConLog = ConsoleLog(); + + ConLog->SetFormatter(std::make_unique<ConsoleFormatter>()); +} + +void InitializeLogging() { ZEN_MEMSCOPE(ELLMTag::Logging); - TheDefaultLogger = *spdlog::default_logger_raw(); + TheDefaultLogger = *Registry::Instance().DefaultLoggerRaw(); } void ShutdownLogging() { - spdlog::shutdown(); + ZEN_MEMSCOPE(ELLMTag::Logging); + + Registry::Instance().Shutdown(); TheDefaultLogger = {}; } @@ -449,7 +409,7 @@ EnableVTMode() void FlushLogging() { - spdlog::details::registry::instance().flush_all(); + Registry::Instance().FlushAll(); } } // namespace zen::logging @@ -457,21 +417,27 @@ FlushLogging() namespace zen { bool -LoggerRef::ShouldLog(int Level) const +LoggerRef::ShouldLog(logging::LogLevel Level) const { - return SpdLogger->should_log(static_cast<spdlog::level::level_enum>(Level)); + return m_Logger->ShouldLog(Level); } void -LoggerRef::SetLogLevel(logging::level::LogLevel NewLogLevel) +LoggerRef::SetLogLevel(logging::LogLevel NewLogLevel) { - SpdLogger->set_level(to_spdlog_level(NewLogLevel)); + m_Logger->SetLevel(NewLogLevel); } -logging::level::LogLevel +logging::LogLevel LoggerRef::GetLogLevel() { - return logging::level::to_logging_level(SpdLogger->level()); + return m_Logger->GetLevel(); +} + +void +LoggerRef::Flush() +{ + m_Logger->Flush(); } thread_local ScopedActivityBase* t_ScopeStack = nullptr; @@ -532,6 +498,8 @@ logging_forcelink() using namespace std::literals; +TEST_SUITE_BEGIN("core.logging"); + TEST_CASE("simple.bread") { ExtendableStringBuilder<256> Crumbs; @@ -580,6 +548,8 @@ TEST_CASE("simple.bread") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/logging/ansicolorsink.cpp b/src/zencore/logging/ansicolorsink.cpp new file mode 100644 index 000000000..540d22359 --- /dev/null +++ b/src/zencore/logging/ansicolorsink.cpp @@ -0,0 +1,273 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/logging/ansicolorsink.h> +#include <zencore/logging/helpers.h> +#include <zencore/logging/messageonlyformatter.h> + +#include <cstdio> +#include <cstdlib> +#include <mutex> + +#if defined(_WIN32) +# include <io.h> +# define ZEN_ISATTY _isatty +# define ZEN_FILENO _fileno +#else +# include <unistd.h> +# define ZEN_ISATTY isatty +# define ZEN_FILENO fileno +#endif + +namespace zen::logging { + +// Default formatter replicating spdlog's %+ pattern: +// [YYYY-MM-DD HH:MM:SS.mmm] [logger_name] [level] message\n +class DefaultConsoleFormatter : public Formatter +{ +public: + void Format(const LogMessage& Msg, MemoryBuffer& Dest) override + { + // timestamp + auto Secs = std::chrono::duration_cast<std::chrono::seconds>(Msg.GetTime().time_since_epoch()); + if (Secs != m_LastLogSecs) + { + m_LastLogSecs = Secs; + m_CachedLocalTm = helpers::SafeLocaltime(LogClock::to_time_t(Msg.GetTime())); + } + + Dest.push_back('['); + helpers::AppendInt(m_CachedLocalTm.tm_year + 1900, Dest); + Dest.push_back('-'); + helpers::Pad2(m_CachedLocalTm.tm_mon + 1, Dest); + Dest.push_back('-'); + helpers::Pad2(m_CachedLocalTm.tm_mday, Dest); + Dest.push_back(' '); + helpers::Pad2(m_CachedLocalTm.tm_hour, Dest); + Dest.push_back(':'); + helpers::Pad2(m_CachedLocalTm.tm_min, Dest); + Dest.push_back(':'); + helpers::Pad2(m_CachedLocalTm.tm_sec, Dest); + Dest.push_back('.'); + auto Millis = helpers::TimeFraction<std::chrono::milliseconds>(Msg.GetTime()); + helpers::Pad3(static_cast<uint32_t>(Millis.count()), Dest); + Dest.push_back(']'); + Dest.push_back(' '); + + // logger name + if (Msg.GetLoggerName().size() > 0) + { + Dest.push_back('['); + helpers::AppendStringView(Msg.GetLoggerName(), Dest); + Dest.push_back(']'); + Dest.push_back(' '); + } + + // level (colored range) + Dest.push_back('['); + Msg.ColorRangeStart = Dest.size(); + helpers::AppendStringView(helpers::LevelToShortString(Msg.GetLevel()), Dest); + Msg.ColorRangeEnd = Dest.size(); + Dest.push_back(']'); + Dest.push_back(' '); + + // message + helpers::AppendStringView(Msg.GetPayload(), Dest); + Dest.push_back('\n'); + } + + std::unique_ptr<Formatter> Clone() const override { return std::make_unique<DefaultConsoleFormatter>(); } + +private: + std::chrono::seconds m_LastLogSecs{0}; + std::tm m_CachedLocalTm{}; +}; + +static constexpr std::string_view s_Reset = "\033[m"; + +static std::string_view +GetColorForLevel(LogLevel InLevel) +{ + using namespace std::string_view_literals; + switch (InLevel) + { + case Trace: + return "\033[37m"sv; // white + case Debug: + return "\033[36m"sv; // cyan + case Info: + return "\033[32m"sv; // green + case Warn: + return "\033[33m\033[1m"sv; // bold yellow + case Err: + return "\033[31m\033[1m"sv; // bold red + case Critical: + return "\033[1m\033[41m"sv; // bold on red background + default: + return s_Reset; + } +} + +struct AnsiColorStdoutSink::Impl +{ + explicit Impl(ColorMode Mode) : m_Formatter(std::make_unique<DefaultConsoleFormatter>()), m_UseColor(ResolveColorMode(Mode)) {} + + static bool IsColorTerminal() + { + // If stdout is not a TTY, no color + if (ZEN_ISATTY(ZEN_FILENO(stdout)) == 0) + { + return false; + } + + // NO_COLOR convention (https://no-color.org/) + if (std::getenv("NO_COLOR") != nullptr) + { + return false; + } + + // COLORTERM is set by terminals that support color (e.g. "truecolor", "24bit") + if (std::getenv("COLORTERM") != nullptr) + { + return true; + } + + // Check TERM for known color-capable values + const char* Term = std::getenv("TERM"); + if (Term != nullptr) + { + std::string_view TermView(Term); + // "dumb" terminals do not support color + if (TermView == "dumb") + { + return false; + } + // Match against known color-capable terminal types. + // TERM often includes suffixes like "-256color", so we use substring matching. + constexpr std::string_view ColorTerms[] = { + "alacritty", + "ansi", + "color", + "console", + "cygwin", + "gnome", + "konsole", + "kterm", + "linux", + "msys", + "putty", + "rxvt", + "screen", + "tmux", + "vt100", + "vt102", + "xterm", + }; + for (std::string_view Candidate : ColorTerms) + { + if (TermView.find(Candidate) != std::string_view::npos) + { + return true; + } + } + } + +#if defined(_WIN32) + // Windows console supports ANSI color by default in modern versions + return true; +#else + // Unknown terminal — be conservative + return false; +#endif + } + + static bool ResolveColorMode(ColorMode Mode) + { + switch (Mode) + { + case ColorMode::On: + return true; + case ColorMode::Off: + return false; + case ColorMode::Auto: + default: + return IsColorTerminal(); + } + } + + void Log(const LogMessage& Msg) + { + std::lock_guard<std::mutex> Lock(m_Mutex); + + MemoryBuffer Formatted; + m_Formatter->Format(Msg, Formatted); + + if (m_UseColor && Msg.ColorRangeEnd > Msg.ColorRangeStart) + { + // Print pre-color range + fwrite(Formatted.data(), 1, Msg.ColorRangeStart, m_File); + + // Print color + std::string_view Color = GetColorForLevel(Msg.GetLevel()); + fwrite(Color.data(), 1, Color.size(), m_File); + + // Print colored range + fwrite(Formatted.data() + Msg.ColorRangeStart, 1, Msg.ColorRangeEnd - Msg.ColorRangeStart, m_File); + + // Reset color + fwrite(s_Reset.data(), 1, s_Reset.size(), m_File); + + // Print remainder + fwrite(Formatted.data() + Msg.ColorRangeEnd, 1, Formatted.size() - Msg.ColorRangeEnd, m_File); + } + else + { + fwrite(Formatted.data(), 1, Formatted.size(), m_File); + } + + fflush(m_File); + } + + void Flush() + { + std::lock_guard<std::mutex> Lock(m_Mutex); + fflush(m_File); + } + + void SetFormatter(std::unique_ptr<Formatter> InFormatter) + { + std::lock_guard<std::mutex> Lock(m_Mutex); + m_Formatter = std::move(InFormatter); + } + +private: + std::mutex m_Mutex; + std::unique_ptr<Formatter> m_Formatter; + FILE* m_File = stdout; + bool m_UseColor = true; +}; + +AnsiColorStdoutSink::AnsiColorStdoutSink(ColorMode Mode) : m_Impl(std::make_unique<Impl>(Mode)) +{ +} + +AnsiColorStdoutSink::~AnsiColorStdoutSink() = default; + +void +AnsiColorStdoutSink::Log(const LogMessage& Msg) +{ + m_Impl->Log(Msg); +} + +void +AnsiColorStdoutSink::Flush() +{ + m_Impl->Flush(); +} + +void +AnsiColorStdoutSink::SetFormatter(std::unique_ptr<Formatter> InFormatter) +{ + m_Impl->SetFormatter(std::move(InFormatter)); +} + +} // namespace zen::logging diff --git a/src/zencore/logging/asyncsink.cpp b/src/zencore/logging/asyncsink.cpp new file mode 100644 index 000000000..02bf9f3ba --- /dev/null +++ b/src/zencore/logging/asyncsink.cpp @@ -0,0 +1,212 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/logging/asyncsink.h> + +#include <zencore/blockingqueue.h> +#include <zencore/logging/logmsg.h> +#include <zencore/thread.h> + +#include <future> +#include <string> +#include <thread> + +namespace zen::logging { + +struct AsyncLogMessage +{ + enum class Type : uint8_t + { + Log, + Flush, + Shutdown + }; + + Type MsgType = Type::Log; + + // Points to the LogPoint from upstream logging code. LogMessage guarantees + // this is always valid (either a static LogPoint from ZEN_LOG macros or one + // of the per-level default LogPoints). + const LogPoint* Point = nullptr; + + int ThreadId = 0; + std::string OwnedPayload; + std::string OwnedLoggerName; + std::chrono::system_clock::time_point Time; + + std::shared_ptr<std::promise<void>> FlushPromise; +}; + +struct AsyncSink::Impl +{ + explicit Impl(std::vector<SinkPtr> InSinks) : m_Sinks(std::move(InSinks)) + { + m_WorkerThread = std::thread([this]() { + zen::SetCurrentThreadName("AsyncLog"); + WorkerLoop(); + }); + } + + ~Impl() + { + AsyncLogMessage ShutdownMsg; + ShutdownMsg.MsgType = AsyncLogMessage::Type::Shutdown; + m_Queue.Enqueue(std::move(ShutdownMsg)); + + if (m_WorkerThread.joinable()) + { + m_WorkerThread.join(); + } + } + + void Log(const LogMessage& Msg) + { + AsyncLogMessage AsyncMsg; + AsyncMsg.OwnedPayload = std::string(Msg.GetPayload()); + AsyncMsg.OwnedLoggerName = std::string(Msg.GetLoggerName()); + AsyncMsg.ThreadId = Msg.GetThreadId(); + AsyncMsg.Time = Msg.GetTime(); + AsyncMsg.Point = &Msg.GetLogPoint(); + AsyncMsg.MsgType = AsyncLogMessage::Type::Log; + + m_Queue.Enqueue(std::move(AsyncMsg)); + } + + void Flush() + { + auto Promise = std::make_shared<std::promise<void>>(); + auto Future = Promise->get_future(); + + AsyncLogMessage FlushMsg; + FlushMsg.MsgType = AsyncLogMessage::Type::Flush; + FlushMsg.FlushPromise = std::move(Promise); + + m_Queue.Enqueue(std::move(FlushMsg)); + + Future.get(); + } + + void SetFormatter(std::unique_ptr<Formatter> InFormatter) + { + for (auto& CurrentSink : m_Sinks) + { + CurrentSink->SetFormatter(InFormatter->Clone()); + } + } + +private: + void ForwardLogToSinks(const AsyncLogMessage& AsyncMsg) + { + LogMessage Reconstructed(*AsyncMsg.Point, AsyncMsg.OwnedLoggerName, AsyncMsg.OwnedPayload); + Reconstructed.SetTime(AsyncMsg.Time); + Reconstructed.SetThreadId(AsyncMsg.ThreadId); + + for (auto& CurrentSink : m_Sinks) + { + if (CurrentSink->ShouldLog(Reconstructed.GetLevel())) + { + try + { + CurrentSink->Log(Reconstructed); + } + catch (const std::exception&) + { + } + } + } + } + + void FlushSinks() + { + for (auto& CurrentSink : m_Sinks) + { + try + { + CurrentSink->Flush(); + } + catch (const std::exception&) + { + } + } + } + + void WorkerLoop() + { + AsyncLogMessage Msg; + while (m_Queue.WaitAndDequeue(Msg)) + { + switch (Msg.MsgType) + { + case AsyncLogMessage::Type::Log: + { + ForwardLogToSinks(Msg); + break; + } + + case AsyncLogMessage::Type::Flush: + { + FlushSinks(); + if (Msg.FlushPromise) + { + Msg.FlushPromise->set_value(); + } + break; + } + + case AsyncLogMessage::Type::Shutdown: + { + m_Queue.CompleteAdding(); + + AsyncLogMessage Remaining; + while (m_Queue.WaitAndDequeue(Remaining)) + { + if (Remaining.MsgType == AsyncLogMessage::Type::Log) + { + ForwardLogToSinks(Remaining); + } + else if (Remaining.MsgType == AsyncLogMessage::Type::Flush) + { + FlushSinks(); + if (Remaining.FlushPromise) + { + Remaining.FlushPromise->set_value(); + } + } + } + + FlushSinks(); + return; + } + } + } + } + + std::vector<SinkPtr> m_Sinks; + BlockingQueue<AsyncLogMessage> m_Queue; + std::thread m_WorkerThread; +}; + +AsyncSink::AsyncSink(std::vector<SinkPtr> InSinks) : m_Impl(std::make_unique<Impl>(std::move(InSinks))) +{ +} + +AsyncSink::~AsyncSink() = default; + +void +AsyncSink::Log(const LogMessage& Msg) +{ + m_Impl->Log(Msg); +} + +void +AsyncSink::Flush() +{ + m_Impl->Flush(); +} + +void +AsyncSink::SetFormatter(std::unique_ptr<Formatter> InFormatter) +{ + m_Impl->SetFormatter(std::move(InFormatter)); +} + +} // namespace zen::logging diff --git a/src/zencore/logging/logger.cpp b/src/zencore/logging/logger.cpp new file mode 100644 index 000000000..dd1675bb1 --- /dev/null +++ b/src/zencore/logging/logger.cpp @@ -0,0 +1,142 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/logging/logger.h> +#include <zencore/thread.h> + +#include <string> +#include <vector> + +namespace zen::logging { + +struct Logger::Impl +{ + std::string m_Name; + std::vector<SinkPtr> m_Sinks; + ErrorHandler* m_ErrorHandler = nullptr; +}; + +Logger::Logger(std::string_view InName, SinkPtr InSink) : m_Impl(std::make_unique<Impl>()) +{ + m_Impl->m_Name = InName; + m_Impl->m_Sinks.push_back(std::move(InSink)); +} + +Logger::Logger(std::string_view InName, std::span<const SinkPtr> InSinks) : m_Impl(std::make_unique<Impl>()) +{ + m_Impl->m_Name = InName; + m_Impl->m_Sinks.assign(InSinks.begin(), InSinks.end()); +} + +Logger::~Logger() = default; + +void +Logger::Log(const LogPoint& Point, fmt::format_args Args) +{ + if (!ShouldLog(Point.Level)) + { + return; + } + + fmt::basic_memory_buffer<char, 250> Buffer; + fmt::vformat_to(fmt::appender(Buffer), Point.FormatString, Args); + + LogMessage LogMsg(Point, m_Impl->m_Name, std::string_view(Buffer.data(), Buffer.size())); + LogMsg.SetThreadId(GetCurrentThreadId()); + SinkIt(LogMsg); + FlushIfNeeded(Point.Level); +} + +void +Logger::SinkIt(const LogMessage& Msg) +{ + for (auto& CurrentSink : m_Impl->m_Sinks) + { + if (CurrentSink->ShouldLog(Msg.GetLevel())) + { + try + { + CurrentSink->Log(Msg); + } + catch (const std::exception& Ex) + { + if (m_Impl->m_ErrorHandler) + { + m_Impl->m_ErrorHandler->HandleError(Ex.what()); + } + } + } + } +} + +void +Logger::FlushIfNeeded(LogLevel InLevel) +{ + if (InLevel >= m_FlushLevel.load(std::memory_order_relaxed)) + { + Flush(); + } +} + +void +Logger::Flush() +{ + for (auto& CurrentSink : m_Impl->m_Sinks) + { + try + { + CurrentSink->Flush(); + } + catch (const std::exception& Ex) + { + if (m_Impl->m_ErrorHandler) + { + m_Impl->m_ErrorHandler->HandleError(Ex.what()); + } + } + } +} + +void +Logger::SetSinks(std::vector<SinkPtr> InSinks) +{ + m_Impl->m_Sinks = std::move(InSinks); +} + +void +Logger::AddSink(SinkPtr InSink) +{ + m_Impl->m_Sinks.push_back(std::move(InSink)); +} + +void +Logger::SetErrorHandler(ErrorHandler* Handler) +{ + m_Impl->m_ErrorHandler = Handler; +} + +void +Logger::SetFormatter(std::unique_ptr<Formatter> InFormatter) +{ + for (auto& CurrentSink : m_Impl->m_Sinks) + { + CurrentSink->SetFormatter(InFormatter->Clone()); + } +} + +std::string_view +Logger::Name() const +{ + return m_Impl->m_Name; +} + +Ref<Logger> +Logger::Clone(std::string_view NewName) const +{ + Ref<Logger> Cloned(new Logger(NewName, m_Impl->m_Sinks)); + Cloned->SetLevel(m_Level.load(std::memory_order_relaxed)); + Cloned->SetFlushLevel(m_FlushLevel.load(std::memory_order_relaxed)); + Cloned->SetErrorHandler(m_Impl->m_ErrorHandler); + return Cloned; +} + +} // namespace zen::logging diff --git a/src/zencore/logging/msvcsink.cpp b/src/zencore/logging/msvcsink.cpp new file mode 100644 index 000000000..457a4d6e1 --- /dev/null +++ b/src/zencore/logging/msvcsink.cpp @@ -0,0 +1,80 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/zencore.h> + +#if ZEN_PLATFORM_WINDOWS + +# include <zencore/logging/helpers.h> +# include <zencore/logging/messageonlyformatter.h> +# include <zencore/logging/msvcsink.h> + +ZEN_THIRD_PARTY_INCLUDES_START +# include <Windows.h> +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen::logging { + +// Default formatter for MSVC debug output: [level] message\n +// For error/critical messages with source info, prepends file(line): so that +// the message is clickable in the Visual Studio Output window. +class DefaultMsvcFormatter : public Formatter +{ +public: + void Format(const LogMessage& Msg, MemoryBuffer& Dest) override + { + const auto& Source = Msg.GetSource(); + if (Msg.GetLevel() >= LogLevel::Err && Source) + { + helpers::AppendStringView(Source.Filename, Dest); + Dest.push_back('('); + helpers::AppendInt(Source.Line, Dest); + Dest.push_back(')'); + Dest.push_back(':'); + Dest.push_back(' '); + } + + Dest.push_back('['); + helpers::AppendStringView(helpers::LevelToShortString(Msg.GetLevel()), Dest); + Dest.push_back(']'); + Dest.push_back(' '); + helpers::AppendStringView(Msg.GetPayload(), Dest); + Dest.push_back('\n'); + } + + std::unique_ptr<Formatter> Clone() const override { return std::make_unique<DefaultMsvcFormatter>(); } +}; + +MsvcSink::MsvcSink() : m_Formatter(std::make_unique<DefaultMsvcFormatter>()) +{ +} + +void +MsvcSink::Log(const LogMessage& Msg) +{ + std::lock_guard<std::mutex> Lock(m_Mutex); + + MemoryBuffer Formatted; + m_Formatter->Format(Msg, Formatted); + + // Null-terminate for OutputDebugStringA + Formatted.push_back('\0'); + + OutputDebugStringA(Formatted.data()); +} + +void +MsvcSink::Flush() +{ + // Nothing to flush for OutputDebugString +} + +void +MsvcSink::SetFormatter(std::unique_ptr<Formatter> InFormatter) +{ + std::lock_guard<std::mutex> Lock(m_Mutex); + m_Formatter = std::move(InFormatter); +} + +} // namespace zen::logging + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/src/zencore/logging/registry.cpp b/src/zencore/logging/registry.cpp new file mode 100644 index 000000000..3ed1fb0df --- /dev/null +++ b/src/zencore/logging/registry.cpp @@ -0,0 +1,330 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/logging/registry.h> + +#include <zencore/logging/ansicolorsink.h> +#include <zencore/logging/messageonlyformatter.h> + +#include <atomic> +#include <condition_variable> +#include <mutex> +#include <thread> +#include <unordered_map> + +namespace zen::logging { + +struct Registry::Impl +{ + Impl() + { + // Create default logger with a stdout color sink + SinkPtr DefaultSink(new AnsiColorStdoutSink()); + m_DefaultLogger = Ref<Logger>(new Logger("", DefaultSink)); + m_Loggers[""] = m_DefaultLogger; + } + + ~Impl() { StopPeriodicFlush(); } + + void Register(Ref<Logger> InLogger) + { + std::lock_guard<std::mutex> Lock(m_Mutex); + if (m_ErrorHandler) + { + InLogger->SetErrorHandler(m_ErrorHandler); + } + m_Loggers[std::string(InLogger->Name())] = std::move(InLogger); + } + + void Drop(const std::string& Name) + { + std::lock_guard<std::mutex> Lock(m_Mutex); + m_Loggers.erase(Name); + } + + Ref<Logger> Get(const std::string& Name) + { + std::lock_guard<std::mutex> Lock(m_Mutex); + auto It = m_Loggers.find(Name); + if (It != m_Loggers.end()) + { + return It->second; + } + return {}; + } + + void SetDefaultLogger(Ref<Logger> InLogger) + { + std::lock_guard<std::mutex> Lock(m_Mutex); + if (InLogger) + { + m_Loggers[std::string(InLogger->Name())] = InLogger; + } + m_DefaultLogger = std::move(InLogger); + } + + Logger* DefaultLoggerRaw() { return m_DefaultLogger.Get(); } + + Ref<Logger> DefaultLogger() + { + std::lock_guard<std::mutex> Lock(m_Mutex); + return m_DefaultLogger; + } + + void SetGlobalLevel(LogLevel Level) + { + m_GlobalLevel.store(Level, std::memory_order_relaxed); + std::lock_guard<std::mutex> Lock(m_Mutex); + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetLevel(Level); + } + } + + LogLevel GetGlobalLevel() const { return m_GlobalLevel.load(std::memory_order_relaxed); } + + void SetLevels(Registry::LogLevels Levels, LogLevel* DefaultLevel) + { + std::lock_guard<std::mutex> Lock(m_Mutex); + + if (DefaultLevel) + { + m_GlobalLevel.store(*DefaultLevel, std::memory_order_relaxed); + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetLevel(*DefaultLevel); + } + } + + for (auto& [LoggerName, Level] : Levels) + { + auto It = m_Loggers.find(LoggerName); + if (It != m_Loggers.end()) + { + It->second->SetLevel(Level); + } + } + } + + void FlushAll() + { + std::lock_guard<std::mutex> Lock(m_Mutex); + for (auto& [Name, CurLogger] : m_Loggers) + { + try + { + CurLogger->Flush(); + } + catch (const std::exception&) + { + } + } + } + + void FlushOn(LogLevel Level) + { + std::lock_guard<std::mutex> Lock(m_Mutex); + m_FlushLevel = Level; + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetFlushLevel(Level); + } + } + + void FlushEvery(std::chrono::seconds Interval) + { + StopPeriodicFlush(); + + m_PeriodicFlushRunning.store(true, std::memory_order_relaxed); + + m_FlushThread = std::thread([this, Interval] { + while (m_PeriodicFlushRunning.load(std::memory_order_relaxed)) + { + { + std::unique_lock<std::mutex> Lock(m_PeriodicFlushMutex); + m_PeriodicFlushCv.wait_for(Lock, Interval, [this] { return !m_PeriodicFlushRunning.load(std::memory_order_relaxed); }); + } + + if (m_PeriodicFlushRunning.load(std::memory_order_relaxed)) + { + FlushAll(); + } + } + }); + } + + void SetFormatter(std::unique_ptr<Formatter> InFormatter) + { + std::lock_guard<std::mutex> Lock(m_Mutex); + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetFormatter(InFormatter->Clone()); + } + } + + void ApplyAll(void (*Func)(void*, Ref<Logger>), void* Context) + { + std::lock_guard<std::mutex> Lock(m_Mutex); + for (auto& [Name, CurLogger] : m_Loggers) + { + Func(Context, CurLogger); + } + } + + void SetErrorHandler(ErrorHandler* Handler) + { + std::lock_guard<std::mutex> Lock(m_Mutex); + m_ErrorHandler = Handler; + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetErrorHandler(Handler); + } + } + + void Shutdown() + { + StopPeriodicFlush(); + FlushAll(); + + std::lock_guard<std::mutex> Lock(m_Mutex); + m_Loggers.clear(); + m_DefaultLogger = nullptr; + } + +private: + void StopPeriodicFlush() + { + if (m_FlushThread.joinable()) + { + m_PeriodicFlushRunning.store(false, std::memory_order_relaxed); + { + std::lock_guard<std::mutex> Lock(m_PeriodicFlushMutex); + m_PeriodicFlushCv.notify_one(); + } + m_FlushThread.join(); + } + } + + std::mutex m_Mutex; + std::unordered_map<std::string, Ref<Logger>> m_Loggers; + Ref<Logger> m_DefaultLogger; + std::atomic<LogLevel> m_GlobalLevel{Trace}; + LogLevel m_FlushLevel{Off}; + ErrorHandler* m_ErrorHandler = nullptr; + + // Periodic flush + std::atomic<bool> m_PeriodicFlushRunning{false}; + std::mutex m_PeriodicFlushMutex; + std::condition_variable m_PeriodicFlushCv; + std::thread m_FlushThread; +}; + +Registry& +Registry::Instance() +{ + static Registry s_Instance; + return s_Instance; +} + +Registry::Registry() : m_Impl(std::make_unique<Impl>()) +{ +} + +Registry::~Registry() = default; + +void +Registry::Register(Ref<Logger> InLogger) +{ + m_Impl->Register(std::move(InLogger)); +} + +void +Registry::Drop(const std::string& Name) +{ + m_Impl->Drop(Name); +} + +Ref<Logger> +Registry::Get(const std::string& Name) +{ + return m_Impl->Get(Name); +} + +void +Registry::SetDefaultLogger(Ref<Logger> InLogger) +{ + m_Impl->SetDefaultLogger(std::move(InLogger)); +} + +Logger* +Registry::DefaultLoggerRaw() +{ + return m_Impl->DefaultLoggerRaw(); +} + +Ref<Logger> +Registry::DefaultLogger() +{ + return m_Impl->DefaultLogger(); +} + +void +Registry::SetGlobalLevel(LogLevel Level) +{ + m_Impl->SetGlobalLevel(Level); +} + +LogLevel +Registry::GetGlobalLevel() const +{ + return m_Impl->GetGlobalLevel(); +} + +void +Registry::SetLevels(LogLevels Levels, LogLevel* DefaultLevel) +{ + m_Impl->SetLevels(Levels, DefaultLevel); +} + +void +Registry::FlushAll() +{ + m_Impl->FlushAll(); +} + +void +Registry::FlushOn(LogLevel Level) +{ + m_Impl->FlushOn(Level); +} + +void +Registry::FlushEvery(std::chrono::seconds Interval) +{ + m_Impl->FlushEvery(Interval); +} + +void +Registry::SetFormatter(std::unique_ptr<Formatter> InFormatter) +{ + m_Impl->SetFormatter(std::move(InFormatter)); +} + +void +Registry::ApplyAllImpl(void (*Func)(void*, Ref<Logger>), void* Context) +{ + m_Impl->ApplyAll(Func, Context); +} + +void +Registry::SetErrorHandler(ErrorHandler* Handler) +{ + m_Impl->SetErrorHandler(Handler); +} + +void +Registry::Shutdown() +{ + m_Impl->Shutdown(); +} + +} // namespace zen::logging diff --git a/src/zencore/logging/tracesink.cpp b/src/zencore/logging/tracesink.cpp new file mode 100644 index 000000000..8a6f4e40c --- /dev/null +++ b/src/zencore/logging/tracesink.cpp @@ -0,0 +1,92 @@ + +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/logbase.h> +#include <zencore/logging/tracesink.h> +#include <zencore/string.h> +#include <zencore/timer.h> +#include <zencore/trace.h> + +#if ZEN_WITH_TRACE + +namespace zen::logging { + +UE_TRACE_CHANNEL_DEFINE(LogChannel) + +UE_TRACE_EVENT_BEGIN(Logging, LogCategory, NoSync | Important) + UE_TRACE_EVENT_FIELD(const void*, CategoryPointer) + UE_TRACE_EVENT_FIELD(uint8_t, DefaultVerbosity) + UE_TRACE_EVENT_FIELD(UE::Trace::AnsiString, Name) +UE_TRACE_EVENT_END() + +UE_TRACE_EVENT_BEGIN(Logging, LogMessageSpec, NoSync | Important) + UE_TRACE_EVENT_FIELD(const void*, LogPoint) + UE_TRACE_EVENT_FIELD(const void*, CategoryPointer) + UE_TRACE_EVENT_FIELD(int32_t, Line) + UE_TRACE_EVENT_FIELD(uint8_t, Verbosity) + UE_TRACE_EVENT_FIELD(UE::Trace::AnsiString, FileName) + UE_TRACE_EVENT_FIELD(UE::Trace::AnsiString, FormatString) +UE_TRACE_EVENT_END() + +UE_TRACE_EVENT_BEGIN(Logging, LogMessage, NoSync) + UE_TRACE_EVENT_FIELD(const void*, LogPoint) + UE_TRACE_EVENT_FIELD(uint64_t, Cycle) + UE_TRACE_EVENT_FIELD(uint8_t[], FormatArgs) +UE_TRACE_EVENT_END() + +void +TraceLogCategory(const logging::Logger* Category, const char* Name, logging::LogLevel DefaultVerbosity) +{ + uint16_t NameLen = uint16_t(strlen(Name)); + UE_TRACE_LOG(Logging, LogCategory, LogChannel, NameLen * sizeof(ANSICHAR)) + << LogCategory.CategoryPointer(Category) << LogCategory.DefaultVerbosity(uint8_t(DefaultVerbosity)) + << LogCategory.Name(Name, NameLen); +} + +void +TraceLogMessageSpec(const void* LogPoint, + const logging::Logger* Category, + logging::LogLevel Verbosity, + const std::string_view File, + int32_t Line, + const std::string_view Format) +{ + uint16_t FileNameLen = uint16_t(File.size()); + uint16_t FormatStringLen = uint16_t(Format.size()); + uint32_t DataSize = (FileNameLen * sizeof(ANSICHAR)) + (FormatStringLen * sizeof(ANSICHAR)); + UE_TRACE_LOG(Logging, LogMessageSpec, LogChannel, DataSize) + << LogMessageSpec.LogPoint(LogPoint) << LogMessageSpec.CategoryPointer(Category) << LogMessageSpec.Line(Line) + << LogMessageSpec.Verbosity(uint8_t(Verbosity)) << LogMessageSpec.FileName(File.data(), FileNameLen) + << LogMessageSpec.FormatString(Format.data(), FormatStringLen); +} + +void +TraceLogMessageInternal(const void* LogPoint, int32_t EncodedFormatArgsSize, const uint8_t* EncodedFormatArgs) +{ + UE_TRACE_LOG(Logging, LogMessage, LogChannel) << LogMessage.LogPoint(LogPoint) << LogMessage.Cycle(GetHifreqTimerValue()) + << LogMessage.FormatArgs(EncodedFormatArgs, EncodedFormatArgsSize); +} + +////////////////////////////////////////////////////////////////////////// + +void +TraceSink::Log(const LogMessage& Msg) +{ + ZEN_UNUSED(Msg); +} + +void +TraceSink::Flush() +{ +} + +void +TraceSink::SetFormatter(std::unique_ptr<Formatter> /*InFormatter*/) +{ + // This sink doesn't use a formatter since it just forwards the raw format + // args to the trace system +} + +} // namespace zen::logging + +#endif diff --git a/src/zencore/md5.cpp b/src/zencore/md5.cpp index 4ec145697..f8cfee3ac 100644 --- a/src/zencore/md5.cpp +++ b/src/zencore/md5.cpp @@ -56,9 +56,9 @@ struct MD5_CTX unsigned char digest[16]; /* actual digest after MD5Final call */ }; -void MD5Init(); -void MD5Update(); -void MD5Final(); +void MD5Init(MD5_CTX* mdContext); +void MD5Update(MD5_CTX* mdContext, unsigned char* inBuf, unsigned int inLen); +void MD5Final(MD5_CTX* mdContext); /* ********************************************************************** @@ -342,6 +342,23 @@ Transform(uint32_t* buf, uint32_t* in) #undef G #undef H #undef I +#undef ROTATE_LEFT +#undef S11 +#undef S12 +#undef S13 +#undef S14 +#undef S21 +#undef S22 +#undef S23 +#undef S24 +#undef S31 +#undef S32 +#undef S33 +#undef S34 +#undef S41 +#undef S42 +#undef S43 +#undef S44 namespace zen { @@ -353,28 +370,32 @@ MD5 MD5::Zero; // Initialized to all zeroes MD5Stream::MD5Stream() { + static_assert(sizeof(MD5_CTX) <= sizeof(m_Context)); Reset(); } void MD5Stream::Reset() { + MD5Init(reinterpret_cast<MD5_CTX*>(m_Context)); } MD5Stream& MD5Stream::Append(const void* Data, size_t ByteCount) { - ZEN_UNUSED(Data); - ZEN_UNUSED(ByteCount); - + MD5Update(reinterpret_cast<MD5_CTX*>(m_Context), (unsigned char*)Data, (unsigned int)ByteCount); return *this; } MD5 MD5Stream::GetHash() { - MD5 md5{}; + MD5_CTX FinalCtx; + memcpy(&FinalCtx, m_Context, sizeof(MD5_CTX)); + MD5Final(&FinalCtx); + MD5 md5{}; + memcpy(md5.Hash, FinalCtx.digest, 16); return md5; } @@ -391,7 +412,7 @@ MD5::FromHexString(const char* string) { MD5 md5; - ParseHexBytes(string, 40, md5.Hash); + ParseHexBytes(string, 2 * sizeof md5.Hash, md5.Hash); return md5; } @@ -411,7 +432,7 @@ MD5::ToHexString(StringBuilderBase& outBuilder) const char str[41]; ToHexString(str); - outBuilder.AppendRange(str, &str[40]); + outBuilder.AppendRange(str, &str[StringLength]); return outBuilder; } @@ -437,6 +458,8 @@ md5_forcelink() // return md5text; // } +TEST_SUITE_BEGIN("core.md5"); + TEST_CASE("MD5") { using namespace std::literals; @@ -451,13 +474,15 @@ TEST_CASE("MD5") MD5::String_t Buffer; Result.ToHexString(Buffer); - CHECK(Output.compare(Buffer)); + CHECK(Output.compare(Buffer) == 0); MD5 Reresult = MD5::FromHexString(Buffer); Reresult.ToHexString(Buffer); - CHECK(Output.compare(Buffer)); + CHECK(Output.compare(Buffer) == 0); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/memoryview.cpp b/src/zencore/memoryview.cpp index 1f6a6996c..1654b1766 100644 --- a/src/zencore/memoryview.cpp +++ b/src/zencore/memoryview.cpp @@ -18,6 +18,8 @@ namespace zen { #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.memoryview"); + TEST_CASE("MemoryView") { { @@ -35,6 +37,8 @@ TEST_CASE("MemoryView") CHECK(MakeMemoryView<float>({1.0f, 1.2f}).GetSize() == 8); } +TEST_SUITE_END(); + void memory_forcelink() { diff --git a/src/zencore/memtrack/callstacktrace.cpp b/src/zencore/memtrack/callstacktrace.cpp index a5b7fede6..4a7068568 100644 --- a/src/zencore/memtrack/callstacktrace.cpp +++ b/src/zencore/memtrack/callstacktrace.cpp @@ -169,13 +169,13 @@ private: std::atomic_uint64_t Key; std::atomic_uint32_t Value; - inline uint64 GetKey() const { return Key.load(std::memory_order_relaxed); } + inline uint64 GetKey() const { return Key.load(std::memory_order_acquire); } inline uint32_t GetValue() const { return Value.load(std::memory_order_relaxed); } - inline bool IsEmpty() const { return Key.load(std::memory_order_relaxed) == 0; } + inline bool IsEmpty() const { return Key.load(std::memory_order_acquire) == 0; } inline void SetKeyValue(uint64_t InKey, uint32_t InValue) { - Value.store(InValue, std::memory_order_release); - Key.store(InKey, std::memory_order_relaxed); + Value.store(InValue, std::memory_order_relaxed); + Key.store(InKey, std::memory_order_release); } static inline uint32_t KeyHash(uint64_t Key) { return static_cast<uint32_t>(Key); } static inline void ClearEntries(FEncounteredCallstackSetEntry* Entries, int32_t EntryCount) diff --git a/src/zencore/memtrack/tagtrace.cpp b/src/zencore/memtrack/tagtrace.cpp index 70a74365d..fca4a2ec3 100644 --- a/src/zencore/memtrack/tagtrace.cpp +++ b/src/zencore/memtrack/tagtrace.cpp @@ -186,7 +186,7 @@ FTagTrace::AnnounceSpecialTags() const { auto EmitTag = [](const char16_t* DisplayString, int32_t Tag, int32_t ParentTag) { const uint32_t DisplayLen = (uint32_t)StringLength(DisplayString); - UE_TRACE_LOG(Memory, TagSpec, MemAllocChannel, DisplayLen * sizeof(ANSICHAR)) + UE_TRACE_LOG(Memory, TagSpec, MemAllocChannel, DisplayLen * sizeof(char16_t)) << TagSpec.Tag(Tag) << TagSpec.Parent(ParentTag) << TagSpec.Display(DisplayString, DisplayLen); }; diff --git a/src/zencore/mpscqueue.cpp b/src/zencore/mpscqueue.cpp index 29c76c3ca..bdd22e20c 100644 --- a/src/zencore/mpscqueue.cpp +++ b/src/zencore/mpscqueue.cpp @@ -7,7 +7,8 @@ namespace zen { -#if ZEN_WITH_TESTS && 0 +#if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.mpscqueue"); TEST_CASE("mpsc") { MpscQueue<std::string> Queue; @@ -15,6 +16,7 @@ TEST_CASE("mpsc") std::optional<std::string> Value = Queue.Dequeue(); CHECK_EQ(Value, "hello"); } +TEST_SUITE_END(); #endif void @@ -22,4 +24,4 @@ mpscqueue_forcelink() { } -} // namespace zen
\ No newline at end of file +} // namespace zen diff --git a/src/zencore/parallelwork.cpp b/src/zencore/parallelwork.cpp index d86d5815f..94696f479 100644 --- a/src/zencore/parallelwork.cpp +++ b/src/zencore/parallelwork.cpp @@ -157,6 +157,8 @@ ParallelWork::RethrowErrors() #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.parallelwork"); + TEST_CASE("parallellwork.nowork") { std::atomic<bool> AbortFlag; @@ -255,6 +257,8 @@ TEST_CASE("parallellwork.limitqueue") Work.Wait(); } +TEST_SUITE_END(); + void parallellwork_forcelink() { diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 56849a10d..f657869dc 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -9,6 +9,7 @@ #include <zencore/string.h> #include <zencore/testing.h> #include <zencore/timer.h> +#include <zencore/trace.h> #include <thread> @@ -490,6 +491,8 @@ CreateProcNormal(const std::filesystem::path& Executable, std::string_view Comma LPSECURITY_ATTRIBUTES ProcessAttributes = nullptr; LPSECURITY_ATTRIBUTES ThreadAttributes = nullptr; + const bool AssignToJob = Options.AssignToJob && Options.AssignToJob->IsValid(); + DWORD CreationFlags = 0; if (Options.Flags & CreateProcOptions::Flag_NewConsole) { @@ -503,6 +506,10 @@ CreateProcNormal(const std::filesystem::path& Executable, std::string_view Comma { CreationFlags |= CREATE_NEW_PROCESS_GROUP; } + if (AssignToJob) + { + CreationFlags |= CREATE_SUSPENDED; + } const wchar_t* WorkingDir = nullptr; if (Options.WorkingDirectory != nullptr) @@ -571,6 +578,15 @@ CreateProcNormal(const std::filesystem::path& Executable, std::string_view Comma return nullptr; } + if (AssignToJob) + { + if (!Options.AssignToJob->AssignProcess(ProcessInfo.hProcess)) + { + ZEN_WARN("Failed to assign newly created process to job object"); + } + ResumeThread(ProcessInfo.hThread); + } + CloseHandle(ProcessInfo.hThread); return ProcessInfo.hProcess; } @@ -644,6 +660,8 @@ CreateProcUnelevated(const std::filesystem::path& Executable, std::string_view C }; PROCESS_INFORMATION ProcessInfo = {}; + const bool AssignToJob = Options.AssignToJob && Options.AssignToJob->IsValid(); + if (Options.Flags & CreateProcOptions::Flag_NewConsole) { CreateProcFlags |= CREATE_NEW_CONSOLE; @@ -652,6 +670,10 @@ CreateProcUnelevated(const std::filesystem::path& Executable, std::string_view C { CreateProcFlags |= CREATE_NO_WINDOW; } + if (AssignToJob) + { + CreateProcFlags |= CREATE_SUSPENDED; + } ExtendableWideStringBuilder<256> CommandLineZ; CommandLineZ << CommandLine; @@ -679,6 +701,15 @@ CreateProcUnelevated(const std::filesystem::path& Executable, std::string_view C return nullptr; } + if (AssignToJob) + { + if (!Options.AssignToJob->AssignProcess(ProcessInfo.hProcess)) + { + ZEN_WARN("Failed to assign newly created process to job object"); + } + ResumeThread(ProcessInfo.hThread); + } + CloseHandle(ProcessInfo.hThread); return ProcessInfo.hProcess; } @@ -715,6 +746,8 @@ CreateProcElevated(const std::filesystem::path& Executable, std::string_view Com CreateProcResult CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options) { + ZEN_TRACE_CPU("CreateProc"); + #if ZEN_PLATFORM_WINDOWS if (Options.Flags & CreateProcOptions::Flag_Unelevated) { @@ -746,6 +779,17 @@ CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine ZEN_UNUSED(Result); } + if (!Options.StdoutFile.empty()) + { + int Fd = open(Options.StdoutFile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (Fd >= 0) + { + dup2(Fd, STDOUT_FILENO); + dup2(Fd, STDERR_FILENO); + close(Fd); + } + } + if (execv(Executable.c_str(), ArgV.data()) < 0) { ThrowLastError("Failed to exec() a new process image"); @@ -845,6 +889,65 @@ ProcessMonitor::IsActive() const ////////////////////////////////////////////////////////////////////////// +#if ZEN_PLATFORM_WINDOWS +JobObject::JobObject() = default; + +JobObject::~JobObject() +{ + if (m_JobHandle) + { + CloseHandle(m_JobHandle); + m_JobHandle = nullptr; + } +} + +void +JobObject::Initialize() +{ + ZEN_ASSERT(m_JobHandle == nullptr, "JobObject already initialized"); + + m_JobHandle = CreateJobObjectW(nullptr, nullptr); + if (!m_JobHandle) + { + ZEN_WARN("Failed to create job object: {}", zen::GetLastError()); + return; + } + + JOBOBJECT_EXTENDED_LIMIT_INFORMATION LimitInfo = {}; + LimitInfo.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + + if (!SetInformationJobObject(m_JobHandle, JobObjectExtendedLimitInformation, &LimitInfo, sizeof(LimitInfo))) + { + ZEN_WARN("Failed to set job object limits: {}", zen::GetLastError()); + CloseHandle(m_JobHandle); + m_JobHandle = nullptr; + } +} + +bool +JobObject::AssignProcess(void* ProcessHandle) +{ + ZEN_ASSERT(m_JobHandle != nullptr, "JobObject not initialized"); + ZEN_ASSERT(ProcessHandle != nullptr, "ProcessHandle is null"); + + if (!AssignProcessToJobObject(m_JobHandle, ProcessHandle)) + { + ZEN_WARN("Failed to assign process to job object: {}", zen::GetLastError()); + return false; + } + + return true; +} + +bool +JobObject::IsValid() const +{ + return m_JobHandle != nullptr; +} +#endif // ZEN_PLATFORM_WINDOWS + +////////////////////////////////////////////////////////////////////////// + bool IsProcessRunning(int pid, std::error_code& OutEc) { @@ -1001,6 +1104,232 @@ GetProcessExecutablePath(int Pid, std::error_code& OutEc) #endif // ZEN_PLATFORM_LINUX } +std::string +GetProcessCommandLine(int Pid, std::error_code& OutEc) +{ +#if ZEN_PLATFORM_WINDOWS + HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, static_cast<DWORD>(Pid)); + if (!hProcess) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + auto _ = MakeGuard([hProcess] { CloseHandle(hProcess); }); + + // NtQueryInformationProcess is an undocumented NT API; load it dynamically. + // Info class 60 = ProcessCommandLine, available since Windows 8.1. + using PFN_NtQIP = LONG(WINAPI*)(HANDLE, UINT, PVOID, ULONG, PULONG); + static const PFN_NtQIP s_NtQIP = + reinterpret_cast<PFN_NtQIP>(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "NtQueryInformationProcess")); + if (!s_NtQIP) + { + return {}; + } + + constexpr UINT ProcessCommandLineClass = 60; + constexpr LONG StatusInfoLengthMismatch = static_cast<LONG>(0xC0000004L); + + ULONG ReturnLength = 0; + LONG Status = s_NtQIP(hProcess, ProcessCommandLineClass, nullptr, 0, &ReturnLength); + if (Status != StatusInfoLengthMismatch || ReturnLength == 0) + { + return {}; + } + + std::vector<char> Buf(ReturnLength); + Status = s_NtQIP(hProcess, ProcessCommandLineClass, Buf.data(), ReturnLength, &ReturnLength); + if (Status < 0) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + + // Output: UNICODE_STRING header immediately followed by the UTF-16 string data. + // The UNICODE_STRING.Buffer field points into our Buf. + struct LocalUnicodeString + { + USHORT Length; + USHORT MaximumLength; + WCHAR* Buffer; + }; + if (ReturnLength < sizeof(LocalUnicodeString)) + { + return {}; + } + const auto* Us = reinterpret_cast<const LocalUnicodeString*>(Buf.data()); + if (Us->Length == 0 || Us->Buffer == nullptr) + { + return {}; + } + + // Skip argv[0]: may be a quoted path ("C:\...\exe.exe") or a bare path + const WCHAR* p = Us->Buffer; + const WCHAR* End = Us->Buffer + Us->Length / sizeof(WCHAR); + if (p < End && *p == L'"') + { + ++p; + while (p < End && *p != L'"') + { + ++p; + } + if (p < End) + { + ++p; // skip closing quote + } + } + else + { + while (p < End && *p != L' ') + { + ++p; + } + } + while (p < End && *p == L' ') + { + ++p; + } + if (p >= End) + { + return {}; + } + + int Utf8Size = WideCharToMultiByte(CP_UTF8, 0, p, static_cast<int>(End - p), nullptr, 0, nullptr, nullptr); + if (Utf8Size <= 0) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + std::string Result(Utf8Size, '\0'); + WideCharToMultiByte(CP_UTF8, 0, p, static_cast<int>(End - p), Result.data(), Utf8Size, nullptr, nullptr); + return Result; + +#elif ZEN_PLATFORM_LINUX + std::string CmdlinePath = fmt::format("/proc/{}/cmdline", Pid); + FILE* F = fopen(CmdlinePath.c_str(), "rb"); + if (!F) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + auto FGuard = MakeGuard([F] { fclose(F); }); + + // /proc/{pid}/cmdline contains null-separated argv entries; read it all + std::string Raw; + char Chunk[4096]; + size_t BytesRead; + while ((BytesRead = fread(Chunk, 1, sizeof(Chunk), F)) > 0) + { + Raw.append(Chunk, BytesRead); + } + if (Raw.empty()) + { + return {}; + } + + // Skip argv[0] (first null-terminated entry) + const char* p = Raw.data(); + const char* End = Raw.data() + Raw.size(); + while (p < End && *p != '\0') + { + ++p; + } + if (p < End) + { + ++p; // skip null terminator of argv[0] + } + + // Build result: remaining entries joined by spaces (inter-arg nulls → spaces) + std::string Result; + Result.reserve(static_cast<size_t>(End - p)); + for (const char* q = p; q < End; ++q) + { + Result += (*q == '\0') ? ' ' : *q; + } + while (!Result.empty() && Result.back() == ' ') + { + Result.pop_back(); + } + return Result; + +#elif ZEN_PLATFORM_MAC + int Mib[3] = {CTL_KERN, KERN_PROCARGS2, Pid}; + size_t BufSize = 0; + if (sysctl(Mib, 3, nullptr, &BufSize, nullptr, 0) != 0 || BufSize == 0) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + + std::vector<char> Buf(BufSize); + if (sysctl(Mib, 3, Buf.data(), &BufSize, nullptr, 0) != 0) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + + // Layout: [int argc][exec_path\0][null padding][argv[0]\0][argv[1]\0]...[envp\0]... + if (BufSize < sizeof(int)) + { + return {}; + } + int Argc = 0; + memcpy(&Argc, Buf.data(), sizeof(int)); + if (Argc <= 1) + { + return {}; + } + + const char* p = Buf.data() + sizeof(int); + const char* End = Buf.data() + BufSize; + + // Skip exec_path and any trailing null padding that follows it + while (p < End && *p != '\0') + { + ++p; + } + while (p < End && *p == '\0') + { + ++p; + } + + // Skip argv[0] + while (p < End && *p != '\0') + { + ++p; + } + if (p < End) + { + ++p; + } + + // Collect argv[1..Argc-1] + std::string Result; + for (int i = 1; i < Argc && p < End; ++i) + { + if (i > 1) + { + Result += ' '; + } + const char* ArgStart = p; + while (p < End && *p != '\0') + { + ++p; + } + Result.append(ArgStart, p); + if (p < End) + { + ++p; + } + } + return Result; + +#else + ZEN_UNUSED(Pid); + ZEN_UNUSED(OutEc); + return {}; +#endif +} + std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle, bool IncludeSelf) { diff --git a/src/zencore/refcount.cpp b/src/zencore/refcount.cpp index a6a86ee12..f19afe715 100644 --- a/src/zencore/refcount.cpp +++ b/src/zencore/refcount.cpp @@ -33,6 +33,8 @@ refcount_forcelink() { } +TEST_SUITE_BEGIN("core.refcount"); + TEST_CASE("RefPtr") { RefPtr<TestRefClass> Ref; @@ -60,6 +62,8 @@ TEST_CASE("RefPtr") CHECK(IsDestroyed == true); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index 00e67dc85..8d087e8c6 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -4,29 +4,23 @@ #include <zencore/config.h> #include <zencore/logging.h> +#include <zencore/logging/registry.h> +#include <zencore/logging/sink.h> #include <zencore/session.h> #include <zencore/uid.h> #include <stdarg.h> #include <stdio.h> -#if ZEN_PLATFORM_LINUX +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC # include <pwd.h> +# include <unistd.h> #endif -#if ZEN_PLATFORM_MAC -# include <pwd.h> -#endif - -ZEN_THIRD_PARTY_INCLUDES_START -#include <spdlog/spdlog.h> -ZEN_THIRD_PARTY_INCLUDES_END - #if ZEN_USE_SENTRY # define SENTRY_BUILD_STATIC 1 ZEN_THIRD_PARTY_INCLUDES_START # include <sentry.h> -# include <spdlog/sinks/base_sink.h> ZEN_THIRD_PARTY_INCLUDES_END namespace sentry { @@ -44,71 +38,58 @@ struct SentryAssertImpl : zen::AssertImpl const zen::CallstackFrames* Callstack) override; }; -class sentry_sink final : public spdlog::sinks::base_sink<spdlog::details::null_mutex> +static constexpr sentry_level_t MapToSentryLevel[zen::logging::LogLevelCount] = {SENTRY_LEVEL_DEBUG, + SENTRY_LEVEL_DEBUG, + SENTRY_LEVEL_INFO, + SENTRY_LEVEL_WARNING, + SENTRY_LEVEL_ERROR, + SENTRY_LEVEL_FATAL, + SENTRY_LEVEL_DEBUG}; + +class SentrySink final : public zen::logging::Sink { public: - sentry_sink(); - ~sentry_sink(); + SentrySink() = default; + ~SentrySink() = default; -protected: - void sink_it_(const spdlog::details::log_msg& msg) override; - void flush_() override; + void Log(const zen::logging::LogMessage& Msg) override + { + if (Msg.GetLevel() != zen::logging::Err && Msg.GetLevel() != zen::logging::Critical) + { + return; + } + try + { + std::string Message = fmt::format("{}\n{}({})", Msg.GetPayload(), Msg.GetSource().Filename, Msg.GetSource().Line); + sentry_value_t Event = sentry_value_new_message_event( + /* level */ MapToSentryLevel[Msg.GetLevel()], + /* logger */ nullptr, + /* message */ Message.c_str()); + sentry_event_value_add_stacktrace(Event, NULL, 0); + sentry_capture_event(Event); + } + catch (const std::exception&) + { + // If our logging with Message formatting fails we do a non-allocating version and just post the payload raw + char TmpBuffer[256]; + size_t MaxCopy = zen::Min<size_t>(Msg.GetPayload().size(), size_t(255)); + memcpy(TmpBuffer, Msg.GetPayload().data(), MaxCopy); + TmpBuffer[MaxCopy] = '\0'; + sentry_value_t Event = sentry_value_new_message_event( + /* level */ SENTRY_LEVEL_ERROR, + /* logger */ nullptr, + /* message */ TmpBuffer); + sentry_event_value_add_stacktrace(Event, NULL, 0); + sentry_capture_event(Event); + } + } + + void Flush() override {} + void SetFormatter(std::unique_ptr<zen::logging::Formatter>) override {} }; ////////////////////////////////////////////////////////////////////////// -static constexpr sentry_level_t MapToSentryLevel[spdlog::level::level_enum::n_levels] = {SENTRY_LEVEL_DEBUG, - SENTRY_LEVEL_DEBUG, - SENTRY_LEVEL_INFO, - SENTRY_LEVEL_WARNING, - SENTRY_LEVEL_ERROR, - SENTRY_LEVEL_FATAL, - SENTRY_LEVEL_DEBUG}; - -sentry_sink::sentry_sink() -{ -} -sentry_sink::~sentry_sink() -{ -} - -void -sentry_sink::sink_it_(const spdlog::details::log_msg& msg) -{ - if (msg.level != spdlog::level::err && msg.level != spdlog::level::critical) - { - return; - } - try - { - std::string Message = fmt::format("{}\n{}({}) [{}]", msg.payload, msg.source.filename, msg.source.line, msg.source.funcname); - sentry_value_t event = sentry_value_new_message_event( - /* level */ MapToSentryLevel[msg.level], - /* logger */ nullptr, - /* message */ Message.c_str()); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); - } - catch (const std::exception&) - { - // If our logging with Message formatting fails we do a non-allocating version and just post the msg.payload raw - char TmpBuffer[256]; - size_t MaxCopy = zen::Min<size_t>(msg.payload.size(), size_t(255)); - memcpy(TmpBuffer, msg.payload.data(), MaxCopy); - TmpBuffer[MaxCopy] = '\0'; - sentry_value_t event = sentry_value_new_message_event( - /* level */ SENTRY_LEVEL_ERROR, - /* logger */ nullptr, - /* message */ TmpBuffer); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); - } -} -void -sentry_sink::flush_() -{ -} - void SentryAssertImpl::OnAssert(const char* Filename, int LineNumber, @@ -145,6 +126,10 @@ SentryAssertImpl::OnAssert(const char* Filename, namespace zen { # if ZEN_USE_SENTRY +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogSentry, "sentry-sdk"); + +static std::atomic<bool> s_SentryLogEnabled{true}; + static void SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[maybe_unused]] void* Userdata) { @@ -163,26 +148,62 @@ SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[may MessagePtr = LogMessage.c_str(); } + // SentryLogFunction can be called before the logging system is initialized + // (during sentry_init which runs before InitializeLogging), or after it has + // been shut down (during sentry_close on a background worker thread). Fall + // back to console logging when the category logger is not available. + // + // Since we want to default to WARN level but this runs before logging has + // been configured, we ignore the callbacks for DEBUG/INFO explicitly here + // which means users don't see every possible log message if they're trying + // to configure the levels using --log-debug=sentry-sdk + if (!TheDefaultLogger || !s_SentryLogEnabled.load(std::memory_order_acquire)) + { + switch (Level) + { + case SENTRY_LEVEL_DEBUG: + // ZEN_CONSOLE_DEBUG("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_INFO: + // ZEN_CONSOLE_INFO("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_WARNING: + ZEN_CONSOLE_WARN("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_ERROR: + ZEN_CONSOLE_ERROR("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_FATAL: + ZEN_CONSOLE_CRITICAL("sentry: {}", MessagePtr); + break; + } + return; + } + switch (Level) { case SENTRY_LEVEL_DEBUG: - ZEN_CONSOLE_DEBUG("sentry: {}", MessagePtr); + ZEN_LOG_DEBUG(LogSentry, "sentry: {}", MessagePtr); break; case SENTRY_LEVEL_INFO: - ZEN_CONSOLE_INFO("sentry: {}", MessagePtr); + ZEN_LOG_INFO(LogSentry, "sentry: {}", MessagePtr); break; case SENTRY_LEVEL_WARNING: - ZEN_CONSOLE_WARN("sentry: {}", MessagePtr); + ZEN_LOG_WARN(LogSentry, "sentry: {}", MessagePtr); break; case SENTRY_LEVEL_ERROR: - ZEN_CONSOLE_ERROR("sentry: {}", MessagePtr); + ZEN_LOG_ERROR(LogSentry, "sentry: {}", MessagePtr); break; case SENTRY_LEVEL_FATAL: - ZEN_CONSOLE_CRITICAL("sentry: {}", MessagePtr); + ZEN_LOG_CRITICAL(LogSentry, "sentry: {}", MessagePtr); break; } } @@ -194,11 +215,21 @@ SentryIntegration::SentryIntegration() SentryIntegration::~SentryIntegration() { + Close(); +} + +void +SentryIntegration::Close() +{ if (m_IsInitialized && m_SentryErrorCode == 0) { logging::SetErrorLog(""); m_SentryAssert.reset(); + // Disable spdlog forwarding before sentry_close() since its background + // worker thread may still log during shutdown via SentryLogFunction + s_SentryLogEnabled.store(false, std::memory_order_release); sentry_close(); + m_IsInitialized = false; } } @@ -298,7 +329,9 @@ SentryIntegration::Initialize(const Config& Conf, const std::string& CommandLine sentry_set_user(SentryUserObject); - m_SentryLogger = spdlog::create<sentry::sentry_sink>("sentry"); + logging::SinkPtr SentrySink(new sentry::SentrySink()); + m_SentryLogger = Ref<logging::Logger>(new logging::Logger("sentry", std::vector<logging::SinkPtr>{SentrySink})); + logging::Registry::Instance().Register(m_SentryLogger); logging::SetErrorLog("sentry"); m_SentryAssert = std::make_unique<sentry::SentryAssertImpl>(); @@ -310,22 +343,31 @@ SentryIntegration::Initialize(const Config& Conf, const std::string& CommandLine void SentryIntegration::LogStartupInformation() { + // Initialize the sentry-sdk log category at Warn level to reduce startup noise. + // The level can be overridden via --log-debug=sentry-sdk or --log-info=sentry-sdk + LogSentry.Logger().SetLogLevel(logging::Warn); + if (m_IsInitialized) { if (m_SentryErrorCode == 0) { if (m_AllowPII) { - ZEN_INFO("sentry initialized, username: '{}', hostname: '{}', id: '{}'", m_SentryUserName, m_SentryHostName, m_SentryId); + ZEN_LOG_INFO(LogSentry, + "sentry initialized, username: '{}', hostname: '{}', id: '{}'", + m_SentryUserName, + m_SentryHostName, + m_SentryId); } else { - ZEN_INFO("sentry initialized with anonymous reports"); + ZEN_LOG_INFO(LogSentry, "sentry initialized with anonymous reports"); } } else { - ZEN_WARN( + ZEN_LOG_WARN( + LogSentry, "sentry_init returned failure! (error code: {}) note that sentry expects crashpad_handler to exist alongside the running " "executable", m_SentryErrorCode); diff --git a/src/zencore/sha1.cpp b/src/zencore/sha1.cpp index 3ee74d7d8..807ae4c30 100644 --- a/src/zencore/sha1.cpp +++ b/src/zencore/sha1.cpp @@ -373,6 +373,8 @@ sha1_forcelink() // return sha1text; // } +TEST_SUITE_BEGIN("core.sha1"); + TEST_CASE("SHA1") { uint8_t sha1_empty[20] = {0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, @@ -438,6 +440,8 @@ TEST_CASE("SHA1") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/sharedbuffer.cpp b/src/zencore/sharedbuffer.cpp index 78efb9d42..8dc6d49d8 100644 --- a/src/zencore/sharedbuffer.cpp +++ b/src/zencore/sharedbuffer.cpp @@ -152,10 +152,14 @@ sharedbuffer_forcelink() { } +TEST_SUITE_BEGIN("core.sharedbuffer"); + TEST_CASE("SharedBuffer") { } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/stream.cpp b/src/zencore/stream.cpp index a800ce121..de67303a4 100644 --- a/src/zencore/stream.cpp +++ b/src/zencore/stream.cpp @@ -79,6 +79,8 @@ BufferReader::Serialize(void* V, int64_t Length) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.stream"); + TEST_CASE("binary.writer.span") { BinaryWriter Writer; @@ -91,6 +93,8 @@ TEST_CASE("binary.writer.span") CHECK(memcmp(Result.GetData(), "apa banan", 9) == 0); } +TEST_SUITE_END(); + void stream_forcelink() { diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp index 0ee863b74..ed0ba6f46 100644 --- a/src/zencore/string.cpp +++ b/src/zencore/string.cpp @@ -4,6 +4,7 @@ #include <zencore/memoryview.h> #include <zencore/string.h> #include <zencore/testing.h> +#include <zencore/testutils.h> #include <inttypes.h> #include <math.h> @@ -24,6 +25,10 @@ utf16to8_impl(u16bit_iterator StartIt, u16bit_iterator EndIt, ::zen::StringBuild // Take care of surrogate pairs first if (utf8::internal::is_lead_surrogate(cp)) { + if (StartIt == EndIt) + { + break; + } uint32_t trail_surrogate = utf8::internal::mask16(*StartIt++); cp = (cp << 10) + trail_surrogate + utf8::internal::SURROGATE_OFFSET; } @@ -180,7 +185,21 @@ Utf8ToWide(const std::u8string_view& Str8, WideStringBuilderBase& OutString) if (!ByteCount) { +#if ZEN_SIZEOF_WCHAR_T == 2 + if (CurrentOutChar > 0xFFFF) + { + // Supplementary plane: emit a UTF-16 surrogate pair + uint32_t Adjusted = uint32_t(CurrentOutChar - 0x10000); + OutString.Append(wchar_t(0xD800 + (Adjusted >> 10))); + OutString.Append(wchar_t(0xDC00 + (Adjusted & 0x3FF))); + } + else + { + OutString.Append(wchar_t(CurrentOutChar)); + } +#else OutString.Append(wchar_t(CurrentOutChar)); +#endif CurrentOutChar = 0; } } @@ -249,6 +268,17 @@ namespace { /* kNicenumTime */ 1000}; } // namespace +uint64_t +IntPow(uint64_t Base, int Exp) +{ + uint64_t Result = 1; + for (int I = 0; I < Exp; ++I) + { + Result *= Base; + } + return Result; +} + /* * Convert a number to an appropriately human-readable output. */ @@ -296,7 +326,7 @@ NiceNumGeneral(uint64_t Num, std::span<char> Buffer, NicenumFormat Format) const char* u = UnitStrings[Format][Index]; - if ((Index == 0) || ((Num % (uint64_t)powl((int)KiloUnit[Format], Index)) == 0)) + if ((Index == 0) || ((Num % IntPow(KiloUnit[Format], Index)) == 0)) { /* * If this is an even multiple of the base, always display @@ -320,7 +350,7 @@ NiceNumGeneral(uint64_t Num, std::span<char> Buffer, NicenumFormat Format) for (int i = 2; i >= 0; i--) { - double Value = (double)Num / (uint64_t)powl((int)KiloUnit[Format], Index); + double Value = (double)Num / IntPow(KiloUnit[Format], Index); /* * Don't print floating point values for time. Note, @@ -520,13 +550,38 @@ UrlDecode(std::string_view InUrl) return std::string(Url.ToView()); } -////////////////////////////////////////////////////////////////////////// -// -// Unit tests -// +std::string +HideSensitiveString(std::string_view String) +{ + const size_t Length = String.length(); + const size_t SourceLength = Length > 16 ? 4 : 0; + const size_t PadLength = Min(Length - SourceLength, 4u); + const bool AddEllipsis = (SourceLength + PadLength) < Length; + StringBuilder<16> SB; + if (SourceLength > 0) + { + SB << String.substr(0, SourceLength); + } + if (PadLength > 0) + { + SB << std::string(PadLength, 'X'); + } + if (AddEllipsis) + { + SB << "..."; + } + return SB.ToString(); +}; + + ////////////////////////////////////////////////////////////////////////// + // + // Unit tests + // #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.string"); + TEST_CASE("url") { using namespace std::literals; @@ -793,11 +848,6 @@ TEST_CASE("niceNum") } } -void -string_forcelink() -{ -} - TEST_CASE("StringBuilder") { StringBuilder<64> sb; @@ -963,33 +1013,131 @@ TEST_CASE("ExtendableWideStringBuilder") TEST_CASE("utf8") { + using namespace utf8test; + SUBCASE("utf8towide") { - // TODO: add more extensive testing here - this covers a very small space - WideStringBuilder<32> wout; Utf8ToWide(u8"abcdefghi", wout); CHECK(StringEquals(L"abcdefghi", wout.c_str())); wout.Reset(); + Utf8ToWide(u8"abc\xC3\xA4\xC3\xB6\xC3\xBC", wout); + CHECK(StringEquals(L"abc\u00E4\u00F6\u00FC", wout.c_str())); + + wout.Reset(); + Utf8ToWide(std::string_view(kLatin), wout); + CHECK(StringEquals(kLatinW, wout.c_str())); + + wout.Reset(); + Utf8ToWide(std::string_view(kCyrillic), wout); + CHECK(StringEquals(kCyrillicW, wout.c_str())); + + wout.Reset(); + Utf8ToWide(std::string_view(kCJK), wout); + CHECK(StringEquals(kCJKW, wout.c_str())); + + wout.Reset(); + Utf8ToWide(std::string_view(kMixed), wout); + CHECK(StringEquals(kMixedW, wout.c_str())); - Utf8ToWide(u8"abc���", wout); - CHECK(StringEquals(L"abc���", wout.c_str())); + wout.Reset(); + Utf8ToWide(std::string_view(kEmoji), wout); + CHECK(StringEquals(kEmojiW, wout.c_str())); } SUBCASE("widetoutf8") { - // TODO: add more extensive testing here - this covers a very small space - - StringBuilder<32> out; + StringBuilder<64> out; WideToUtf8(L"abcdefghi", out); CHECK(StringEquals("abcdefghi", out.c_str())); out.Reset(); + WideToUtf8(kLatinW, out); + CHECK(StringEquals(kLatin, out.c_str())); + + out.Reset(); + WideToUtf8(kCyrillicW, out); + CHECK(StringEquals(kCyrillic, out.c_str())); + + out.Reset(); + WideToUtf8(kCJKW, out); + CHECK(StringEquals(kCJK, out.c_str())); + + out.Reset(); + WideToUtf8(kMixedW, out); + CHECK(StringEquals(kMixed, out.c_str())); - WideToUtf8(L"abc���", out); - CHECK(StringEquals(u8"abc���", out.c_str())); + out.Reset(); + WideToUtf8(kEmojiW, out); + CHECK(StringEquals(kEmoji, out.c_str())); + } + + SUBCASE("roundtrip") + { + // UTF-8 -> Wide -> UTF-8 identity + const char* Utf8Strings[] = {kLatin, kCyrillic, kCJK, kMixed, kEmoji}; + for (const char* Utf8Str : Utf8Strings) + { + ExtendableWideStringBuilder<64> Wide; + Utf8ToWide(std::string_view(Utf8Str), Wide); + + ExtendableStringBuilder<64> Back; + WideToUtf8(std::wstring_view(Wide.c_str()), Back); + CHECK(StringEquals(Utf8Str, Back.c_str())); + } + + // Wide -> UTF-8 -> Wide identity + const wchar_t* WideStrings[] = {kLatinW, kCyrillicW, kCJKW, kMixedW, kEmojiW}; + for (const wchar_t* WideStr : WideStrings) + { + ExtendableStringBuilder<64> Utf8; + WideToUtf8(std::wstring_view(WideStr), Utf8); + + ExtendableWideStringBuilder<64> Back; + Utf8ToWide(std::string_view(Utf8.c_str()), Back); + CHECK(StringEquals(WideStr, Back.c_str())); + } + + // Empty string round-trip + { + ExtendableWideStringBuilder<8> Wide; + Utf8ToWide(std::string_view(""), Wide); + CHECK(Wide.Size() == 0); + + ExtendableStringBuilder<8> Narrow; + WideToUtf8(std::wstring_view(L""), Narrow); + CHECK(Narrow.Size() == 0); + } + } + + SUBCASE("IsValidUtf8") + { + // Valid inputs + CHECK(IsValidUtf8("")); + CHECK(IsValidUtf8("hello world")); + CHECK(IsValidUtf8(kLatin)); + CHECK(IsValidUtf8(kCyrillic)); + CHECK(IsValidUtf8(kCJK)); + CHECK(IsValidUtf8(kMixed)); + CHECK(IsValidUtf8(kEmoji)); + + // Invalid: truncated 2-byte sequence + CHECK(!IsValidUtf8(std::string_view("\xC3", 1))); + + // Invalid: truncated 3-byte sequence + CHECK(!IsValidUtf8(std::string_view("\xE6\x97", 2))); + + // Invalid: truncated 4-byte sequence + CHECK(!IsValidUtf8(std::string_view("\xF0\x9F\x93", 3))); + + // Invalid: bad start byte + CHECK(!IsValidUtf8(std::string_view("\xFF", 1))); + CHECK(!IsValidUtf8(std::string_view("\xFE", 1))); + + // Invalid: overlong encoding of '/' (U+002F) + CHECK(!IsValidUtf8(std::string_view("\xC0\xAF", 2))); } } @@ -1105,6 +1253,28 @@ TEST_CASE("string") } } +TEST_CASE("hidesensitivestring") +{ + using namespace std::literals; + + CHECK_EQ(HideSensitiveString(""sv), ""sv); + CHECK_EQ(HideSensitiveString("A"sv), "X"sv); + CHECK_EQ(HideSensitiveString("ABCD"sv), "XXXX"sv); + CHECK_EQ(HideSensitiveString("ABCDE"sv), "XXXX..."sv); + CHECK_EQ(HideSensitiveString("ABCDEFGH"sv), "XXXX..."sv); + CHECK_EQ(HideSensitiveString("ABCDEFGHIJKLMNOP"sv), "XXXX..."sv); + CHECK_EQ(HideSensitiveString("ABCDEFGHIJKLMNOPQ"sv), "ABCDXXXX..."sv); + CHECK_EQ(HideSensitiveString("ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"sv), "ABCDXXXX..."sv); + CHECK_EQ(HideSensitiveString("1234567890123456789"sv), "1234XXXX..."sv); +} + +TEST_SUITE_END(); + +void +string_forcelink() +{ +} + #endif } // namespace zen diff --git a/src/zencore/system.cpp b/src/zencore/system.cpp index b9ac3bdee..141450b84 100644 --- a/src/zencore/system.cpp +++ b/src/zencore/system.cpp @@ -4,15 +4,20 @@ #include <zencore/compactbinarybuilder.h> #include <zencore/except.h> +#include <zencore/fmtutils.h> #include <zencore/memory/memory.h> #include <zencore/string.h> +#include <mutex> + #if ZEN_PLATFORM_WINDOWS # include <zencore/windows.h> ZEN_THIRD_PARTY_INCLUDES_START # include <iphlpapi.h> # include <winsock2.h> +# include <pdh.h> +# pragma comment(lib, "pdh.lib") ZEN_THIRD_PARTY_INCLUDES_END #elif ZEN_PLATFORM_LINUX # include <sys/utsname.h> @@ -65,55 +70,73 @@ GetSystemMetrics() // Determine physical core count - DWORD BufferSize = 0; - BOOL Result = GetLogicalProcessorInformation(nullptr, &BufferSize); - if (int32_t Error = GetLastError(); Error != ERROR_INSUFFICIENT_BUFFER) { - ThrowSystemError(Error, "Failed to get buffer size for logical processor information"); - } + DWORD BufferSize = 0; + BOOL Result = GetLogicalProcessorInformationEx(RelationAll, nullptr, &BufferSize); + if (int32_t Error = GetLastError(); Error != ERROR_INSUFFICIENT_BUFFER) + { + ThrowSystemError(Error, "Failed to get buffer size for logical processor information"); + } - PSYSTEM_LOGICAL_PROCESSOR_INFORMATION Buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)Memory::Alloc(BufferSize); + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX Buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)Memory::Alloc(BufferSize); - Result = GetLogicalProcessorInformation(Buffer, &BufferSize); - if (!Result) - { - Memory::Free(Buffer); - throw std::runtime_error("Failed to get logical processor information"); - } - - DWORD ProcessorPkgCount = 0; - DWORD ProcessorCoreCount = 0; - DWORD ByteOffset = 0; - while (ByteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= BufferSize) - { - const SYSTEM_LOGICAL_PROCESSOR_INFORMATION& Slpi = Buffer[ByteOffset / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION)]; - if (Slpi.Relationship == RelationProcessorCore) + Result = GetLogicalProcessorInformationEx(RelationAll, Buffer, &BufferSize); + if (!Result) { - ProcessorCoreCount++; + Memory::Free(Buffer); + throw std::runtime_error("Failed to get logical processor information"); } - else if (Slpi.Relationship == RelationProcessorPackage) + + DWORD ProcessorPkgCount = 0; + DWORD ProcessorCoreCount = 0; + DWORD LogicalProcessorCount = 0; + + BYTE* Ptr = reinterpret_cast<BYTE*>(Buffer); + BYTE* const End = Ptr + BufferSize; + while (Ptr < End) { - ProcessorPkgCount++; + const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX& Slpi = *reinterpret_cast<const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX*>(Ptr); + if (Slpi.Relationship == RelationProcessorCore) + { + ++ProcessorCoreCount; + + // Count logical processors (threads) across all processor groups for this core. + // Each core entry lists one GROUP_AFFINITY per group it spans; each set bit + // in the Mask represents one logical processor (HyperThreading sibling). + for (WORD g = 0; g < Slpi.Processor.GroupCount; ++g) + { + LogicalProcessorCount += static_cast<DWORD>(__popcnt64(Slpi.Processor.GroupMask[g].Mask)); + } + } + else if (Slpi.Relationship == RelationProcessorPackage) + { + ++ProcessorPkgCount; + } + Ptr += Slpi.Size; } - ByteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); - } - Metrics.CoreCount = ProcessorCoreCount; - Metrics.CpuCount = ProcessorPkgCount; + Metrics.CoreCount = ProcessorCoreCount; + Metrics.CpuCount = ProcessorPkgCount; + Metrics.LogicalProcessorCount = LogicalProcessorCount; - Memory::Free(Buffer); + Memory::Free(Buffer); + } // Query memory status - MEMORYSTATUSEX MemStatus{.dwLength = sizeof(MEMORYSTATUSEX)}; - GlobalMemoryStatusEx(&MemStatus); + { + MEMORYSTATUSEX MemStatus{.dwLength = sizeof(MEMORYSTATUSEX)}; + GlobalMemoryStatusEx(&MemStatus); + + Metrics.SystemMemoryMiB = MemStatus.ullTotalPhys / 1024 / 1024; + Metrics.AvailSystemMemoryMiB = MemStatus.ullAvailPhys / 1024 / 1024; + Metrics.VirtualMemoryMiB = MemStatus.ullTotalVirtual / 1024 / 1024; + Metrics.AvailVirtualMemoryMiB = MemStatus.ullAvailVirtual / 1024 / 1024; + Metrics.PageFileMiB = MemStatus.ullTotalPageFile / 1024 / 1024; + Metrics.AvailPageFileMiB = MemStatus.ullAvailPageFile / 1024 / 1024; + } - Metrics.SystemMemoryMiB = MemStatus.ullTotalPhys / 1024 / 1024; - Metrics.AvailSystemMemoryMiB = MemStatus.ullAvailPhys / 1024 / 1024; - Metrics.VirtualMemoryMiB = MemStatus.ullTotalVirtual / 1024 / 1024; - Metrics.AvailVirtualMemoryMiB = MemStatus.ullAvailVirtual / 1024 / 1024; - Metrics.PageFileMiB = MemStatus.ullTotalPageFile / 1024 / 1024; - Metrics.AvailPageFileMiB = MemStatus.ullAvailPageFile / 1024 / 1024; + Metrics.UptimeSeconds = GetTickCount64() / 1000; return Metrics; } @@ -206,6 +229,17 @@ GetSystemMetrics() Metrics.VirtualMemoryMiB = Metrics.SystemMemoryMiB; Metrics.AvailVirtualMemoryMiB = Metrics.AvailSystemMemoryMiB; + // System uptime + if (FILE* UptimeFile = fopen("/proc/uptime", "r")) + { + double UptimeSec = 0; + if (fscanf(UptimeFile, "%lf", &UptimeSec) == 1) + { + Metrics.UptimeSeconds = static_cast<uint64_t>(UptimeSec); + } + fclose(UptimeFile); + } + // Parse /proc/meminfo for swap/page file information Metrics.PageFileMiB = 0; Metrics.AvailPageFileMiB = 0; @@ -298,12 +332,35 @@ GetSystemMetrics() Metrics.PageFileMiB = SwapUsage.xsu_total / 1024 / 1024; Metrics.AvailPageFileMiB = (SwapUsage.xsu_total - SwapUsage.xsu_used) / 1024 / 1024; + // System uptime via boot time + { + struct timeval BootTime + { + }; + Size = sizeof(BootTime); + if (sysctlbyname("kern.boottime", &BootTime, &Size, nullptr, 0) == 0) + { + Metrics.UptimeSeconds = static_cast<uint64_t>(time(nullptr) - BootTime.tv_sec); + } + } + return Metrics; } #else # error "Unknown platform" #endif +ExtendedSystemMetrics +ApplyReportingOverrides(ExtendedSystemMetrics Metrics) +{ + if (g_FakeCpuCount) + { + Metrics.CoreCount = g_FakeCpuCount; + Metrics.LogicalProcessorCount = g_FakeCpuCount; + } + return Metrics; +} + SystemMetrics GetSystemMetricsForReporting() { @@ -318,12 +375,281 @@ GetSystemMetricsForReporting() return Sm; } +/////////////////////////////////////////////////////////////////////////// +// SystemMetricsTracker +/////////////////////////////////////////////////////////////////////////// + +// Per-platform CPU sampling helper. Called with m_Mutex held. + +#if ZEN_PLATFORM_WINDOWS || ZEN_PLATFORM_LINUX + +// Samples CPU usage by reading /proc/stat. Used natively on Linux and as a +// Wine fallback on Windows (where /proc/stat is accessible via the Z: drive). +struct ProcStatCpuSampler +{ + const char* Path = "/proc/stat"; + unsigned long PrevUser = 0; + unsigned long PrevNice = 0; + unsigned long PrevSystem = 0; + unsigned long PrevIdle = 0; + unsigned long PrevIoWait = 0; + unsigned long PrevIrq = 0; + unsigned long PrevSoftIrq = 0; + + explicit ProcStatCpuSampler(const char* InPath = "/proc/stat") : Path(InPath) {} + + float Sample() + { + float CpuUsage = 0.0f; + + if (FILE* Stat = fopen(Path, "r")) + { + char Line[256]; + unsigned long User, Nice, System, Idle, IoWait, Irq, SoftIrq; + + if (fgets(Line, sizeof(Line), Stat)) + { + if (sscanf(Line, "cpu %lu %lu %lu %lu %lu %lu %lu", &User, &Nice, &System, &Idle, &IoWait, &Irq, &SoftIrq) == 7) + { + unsigned long TotalDelta = (User + Nice + System + Idle + IoWait + Irq + SoftIrq) - + (PrevUser + PrevNice + PrevSystem + PrevIdle + PrevIoWait + PrevIrq + PrevSoftIrq); + unsigned long IdleDelta = Idle - PrevIdle; + + if (TotalDelta > 0) + { + CpuUsage = 100.0f * (TotalDelta - IdleDelta) / TotalDelta; + } + + PrevUser = User; + PrevNice = Nice; + PrevSystem = System; + PrevIdle = Idle; + PrevIoWait = IoWait; + PrevIrq = Irq; + PrevSoftIrq = SoftIrq; + } + } + fclose(Stat); + } + + return CpuUsage; + } +}; + +#endif + +#if ZEN_PLATFORM_WINDOWS + +struct CpuSampler +{ + PDH_HQUERY QueryHandle = nullptr; + PDH_HCOUNTER CounterHandle = nullptr; + bool HasPreviousSample = false; + bool IsWine = false; + ProcStatCpuSampler ProcStat{"Z:\\proc\\stat"}; + + CpuSampler() + { + IsWine = zen::windows::IsRunningOnWine(); + + if (!IsWine) + { + if (PdhOpenQueryW(nullptr, 0, &QueryHandle) == ERROR_SUCCESS) + { + if (PdhAddEnglishCounterW(QueryHandle, L"\\Processor(_Total)\\% Processor Time", 0, &CounterHandle) != ERROR_SUCCESS) + { + CounterHandle = nullptr; + } + } + } + } + + ~CpuSampler() + { + if (QueryHandle) + { + PdhCloseQuery(QueryHandle); + } + } + + float Sample() + { + if (IsWine) + { + return ProcStat.Sample(); + } + + if (!QueryHandle || !CounterHandle) + { + return 0.0f; + } + + PdhCollectQueryData(QueryHandle); + + if (!HasPreviousSample) + { + HasPreviousSample = true; + return 0.0f; + } + + PDH_FMT_COUNTERVALUE CounterValue; + if (PdhGetFormattedCounterValue(CounterHandle, PDH_FMT_DOUBLE, nullptr, &CounterValue) == ERROR_SUCCESS) + { + return static_cast<float>(CounterValue.doubleValue); + } + + return 0.0f; + } +}; + +#elif ZEN_PLATFORM_LINUX + +struct CpuSampler +{ + ProcStatCpuSampler ProcStat; + + float Sample() { return ProcStat.Sample(); } +}; + +#elif ZEN_PLATFORM_MAC + +struct CpuSampler +{ + unsigned long PrevTotalTicks = 0; + unsigned long PrevIdleTicks = 0; + + float Sample() + { + float CpuUsage = 0.0f; + + host_cpu_load_info_data_t CpuLoad; + mach_msg_type_number_t Count = sizeof(CpuLoad) / sizeof(natural_t); + if (host_statistics(mach_host_self(), HOST_CPU_LOAD_INFO, (host_info_t)&CpuLoad, &Count) == KERN_SUCCESS) + { + unsigned long TotalTicks = 0; + for (int i = 0; i < CPU_STATE_MAX; ++i) + { + TotalTicks += CpuLoad.cpu_ticks[i]; + } + unsigned long IdleTicks = CpuLoad.cpu_ticks[CPU_STATE_IDLE]; + + unsigned long TotalDelta = TotalTicks - PrevTotalTicks; + unsigned long IdleDelta = IdleTicks - PrevIdleTicks; + + if (TotalDelta > 0 && PrevTotalTicks > 0) + { + CpuUsage = 100.0f * (TotalDelta - IdleDelta) / TotalDelta; + } + + PrevTotalTicks = TotalTicks; + PrevIdleTicks = IdleTicks; + } + + return CpuUsage; + } +}; + +#endif + +struct SystemMetricsTracker::Impl +{ + using Clock = std::chrono::steady_clock; + + std::mutex Mutex; + CpuSampler Sampler; + float CachedCpuPercent = 0.0f; + Clock::time_point NextSampleTime = Clock::now(); + std::chrono::milliseconds MinInterval; + + explicit Impl(std::chrono::milliseconds InMinInterval) : MinInterval(InMinInterval) {} + + float SampleCpu() + { + const auto Now = Clock::now(); + if (Now >= NextSampleTime) + { + CachedCpuPercent = Sampler.Sample(); + NextSampleTime = Now + MinInterval; + } + return CachedCpuPercent; + } +}; + +SystemMetricsTracker::SystemMetricsTracker(std::chrono::milliseconds MinInterval) : m_Impl(std::make_unique<Impl>(MinInterval)) +{ +} + +SystemMetricsTracker::~SystemMetricsTracker() = default; + +ExtendedSystemMetrics +SystemMetricsTracker::Query() +{ + ExtendedSystemMetrics Metrics; + static_cast<SystemMetrics&>(Metrics) = GetSystemMetrics(); + + std::lock_guard Lock(m_Impl->Mutex); + Metrics.CpuUsagePercent = m_Impl->SampleCpu(); + return Metrics; +} + +/////////////////////////////////////////////////////////////////////////// + std::string_view GetOperatingSystemName() { return ZEN_PLATFORM_NAME; } +std::string +GetOperatingSystemVersion() +{ +#if ZEN_PLATFORM_WINDOWS + // Use RtlGetVersion to avoid the compatibility shim that GetVersionEx applies + using RtlGetVersionFn = LONG(WINAPI*)(PRTL_OSVERSIONINFOW); + RTL_OSVERSIONINFOW OsVer{.dwOSVersionInfoSize = sizeof(OsVer)}; + if (auto Fn = (RtlGetVersionFn)GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "RtlGetVersion")) + { + Fn(&OsVer); + } + return fmt::format("Windows {}.{} Build {}", OsVer.dwMajorVersion, OsVer.dwMinorVersion, OsVer.dwBuildNumber); +#elif ZEN_PLATFORM_LINUX + struct utsname Info + { + }; + if (uname(&Info) == 0) + { + return fmt::format("{} {}", Info.sysname, Info.release); + } + return "Linux"; +#elif ZEN_PLATFORM_MAC + char OsVersion[64] = ""; + size_t Size = sizeof(OsVersion); + if (sysctlbyname("kern.osproductversion", OsVersion, &Size, nullptr, 0) == 0) + { + return fmt::format("macOS {}", OsVersion); + } + return "macOS"; +#endif +} + +std::string_view +GetRuntimePlatformName() +{ +#if ZEN_PLATFORM_WINDOWS + if (zen::windows::IsRunningOnWine()) + { + return "wine"sv; + } + return "windows"sv; +#elif ZEN_PLATFORM_LINUX + return "linux"sv; +#elif ZEN_PLATFORM_MAC + return "macos"sv; +#else + return "unknown"sv; +#endif +} + std::string_view GetCpuName() { @@ -340,7 +666,14 @@ Describe(const SystemMetrics& Metrics, CbWriter& Writer) Writer << "cpu_count" << Metrics.CpuCount << "core_count" << Metrics.CoreCount << "lp_count" << Metrics.LogicalProcessorCount << "total_memory_mb" << Metrics.SystemMemoryMiB << "avail_memory_mb" << Metrics.AvailSystemMemoryMiB << "total_virtual_mb" << Metrics.VirtualMemoryMiB << "avail_virtual_mb" << Metrics.AvailVirtualMemoryMiB << "total_pagefile_mb" << Metrics.PageFileMiB - << "avail_pagefile_mb" << Metrics.AvailPageFileMiB; + << "avail_pagefile_mb" << Metrics.AvailPageFileMiB << "uptime_seconds" << Metrics.UptimeSeconds; +} + +void +Describe(const ExtendedSystemMetrics& Metrics, CbWriter& Writer) +{ + Describe(static_cast<const SystemMetrics&>(Metrics), Writer); + Writer << "cpu_usage_percent" << Metrics.CpuUsagePercent; } } // namespace zen diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp index 936424e0f..089e376bb 100644 --- a/src/zencore/testing.cpp +++ b/src/zencore/testing.cpp @@ -1,11 +1,22 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#define ZEN_TEST_WITH_RUNNER 1 + #include "zencore/testing.h" + +#include "zencore/filesystem.h" #include "zencore/logging.h" +#include "zencore/process.h" +#include "zencore/trace.h" #if ZEN_WITH_TESTS -# include <doctest/doctest.h> +# include <chrono> +# include <clocale> +# include <cstdlib> +# include <cstdio> +# include <string> +# include <vector> namespace zen::testing { @@ -21,9 +32,35 @@ struct TestListener : public doctest::IReporter void report_query(const doctest::QueryData& /*in*/) override {} - void test_run_start() override {} + void test_run_start() override { RunStart = std::chrono::steady_clock::now(); } - void test_run_end(const doctest::TestRunStats& /*in*/) override {} + void test_run_end(const doctest::TestRunStats& in) override + { + auto elapsed = std::chrono::steady_clock::now() - RunStart; + double elapsedSeconds = std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count() / 1000.0; + + // Write machine-readable summary to file if requested (used by xmake test summary table) + const char* summaryFile = std::getenv("ZEN_TEST_SUMMARY_FILE"); + if (summaryFile && summaryFile[0] != '\0') + { + if (FILE* f = std::fopen(summaryFile, "w")) + { + std::fprintf(f, + "cases_total=%u\ncases_passed=%u\nassertions_total=%d\nassertions_passed=%d\n" + "elapsed_seconds=%.3f\n", + in.numTestCasesPassingFilters, + in.numTestCasesPassingFilters - in.numTestCasesFailed, + in.numAsserts, + in.numAsserts - in.numAssertsFailed, + elapsedSeconds); + for (const auto& failure : FailedTests) + { + std::fprintf(f, "failed=%s|%s|%u\n", failure.Name.c_str(), failure.File.c_str(), failure.Line); + } + std::fclose(f); + } + } + } void test_case_start(const doctest::TestCaseData& in) override { @@ -37,7 +74,14 @@ struct TestListener : public doctest::IReporter ZEN_CONSOLE("{}-------------------------------------------------------------------------------{}", ColorYellow, ColorNone); } - void test_case_end(const doctest::CurrentTestCaseStats& /*in*/) override { Current = nullptr; } + void test_case_end(const doctest::CurrentTestCaseStats& in) override + { + if (!in.testCaseSuccess && Current) + { + FailedTests.push_back({Current->m_name, Current->m_file.c_str(), Current->m_line}); + } + Current = nullptr; + } void test_case_exception(const doctest::TestCaseException& /*in*/) override {} @@ -57,7 +101,16 @@ struct TestListener : public doctest::IReporter void test_case_skipped(const doctest::TestCaseData& /*in*/) override {} - const doctest::TestCaseData* Current = nullptr; + const doctest::TestCaseData* Current = nullptr; + std::chrono::steady_clock::time_point RunStart = {}; + + struct FailedTestInfo + { + std::string Name; + std::string File; + unsigned Line; + }; + std::vector<FailedTestInfo> FailedTests; }; struct TestRunner::Impl @@ -75,20 +128,26 @@ TestRunner::~TestRunner() { } +void +TestRunner::SetDefaultSuiteFilter(const char* Pattern) +{ + m_Impl->Session.setOption("test-suite", Pattern); +} + int -TestRunner::ApplyCommandLine(int argc, char const* const* argv) +TestRunner::ApplyCommandLine(int Argc, char const* const* Argv) { - m_Impl->Session.applyCommandLine(argc, argv); + m_Impl->Session.applyCommandLine(Argc, Argv); - for (int i = 1; i < argc; ++i) + for (int i = 1; i < Argc; ++i) { - if (argv[i] == "--debug"sv) + if (Argv[i] == "--debug"sv) { - zen::logging::SetLogLevel(zen::logging::level::Debug); + zen::logging::SetLogLevel(zen::logging::Debug); } - else if (argv[i] == "--verbose"sv) + else if (Argv[i] == "--verbose"sv) { - zen::logging::SetLogLevel(zen::logging::level::Trace); + zen::logging::SetLogLevel(zen::logging::Trace); } } @@ -101,6 +160,57 @@ TestRunner::Run() return m_Impl->Session.run(); } +int +RunTestMain(int Argc, char* Argv[], const char* ExecutableName, void (*ForceLink)()) +{ +# if ZEN_PLATFORM_WINDOWS + setlocale(LC_ALL, "en_us.UTF8"); +# endif + + ForceLink(); + +# if ZEN_PLATFORM_LINUX + zen::IgnoreChildSignals(); +# endif + +# if ZEN_WITH_TRACE + zen::TraceInit(ExecutableName); + zen::TraceOptions TraceCommandlineOptions; + if (GetTraceOptionsFromCommandline(TraceCommandlineOptions)) + { + TraceConfigure(TraceCommandlineOptions); + } +# endif + + zen::logging::InitializeLogging(); + zen::MaximizeOpenFileCount(); + + TestRunner Runner; + + // Derive default suite filter from ExecutableName: "zencore-test" -> "core.*" + if (ExecutableName) + { + std::string_view Name = ExecutableName; + if (Name.starts_with("zen")) + { + Name.remove_prefix(3); + } + if (Name.ends_with("-test")) + { + Name.remove_suffix(5); + } + if (!Name.empty()) + { + std::string Filter(Name); + Filter += ".*"; + Runner.SetDefaultSuiteFilter(Filter.c_str()); + } + } + + Runner.ApplyCommandLine(Argc, Argv); + return Runner.Run(); +} + } // namespace zen::testing #endif // ZEN_WITH_TESTS diff --git a/src/zencore/testutils.cpp b/src/zencore/testutils.cpp index 5bc2841ae..0cd3f8121 100644 --- a/src/zencore/testutils.cpp +++ b/src/zencore/testutils.cpp @@ -46,7 +46,7 @@ ScopedTemporaryDirectory::~ScopedTemporaryDirectory() IoBuffer CreateRandomBlob(uint64_t Size) { - static FastRandom Rand{.Seed = 0x7CEBF54E45B9F5D1}; + thread_local FastRandom Rand{.Seed = 0x7CEBF54E45B9F5D1}; return CreateRandomBlob(Rand, Size); }; diff --git a/src/zencore/thread.cpp b/src/zencore/thread.cpp index 9e3486e49..54459cbaa 100644 --- a/src/zencore/thread.cpp +++ b/src/zencore/thread.cpp @@ -133,7 +133,10 @@ SetCurrentThreadName([[maybe_unused]] std::string_view ThreadName) #elif ZEN_PLATFORM_MAC pthread_setname_np(ThreadNameZ.c_str()); #else - pthread_setname_np(pthread_self(), ThreadNameZ.c_str()); + // Linux pthread_setname_np has a 16-byte limit (15 chars + NUL) + StringBuilder<16> LinuxThreadName; + LinuxThreadName << LimitedThreadName.substr(0, 15); + pthread_setname_np(pthread_self(), LinuxThreadName.c_str()); #endif } // namespace zen @@ -233,12 +236,15 @@ Event::Close() #else std::atomic_thread_fence(std::memory_order_acquire); auto* Inner = (EventInner*)m_EventHandle.load(); + if (Inner) { - std::unique_lock Lock(Inner->Mutex); - Inner->bSet.store(true); - m_EventHandle = nullptr; + { + std::unique_lock Lock(Inner->Mutex); + Inner->bSet.store(true); + m_EventHandle = nullptr; + } + delete Inner; } - delete Inner; #endif } @@ -351,7 +357,7 @@ NamedEvent::NamedEvent(std::string_view EventName) intptr_t Packed; Packed = intptr_t(Sem) << 32; Packed |= intptr_t(Fd) & 0xffff'ffff; - m_EventHandle = (void*)Packed; + m_EventHandle = (void*)Packed; #endif ZEN_ASSERT(m_EventHandle != nullptr); } @@ -372,7 +378,9 @@ NamedEvent::Close() #if ZEN_PLATFORM_WINDOWS CloseHandle(m_EventHandle); #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - int Fd = int(intptr_t(m_EventHandle.load()) & 0xffff'ffff); + const intptr_t Handle = intptr_t(m_EventHandle.load()); + const int Fd = int(Handle & 0xffff'ffff); + const int Sem = int(Handle >> 32); if (flock(Fd, LOCK_EX | LOCK_NB) == 0) { @@ -388,11 +396,10 @@ NamedEvent::Close() } flock(Fd, LOCK_UN | LOCK_NB); - close(Fd); - - int Sem = int(intptr_t(m_EventHandle.load()) >> 32); semctl(Sem, 0, IPC_RMID); } + + close(Fd); #endif m_EventHandle = nullptr; @@ -481,9 +488,12 @@ NamedMutex::~NamedMutex() CloseHandle(m_MutexHandle); } #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - int Inner = int(intptr_t(m_MutexHandle)); - flock(Inner, LOCK_UN); - close(Inner); + if (m_MutexHandle) + { + int Inner = int(intptr_t(m_MutexHandle)); + flock(Inner, LOCK_UN); + close(Inner); + } #endif } @@ -516,7 +526,6 @@ NamedMutex::Create(std::string_view MutexName) if (flock(Inner, LOCK_EX) != 0) { close(Inner); - Inner = 0; return false; } @@ -583,6 +592,11 @@ GetCurrentThreadId() void Sleep(int ms) { + if (ms <= 0) + { + return; + } + #if ZEN_PLATFORM_WINDOWS ::Sleep(ms); #else diff --git a/src/zencore/trace.cpp b/src/zencore/trace.cpp index 87035554f..7c195e69f 100644 --- a/src/zencore/trace.cpp +++ b/src/zencore/trace.cpp @@ -10,7 +10,16 @@ # define TRACE_IMPLEMENT 1 # undef _WINSOCK_DEPRECATED_NO_WARNINGS +// GCC false positives in thirdparty trace.h (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100137) +# if ZEN_COMPILER_GCC +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wstringop-overread" +# pragma GCC diagnostic ignored "-Wdangling-pointer" +# endif # include <zencore/trace.h> +# if ZEN_COMPILER_GCC +# pragma GCC diagnostic pop +# endif # include <zencore/memory/fmalloc.h> # include <zencore/memory/memorytrace.h> @@ -165,10 +174,17 @@ GetTraceOptionsFromCommandline(TraceOptions& OutOptions) auto MatchesArg = [](std::string_view Option, std::string_view Arg) -> std::optional<std::string_view> { if (Arg.starts_with(Option)) { - std::string_view::value_type DelimChar = Arg[Option.length()]; - if (DelimChar == ' ' || DelimChar == '=') + if (Arg.length() > Option.length()) + { + std::string_view::value_type DelimChar = Arg[Option.length()]; + if (DelimChar == ' ' || DelimChar == '=') + { + return Arg.substr(Option.size() + 1); + } + } + else { - return Arg.substr(Option.size() + 1); + return ""sv; } } return {}; diff --git a/src/zencore/uid.cpp b/src/zencore/uid.cpp index d7636f2ad..971683721 100644 --- a/src/zencore/uid.cpp +++ b/src/zencore/uid.cpp @@ -156,6 +156,8 @@ Oid::FromMemory(const void* Ptr) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.uid"); + TEST_CASE("Oid") { SUBCASE("Basic") @@ -185,6 +187,8 @@ TEST_CASE("Oid") } } +TEST_SUITE_END(); + void uid_forcelink() { diff --git a/src/zencore/windows.cpp b/src/zencore/windows.cpp index d02fcd35e..87f854b90 100644 --- a/src/zencore/windows.cpp +++ b/src/zencore/windows.cpp @@ -12,14 +12,12 @@ namespace zen::windows { bool IsRunningOnWine() { - HMODULE NtDll = GetModuleHandleA("ntdll.dll"); + static bool s_Result = [] { + HMODULE NtDll = GetModuleHandleA("ntdll.dll"); + return NtDll && !!GetProcAddress(NtDll, "wine_get_version"); + }(); - if (NtDll) - { - return !!GetProcAddress(NtDll, "wine_get_version"); - } - - return false; + return s_Result; } FileMapping::FileMapping(_In_ FileMapping& orig) diff --git a/src/zencore/workthreadpool.cpp b/src/zencore/workthreadpool.cpp index cb84bbe06..1cb338c66 100644 --- a/src/zencore/workthreadpool.cpp +++ b/src/zencore/workthreadpool.cpp @@ -354,6 +354,8 @@ workthreadpool_forcelink() using namespace std::literals; +TEST_SUITE_BEGIN("core.workthreadpool"); + TEST_CASE("threadpool.basic") { WorkerThreadPool Threadpool{1}; @@ -368,6 +370,8 @@ TEST_CASE("threadpool.basic") CHECK_THROWS(FutureThrow.get()); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index a3fd4dacb..171f4c533 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -15,6 +15,7 @@ target('zencore') set_configdir("include/zencore") add_files("**.cpp") add_files("trace.cpp", {unity_ignored = true }) + add_files("testing.cpp", {unity_ignored = true }) if has_config("zenrpmalloc") then add_deps("rpmalloc") @@ -25,7 +26,6 @@ target('zencore') end add_deps("zenbase") - add_deps("spdlog") add_deps("utfcpp") add_deps("oodle") add_deps("blake3") @@ -33,8 +33,6 @@ target('zencore') add_deps("timesinceprocessstart") add_deps("doctest") add_deps("fmt") - add_deps("ryml") - add_packages("json11") if is_plat("linux", "macosx") then diff --git a/src/zencore/xxhash.cpp b/src/zencore/xxhash.cpp index 6d1050531..88a48dd68 100644 --- a/src/zencore/xxhash.cpp +++ b/src/zencore/xxhash.cpp @@ -59,6 +59,8 @@ xxhash_forcelink() { } +TEST_SUITE_BEGIN("core.xxhash"); + TEST_CASE("XXH3_128") { using namespace std::literals; @@ -96,6 +98,8 @@ TEST_CASE("XXH3_128") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/zencore.cpp b/src/zencore/zencore.cpp index 4ff79edc7..8c29a8962 100644 --- a/src/zencore/zencore.cpp +++ b/src/zencore/zencore.cpp @@ -147,7 +147,7 @@ AssertImpl::OnAssert(const char* Filename, int LineNumber, const char* FunctionN Message.push_back('\0'); // We use direct ZEN_LOG here instead of ZEN_ERROR as we don't care about *this* code location in the log - ZEN_LOG(Log(), zen::logging::level::Err, "{}", Message.data()); + ZEN_LOG(Log(), zen::logging::Err, "{}", Message.data()); zen::logging::FlushLogging(); } @@ -285,7 +285,7 @@ zencore_forcelinktests() namespace zen { -TEST_SUITE_BEGIN("core.assert"); +TEST_SUITE_BEGIN("core.zencore"); TEST_CASE("Assert.Default") { |