aboutsummaryrefslogtreecommitdiff
path: root/src/zencore
diff options
context:
space:
mode:
Diffstat (limited to 'src/zencore')
-rw-r--r--src/zencore/base64.cpp192
-rw-r--r--src/zencore/compactbinaryyaml.cpp427
-rw-r--r--src/zencore/filesystem.cpp16
-rw-r--r--src/zencore/include/zencore/base64.h4
-rw-r--r--src/zencore/include/zencore/compactbinaryvalue.h24
-rw-r--r--src/zencore/memtrack/callstacktrace.cpp8
-rw-r--r--src/zencore/string.cpp4
-rw-r--r--src/zencore/xmake.lua2
8 files changed, 523 insertions, 154 deletions
diff --git a/src/zencore/base64.cpp b/src/zencore/base64.cpp
index 1f56ee6c3..fdf5f2d66 100644
--- a/src/zencore/base64.cpp
+++ b/src/zencore/base64.cpp
@@ -1,6 +1,10 @@
// Copyright Epic Games, Inc. All Rights Reserved.
#include <zencore/base64.h>
+#include <zencore/string.h>
+#include <zencore/testing.h>
+
+#include <string>
namespace zen {
@@ -11,7 +15,6 @@ static const uint8_t EncodingAlphabet[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G',
'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'};
/** The table used to convert an ascii character into a 6 bit value */
-#if 0
static const uint8_t DecodingAlphabet[256] = {
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x00-0x0f
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x10-0x1f
@@ -30,7 +33,6 @@ static const uint8_t DecodingAlphabet[256] = {
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xe0-0xef
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // 0xf0-0xff
};
-#endif // 0
template<typename CharType>
uint32_t
@@ -104,4 +106,190 @@ Base64::Encode(const uint8_t* Source, uint32_t Length, CharType* Dest)
template uint32_t Base64::Encode<char>(const uint8_t* Source, uint32_t Length, char* Dest);
template uint32_t Base64::Encode<wchar_t>(const uint8_t* Source, uint32_t Length, wchar_t* Dest);
+template<typename CharType>
+bool
+Base64::Decode(const CharType* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength)
+{
+ // Length must be a multiple of 4
+ if (Length % 4 != 0)
+ {
+ OutLength = 0;
+ return false;
+ }
+
+ uint8_t* DecodedBytes = Dest;
+
+ // Process 4 encoded characters at a time, producing 3 decoded bytes
+ while (Length > 0)
+ {
+ // Count padding characters at the end
+ uint32_t PadCount = 0;
+ if (Source[3] == '=')
+ {
+ PadCount++;
+ if (Source[2] == '=')
+ {
+ PadCount++;
+ }
+ }
+
+ // Look up each character in the decoding table
+ uint8_t A = DecodingAlphabet[static_cast<uint8_t>(Source[0])];
+ uint8_t B = DecodingAlphabet[static_cast<uint8_t>(Source[1])];
+ uint8_t C = (PadCount >= 2) ? 0 : DecodingAlphabet[static_cast<uint8_t>(Source[2])];
+ uint8_t D = (PadCount >= 1) ? 0 : DecodingAlphabet[static_cast<uint8_t>(Source[3])];
+
+ // Check for invalid characters (0xFF means not in the base64 alphabet)
+ if (A == 0xFF || B == 0xFF || C == 0xFF || D == 0xFF)
+ {
+ OutLength = 0;
+ return false;
+ }
+
+ // Reconstruct the 24-bit value from 4 6-bit chunks
+ uint32_t ByteTriplet = (A << 18) | (B << 12) | (C << 6) | D;
+
+ // Extract the 3 bytes
+ *DecodedBytes++ = static_cast<uint8_t>(ByteTriplet >> 16);
+ if (PadCount < 2)
+ {
+ *DecodedBytes++ = static_cast<uint8_t>((ByteTriplet >> 8) & 0xFF);
+ }
+ if (PadCount < 1)
+ {
+ *DecodedBytes++ = static_cast<uint8_t>(ByteTriplet & 0xFF);
+ }
+
+ Source += 4;
+ Length -= 4;
+ }
+
+ OutLength = uint32_t(DecodedBytes - Dest);
+ return true;
+}
+
+template bool Base64::Decode<char>(const char* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength);
+template bool Base64::Decode<wchar_t>(const wchar_t* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength);
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Testing related code follows...
+//
+
+#if ZEN_WITH_TESTS
+
+using namespace std::string_literals;
+
+TEST_CASE("Base64")
+{
+ auto EncodeString = [](std::string_view Input) -> std::string {
+ std::string Result;
+ Result.resize(Base64::GetEncodedDataSize(uint32_t(Input.size())));
+ Base64::Encode(reinterpret_cast<const uint8_t*>(Input.data()), uint32_t(Input.size()), Result.data());
+ return Result;
+ };
+
+ auto DecodeString = [](std::string_view Input) -> std::string {
+ std::string Result;
+ Result.resize(Base64::GetMaxDecodedDataSize(uint32_t(Input.size())));
+ uint32_t DecodedLength = 0;
+ bool Success = Base64::Decode(Input.data(), uint32_t(Input.size()), reinterpret_cast<uint8_t*>(Result.data()), DecodedLength);
+ CHECK(Success);
+ Result.resize(DecodedLength);
+ return Result;
+ };
+
+ SUBCASE("Encode")
+ {
+ CHECK(EncodeString("") == ""s);
+ CHECK(EncodeString("f") == "Zg=="s);
+ CHECK(EncodeString("fo") == "Zm8="s);
+ CHECK(EncodeString("foo") == "Zm9v"s);
+ CHECK(EncodeString("foob") == "Zm9vYg=="s);
+ CHECK(EncodeString("fooba") == "Zm9vYmE="s);
+ CHECK(EncodeString("foobar") == "Zm9vYmFy"s);
+ }
+
+ SUBCASE("Decode")
+ {
+ CHECK(DecodeString("") == ""s);
+ CHECK(DecodeString("Zg==") == "f"s);
+ CHECK(DecodeString("Zm8=") == "fo"s);
+ CHECK(DecodeString("Zm9v") == "foo"s);
+ CHECK(DecodeString("Zm9vYg==") == "foob"s);
+ CHECK(DecodeString("Zm9vYmE=") == "fooba"s);
+ CHECK(DecodeString("Zm9vYmFy") == "foobar"s);
+ }
+
+ SUBCASE("RoundTrip")
+ {
+ auto RoundTrip = [&](const std::string& Input) {
+ std::string Encoded = EncodeString(Input);
+ std::string Decoded = DecodeString(Encoded);
+ CHECK(Decoded == Input);
+ };
+
+ RoundTrip("Hello, World!");
+ RoundTrip("Base64 encoding test with various lengths");
+ RoundTrip("A");
+ RoundTrip("AB");
+ RoundTrip("ABC");
+ RoundTrip("ABCD");
+ RoundTrip("\x00\x01\x02\xff\xfe\xfd"s);
+ }
+
+ SUBCASE("BinaryRoundTrip")
+ {
+ // Test with all byte values 0-255
+ uint8_t AllBytes[256];
+ for (int i = 0; i < 256; ++i)
+ {
+ AllBytes[i] = static_cast<uint8_t>(i);
+ }
+
+ char Encoded[Base64::GetEncodedDataSize(256) + 1];
+ Base64::Encode(AllBytes, 256, Encoded);
+
+ uint8_t Decoded[256];
+ uint32_t DecodedLength = 0;
+ bool Success = Base64::Decode(Encoded, uint32_t(strlen(Encoded)), Decoded, DecodedLength);
+ CHECK(Success);
+ CHECK(DecodedLength == 256);
+ CHECK(memcmp(AllBytes, Decoded, 256) == 0);
+ }
+
+ SUBCASE("DecodeInvalidInput")
+ {
+ uint8_t Dest[64];
+ uint32_t OutLength = 0;
+
+ // Length not a multiple of 4
+ CHECK_FALSE(Base64::Decode("abc", 3u, Dest, OutLength));
+
+ // Invalid character
+ CHECK_FALSE(Base64::Decode("ab!d", 4u, Dest, OutLength));
+ }
+
+ SUBCASE("EncodedDataSize")
+ {
+ CHECK(Base64::GetEncodedDataSize(0) == 0);
+ CHECK(Base64::GetEncodedDataSize(1) == 4);
+ CHECK(Base64::GetEncodedDataSize(2) == 4);
+ CHECK(Base64::GetEncodedDataSize(3) == 4);
+ CHECK(Base64::GetEncodedDataSize(4) == 8);
+ CHECK(Base64::GetEncodedDataSize(5) == 8);
+ CHECK(Base64::GetEncodedDataSize(6) == 8);
+ }
+
+ SUBCASE("MaxDecodedDataSize")
+ {
+ CHECK(Base64::GetMaxDecodedDataSize(0) == 0);
+ CHECK(Base64::GetMaxDecodedDataSize(4) == 3);
+ CHECK(Base64::GetMaxDecodedDataSize(8) == 6);
+ CHECK(Base64::GetMaxDecodedDataSize(12) == 9);
+ }
+}
+
+#endif
+
} // namespace zen
diff --git a/src/zencore/compactbinaryyaml.cpp b/src/zencore/compactbinaryyaml.cpp
index 5122e952a..b308af418 100644
--- a/src/zencore/compactbinaryyaml.cpp
+++ b/src/zencore/compactbinaryyaml.cpp
@@ -14,11 +14,6 @@
#include <string_view>
#include <vector>
-ZEN_THIRD_PARTY_INCLUDES_START
-#include <ryml.hpp>
-#include <ryml_std.hpp>
-ZEN_THIRD_PARTY_INCLUDES_END
-
namespace zen {
//////////////////////////////////////////////////////////////////////////
@@ -26,193 +21,349 @@ namespace zen {
class CbYamlWriter
{
public:
- explicit CbYamlWriter(StringBuilderBase& InBuilder) : m_StrBuilder(InBuilder) { m_NodeStack.push_back(m_Tree.rootref()); }
+ explicit CbYamlWriter(StringBuilderBase& InBuilder) : m_Builder(InBuilder) {}
void WriteField(CbFieldView Field)
{
- ryml::NodeRef Node;
+ CbValue Accessor = Field.GetValue();
+ CbFieldType Type = Accessor.GetType();
- if (m_IsFirst)
+ switch (Type)
{
- Node = Top();
+ case CbFieldType::Object:
+ case CbFieldType::UniformObject:
+ WriteMapEntries(Field, 0);
+ break;
+ case CbFieldType::Array:
+ case CbFieldType::UniformArray:
+ WriteSeqEntries(Field, 0);
+ break;
+ default:
+ WriteScalarValue(Field);
+ m_Builder << '\n';
+ break;
+ }
+ }
+
+ void WriteMapEntry(CbFieldView Field, int32_t Indent)
+ {
+ WriteIndent(Indent);
+ WriteMapEntryContent(Field, Indent);
+ }
+
+ void WriteSeqEntry(CbFieldView Field, int32_t Indent)
+ {
+ CbValue Accessor = Field.GetValue();
+ CbFieldType Type = Accessor.GetType();
- m_IsFirst = false;
+ if (Type == CbFieldType::Object || Type == CbFieldType::UniformObject)
+ {
+ bool First = true;
+ for (CbFieldView MapChild : Field)
+ {
+ if (First)
+ {
+ WriteIndent(Indent);
+ m_Builder << "- ";
+ First = false;
+ }
+ else
+ {
+ WriteIndent(Indent + 1);
+ }
+ WriteMapEntryContent(MapChild, Indent + 1);
+ }
+ }
+ else if (Type == CbFieldType::Array || Type == CbFieldType::UniformArray)
+ {
+ WriteIndent(Indent);
+ m_Builder << "-\n";
+ WriteSeqEntries(Field, Indent + 1);
}
else
{
- Node = Top().append_child();
+ WriteIndent(Indent);
+ m_Builder << "- ";
+ WriteScalarValue(Field);
+ m_Builder << '\n';
}
+ }
- if (std::u8string_view Name = Field.GetU8Name(); !Name.empty())
+private:
+ void WriteMapEntries(CbFieldView MapField, int32_t Indent)
+ {
+ for (CbFieldView Child : MapField)
{
- Node.set_key_serialized(ryml::csubstr((const char*)Name.data(), Name.size()));
+ WriteIndent(Indent);
+ WriteMapEntryContent(Child, Indent);
}
+ }
+
+ void WriteMapEntryContent(CbFieldView Field, int32_t Indent)
+ {
+ std::u8string_view Name = Field.GetU8Name();
+ m_Builder << std::string_view(reinterpret_cast<const char*>(Name.data()), Name.size());
- switch (CbValue Accessor = Field.GetValue(); Accessor.GetType())
+ CbValue Accessor = Field.GetValue();
+ CbFieldType Type = Accessor.GetType();
+
+ if (IsContainer(Type))
{
- case CbFieldType::Null:
- Node.set_val("null");
- break;
- case CbFieldType::Object:
- case CbFieldType::UniformObject:
- Node |= ryml::MAP;
- m_NodeStack.push_back(Node);
- for (CbFieldView It : Field)
+ m_Builder << ":\n";
+ WriteFieldValue(Field, Indent + 1);
+ }
+ else
+ {
+ m_Builder << ": ";
+ WriteScalarValue(Field);
+ m_Builder << '\n';
+ }
+ }
+
+ void WriteSeqEntries(CbFieldView SeqField, int32_t Indent)
+ {
+ for (CbFieldView Child : SeqField)
+ {
+ CbValue Accessor = Child.GetValue();
+ CbFieldType Type = Accessor.GetType();
+
+ if (Type == CbFieldType::Object || Type == CbFieldType::UniformObject)
+ {
+ bool First = true;
+ for (CbFieldView MapChild : Child)
{
- WriteField(It);
+ if (First)
+ {
+ WriteIndent(Indent);
+ m_Builder << "- ";
+ First = false;
+ }
+ else
+ {
+ WriteIndent(Indent + 1);
+ }
+ WriteMapEntryContent(MapChild, Indent + 1);
}
- m_NodeStack.pop_back();
+ }
+ else if (Type == CbFieldType::Array || Type == CbFieldType::UniformArray)
+ {
+ WriteIndent(Indent);
+ m_Builder << "-\n";
+ WriteSeqEntries(Child, Indent + 1);
+ }
+ else
+ {
+ WriteIndent(Indent);
+ m_Builder << "- ";
+ WriteScalarValue(Child);
+ m_Builder << '\n';
+ }
+ }
+ }
+
+ void WriteFieldValue(CbFieldView Field, int32_t Indent)
+ {
+ CbValue Accessor = Field.GetValue();
+ CbFieldType Type = Accessor.GetType();
+
+ switch (Type)
+ {
+ case CbFieldType::Object:
+ case CbFieldType::UniformObject:
+ WriteMapEntries(Field, Indent);
break;
case CbFieldType::Array:
case CbFieldType::UniformArray:
- Node |= ryml::SEQ;
- m_NodeStack.push_back(Node);
- for (CbFieldView It : Field)
- {
- WriteField(It);
- }
- m_NodeStack.pop_back();
+ WriteSeqEntries(Field, Indent);
break;
- case CbFieldType::Binary:
- {
- ExtendableStringBuilder<256> Builder;
- const MemoryView Value = Accessor.AsBinary();
- ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024);
- const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
- const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize));
- Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex);
-
- Node.set_key_serialized(Builder.c_str());
- }
+ case CbFieldType::CustomById:
+ WriteCustomById(Field.GetValue().AsCustomById(), Indent);
break;
- case CbFieldType::String:
- {
- const std::u8string_view U8String = Accessor.AsU8String();
- Node.set_val(ryml::csubstr((const char*)U8String.data(), U8String.size()));
- }
+ case CbFieldType::CustomByName:
+ WriteCustomByName(Field.GetValue().AsCustomByName(), Indent);
+ break;
+ default:
+ WriteScalarValue(Field);
+ m_Builder << '\n';
+ break;
+ }
+ }
+
+ void WriteScalarValue(CbFieldView Field)
+ {
+ CbValue Accessor = Field.GetValue();
+ switch (Accessor.GetType())
+ {
+ case CbFieldType::Null:
+ m_Builder << "null";
+ break;
+ case CbFieldType::BoolFalse:
+ m_Builder << "false";
+ break;
+ case CbFieldType::BoolTrue:
+ m_Builder << "true";
break;
case CbFieldType::IntegerPositive:
- Node << Accessor.AsIntegerPositive();
+ m_Builder << Accessor.AsIntegerPositive();
break;
case CbFieldType::IntegerNegative:
- Node << Accessor.AsIntegerNegative();
+ m_Builder << Accessor.AsIntegerNegative();
break;
case CbFieldType::Float32:
if (const float Value = Accessor.AsFloat32(); std::isfinite(Value))
- {
- Node << Value;
- }
+ m_Builder.Append(fmt::format("{}", Value));
else
- {
- Node << "null";
- }
+ m_Builder << "null";
break;
case CbFieldType::Float64:
if (const double Value = Accessor.AsFloat64(); std::isfinite(Value))
- {
- Node << Value;
- }
+ m_Builder.Append(fmt::format("{}", Value));
else
+ m_Builder << "null";
+ break;
+ case CbFieldType::String:
{
- Node << "null";
+ const std::u8string_view U8String = Accessor.AsU8String();
+ WriteString(std::string_view(reinterpret_cast<const char*>(U8String.data()), U8String.size()));
}
break;
- case CbFieldType::BoolFalse:
- Node << "false";
- break;
- case CbFieldType::BoolTrue:
- Node << "true";
+ case CbFieldType::Hash:
+ WriteString(Accessor.AsHash().ToHexString());
break;
case CbFieldType::ObjectAttachment:
case CbFieldType::BinaryAttachment:
- Node << Accessor.AsAttachment().ToHexString();
- break;
- case CbFieldType::Hash:
- Node << Accessor.AsHash().ToHexString();
+ WriteString(Accessor.AsAttachment().ToHexString());
break;
case CbFieldType::Uuid:
- Node << fmt::format("{}", Accessor.AsUuid());
+ WriteString(fmt::format("{}", Accessor.AsUuid()));
break;
case CbFieldType::DateTime:
- Node << DateTime(Accessor.AsDateTimeTicks()).ToIso8601();
+ WriteString(DateTime(Accessor.AsDateTimeTicks()).ToIso8601());
break;
case CbFieldType::TimeSpan:
if (const TimeSpan Span(Accessor.AsTimeSpanTicks()); Span.GetDays() == 0)
- {
- Node << Span.ToString("%h:%m:%s.%n");
- }
+ WriteString(Span.ToString("%h:%m:%s.%n"));
else
- {
- Node << Span.ToString("%d.%h:%m:%s.%n");
- }
+ WriteString(Span.ToString("%d.%h:%m:%s.%n"));
break;
case CbFieldType::ObjectId:
- Node << fmt::format("{}", Accessor.AsObjectId());
+ WriteString(fmt::format("{}", Accessor.AsObjectId()));
break;
- case CbFieldType::CustomById:
- {
- CbCustomById Custom = Accessor.AsCustomById();
+ case CbFieldType::Binary:
+ WriteBase64(Accessor.AsBinary());
+ break;
+ default:
+ ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType()));
+ break;
+ }
+ }
- Node |= ryml::MAP;
+ void WriteCustomById(CbCustomById Custom, int32_t Indent)
+ {
+ WriteIndent(Indent);
+ m_Builder << "Id: ";
+ m_Builder.Append(fmt::format("{}", Custom.Id));
+ m_Builder << '\n';
+
+ WriteIndent(Indent);
+ m_Builder << "Data: ";
+ WriteBase64(Custom.Data);
+ m_Builder << '\n';
+ }
- ryml::NodeRef IdNode = Node.append_child();
- IdNode.set_key("Id");
- IdNode.set_val_serialized(fmt::format("{}", Custom.Id));
+ void WriteCustomByName(CbCustomByName Custom, int32_t Indent)
+ {
+ WriteIndent(Indent);
+ m_Builder << "Name: ";
+ WriteString(std::string_view(reinterpret_cast<const char*>(Custom.Name.data()), Custom.Name.size()));
+ m_Builder << '\n';
+
+ WriteIndent(Indent);
+ m_Builder << "Data: ";
+ WriteBase64(Custom.Data);
+ m_Builder << '\n';
+ }
- ryml::NodeRef DataNode = Node.append_child();
- DataNode.set_key("Data");
+ void WriteBase64(MemoryView Value)
+ {
+ ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024);
+ ExtendableStringBuilder<256> Buf;
+ const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
+ const size_t EncodedIndex = Buf.AddUninitialized(size_t(EncodedSize));
+ Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Buf.Data() + EncodedIndex);
+ WriteString(Buf.ToView());
+ }
- ExtendableStringBuilder<256> Builder;
- const MemoryView& Value = Custom.Data;
- const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
- const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize));
- Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex);
+ void WriteString(std::string_view Str)
+ {
+ if (NeedsQuoting(Str))
+ {
+ m_Builder << '\'';
+ for (char C : Str)
+ {
+ if (C == '\'')
+ m_Builder << "''";
+ else
+ m_Builder << C;
+ }
+ m_Builder << '\'';
+ }
+ else
+ {
+ m_Builder << Str;
+ }
+ }
- DataNode.set_val_serialized(Builder.c_str());
- }
- break;
- case CbFieldType::CustomByName:
- {
- CbCustomByName Custom = Accessor.AsCustomByName();
+ void WriteIndent(int32_t Indent)
+ {
+ for (int32_t I = 0; I < Indent; ++I)
+ m_Builder << " ";
+ }
- Node |= ryml::MAP;
+ static bool NeedsQuoting(std::string_view Str)
+ {
+ if (Str.empty())
+ return false;
- ryml::NodeRef NameNode = Node.append_child();
- NameNode.set_key("Name");
- std::string_view Name = std::string_view((const char*)Custom.Name.data(), Custom.Name.size());
- NameNode.set_val_serialized(std::string(Name));
+ char First = Str[0];
+ if (First == ' ' || First == '\n' || First == '\t' || First == '\r' || First == '*' || First == '&' || First == '%' ||
+ First == '@' || First == '`')
+ return true;
- ryml::NodeRef DataNode = Node.append_child();
- DataNode.set_key("Data");
+ if (Str.size() >= 2 && Str[0] == '<' && Str[1] == '<')
+ return true;
- ExtendableStringBuilder<256> Builder;
- const MemoryView& Value = Custom.Data;
- const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
- const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize));
- Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex);
+ char Last = Str.back();
+ if (Last == ' ' || Last == '\n' || Last == '\t' || Last == '\r')
+ return true;
- DataNode.set_val_serialized(Builder.c_str());
- }
- break;
- default:
- ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType()));
- break;
+ for (char C : Str)
+ {
+ if (C == '#' || C == ':' || C == '-' || C == '?' || C == ',' || C == '\n' || C == '{' || C == '}' || C == '[' || C == ']' ||
+ C == '\'' || C == '"')
+ return true;
}
- if (m_NodeStack.size() == 1)
+ return false;
+ }
+
+ static bool IsContainer(CbFieldType Type)
+ {
+ switch (Type)
{
- std::string Yaml = ryml::emitrs_yaml<std::string>(m_Tree);
- m_StrBuilder << Yaml;
+ case CbFieldType::Object:
+ case CbFieldType::UniformObject:
+ case CbFieldType::Array:
+ case CbFieldType::UniformArray:
+ case CbFieldType::CustomById:
+ case CbFieldType::CustomByName:
+ return true;
+ default:
+ return false;
}
}
-private:
- StringBuilderBase& m_StrBuilder;
- bool m_IsFirst = true;
-
- ryml::Tree m_Tree;
- std::vector<ryml::NodeRef> m_NodeStack;
- ryml::NodeRef& Top() { return m_NodeStack.back(); }
+ StringBuilderBase& m_Builder;
};
void
@@ -229,6 +380,32 @@ CompactBinaryToYaml(const CbArrayView& Array, StringBuilderBase& Builder)
Writer.WriteField(Array.AsFieldView());
}
+void
+CompactBinaryToYaml(MemoryView Data, StringBuilderBase& InBuilder)
+{
+ std::vector<CbFieldView> Fields = ReadCompactBinaryStream(Data);
+ if (Fields.empty())
+ return;
+
+ CbYamlWriter Writer(InBuilder);
+ if (Fields.size() == 1)
+ {
+ Writer.WriteField(Fields[0]);
+ return;
+ }
+
+ if (Fields[0].HasName())
+ {
+ for (const CbFieldView& Field : Fields)
+ Writer.WriteMapEntry(Field, 0);
+ }
+ else
+ {
+ for (const CbFieldView& Field : Fields)
+ Writer.WriteSeqEntry(Field, 0);
+ }
+}
+
#if ZEN_WITH_TESTS
void
cbyaml_forcelink()
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 1a4ee4b9b..553897407 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -1326,11 +1326,6 @@ ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uin
{
BytesRead = size_t(dwNumberOfBytesRead);
}
- else if ((BytesRead != NumberOfBytesToRead))
- {
- Ec = MakeErrorCode(ERROR_HANDLE_EOF);
- return;
- }
else
{
Ec = MakeErrorCodeFromLastError();
@@ -1344,20 +1339,15 @@ ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uin
{
BytesRead = size_t(ReadResult);
}
- else if ((BytesRead != NumberOfBytesToRead))
- {
- Ec = MakeErrorCode(EIO);
- return;
- }
else
{
Ec = MakeErrorCodeFromLastError();
return;
}
#endif
- Size -= NumberOfBytesToRead;
- FileOffset += NumberOfBytesToRead;
- Data = reinterpret_cast<uint8_t*>(Data) + NumberOfBytesToRead;
+ Size -= BytesRead;
+ FileOffset += BytesRead;
+ Data = reinterpret_cast<uint8_t*>(Data) + BytesRead;
}
}
diff --git a/src/zencore/include/zencore/base64.h b/src/zencore/include/zencore/base64.h
index 4d78b085f..08d9f3043 100644
--- a/src/zencore/include/zencore/base64.h
+++ b/src/zencore/include/zencore/base64.h
@@ -11,7 +11,11 @@ struct Base64
template<typename CharType>
static uint32_t Encode(const uint8_t* Source, uint32_t Length, CharType* Dest);
+ template<typename CharType>
+ static bool Decode(const CharType* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength);
+
static inline constexpr int32_t GetEncodedDataSize(uint32_t Size) { return ((Size + 2) / 3) * 4; }
+ static inline constexpr int32_t GetMaxDecodedDataSize(uint32_t Length) { return (Length / 4) * 3; }
};
} // namespace zen
diff --git a/src/zencore/include/zencore/compactbinaryvalue.h b/src/zencore/include/zencore/compactbinaryvalue.h
index aa2d2821d..4ce8009b8 100644
--- a/src/zencore/include/zencore/compactbinaryvalue.h
+++ b/src/zencore/include/zencore/compactbinaryvalue.h
@@ -128,17 +128,21 @@ CbValue::AsString(CbFieldError* OutError, std::string_view Default) const
uint32_t ValueSizeByteCount;
const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount);
- if (OutError)
+ if (ValueSize >= (uint64_t(1) << 31))
{
- if (ValueSize >= (uint64_t(1) << 31))
+ if (OutError)
{
*OutError = CbFieldError::RangeError;
- return Default;
}
+ return Default;
+ }
+
+ if (OutError)
+ {
*OutError = CbFieldError::None;
}
- return std::string_view(Chars + ValueSizeByteCount, int32_t(ValueSize));
+ return std::string_view(Chars + ValueSizeByteCount, size_t(ValueSize));
}
inline std::u8string_view
@@ -148,17 +152,21 @@ CbValue::AsU8String(CbFieldError* OutError, std::u8string_view Default) const
uint32_t ValueSizeByteCount;
const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount);
- if (OutError)
+ if (ValueSize >= (uint64_t(1) << 31))
{
- if (ValueSize >= (uint64_t(1) << 31))
+ if (OutError)
{
*OutError = CbFieldError::RangeError;
- return Default;
}
+ return Default;
+ }
+
+ if (OutError)
+ {
*OutError = CbFieldError::None;
}
- return std::u8string_view(Chars + ValueSizeByteCount, int32_t(ValueSize));
+ return std::u8string_view(Chars + ValueSizeByteCount, size_t(ValueSize));
}
inline uint64_t
diff --git a/src/zencore/memtrack/callstacktrace.cpp b/src/zencore/memtrack/callstacktrace.cpp
index a5b7fede6..4a7068568 100644
--- a/src/zencore/memtrack/callstacktrace.cpp
+++ b/src/zencore/memtrack/callstacktrace.cpp
@@ -169,13 +169,13 @@ private:
std::atomic_uint64_t Key;
std::atomic_uint32_t Value;
- inline uint64 GetKey() const { return Key.load(std::memory_order_relaxed); }
+ inline uint64 GetKey() const { return Key.load(std::memory_order_acquire); }
inline uint32_t GetValue() const { return Value.load(std::memory_order_relaxed); }
- inline bool IsEmpty() const { return Key.load(std::memory_order_relaxed) == 0; }
+ inline bool IsEmpty() const { return Key.load(std::memory_order_acquire) == 0; }
inline void SetKeyValue(uint64_t InKey, uint32_t InValue)
{
- Value.store(InValue, std::memory_order_release);
- Key.store(InKey, std::memory_order_relaxed);
+ Value.store(InValue, std::memory_order_relaxed);
+ Key.store(InKey, std::memory_order_release);
}
static inline uint32_t KeyHash(uint64_t Key) { return static_cast<uint32_t>(Key); }
static inline void ClearEntries(FEncounteredCallstackSetEntry* Entries, int32_t EntryCount)
diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp
index 0ee863b74..a9aed6309 100644
--- a/src/zencore/string.cpp
+++ b/src/zencore/string.cpp
@@ -24,6 +24,10 @@ utf16to8_impl(u16bit_iterator StartIt, u16bit_iterator EndIt, ::zen::StringBuild
// Take care of surrogate pairs first
if (utf8::internal::is_lead_surrogate(cp))
{
+ if (StartIt == EndIt)
+ {
+ break;
+ }
uint32_t trail_surrogate = utf8::internal::mask16(*StartIt++);
cp = (cp << 10) + trail_surrogate + utf8::internal::SURROGATE_OFFSET;
}
diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua
index 4b6e7c44b..322001ef9 100644
--- a/src/zencore/xmake.lua
+++ b/src/zencore/xmake.lua
@@ -32,8 +32,6 @@ target('zencore')
add_deps("timesinceprocessstart")
add_deps("doctest")
add_deps("fmt")
- add_deps("ryml")
-
add_packages("json11")
if is_plat("linux", "macosx") then
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-11 10:59:44 +0000