diff options
| author | Stefan Boberg <[email protected]> | 2025-10-06 22:33:00 +0200 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2025-10-06 22:33:00 +0200 |
| commit | 1383dbdc563d90c170ab30ba622ee44e2e37e723 (patch) | |
| tree | 59777db60000fe2ab2334f05776fb9ded4ca41fb /src/zencore/compactbinary.cpp | |
| parent | Merge branch 'main' into sb/rpc-analysis (diff) | |
| parent | 5.7.6 (diff) | |
| download | zen-1383dbdc563d90c170ab30ba622ee44e2e37e723.tar.xz zen-1383dbdc563d90c170ab30ba622ee44e2e37e723.zip | |
Merge remote-tracking branch 'origin/main' into sb/rpc-analysis
Diffstat (limited to 'src/zencore/compactbinary.cpp')
| -rw-r--r-- | src/zencore/compactbinary.cpp | 817 |
1 files changed, 28 insertions, 789 deletions
diff --git a/src/zencore/compactbinary.cpp b/src/zencore/compactbinary.cpp index 6677b5a61..b43cc18f1 100644 --- a/src/zencore/compactbinary.cpp +++ b/src/zencore/compactbinary.cpp @@ -15,6 +15,8 @@ #include <zencore/testing.h> #include <zencore/uid.h> +#include <EASTL/fixed_vector.h> + #include <fmt/format.h> #include <string_view> @@ -24,10 +26,6 @@ # include <time.h> #endif -ZEN_THIRD_PARTY_INCLUDES_START -#include <json11.hpp> -ZEN_THIRD_PARTY_INCLUDES_END - namespace zen { const int DaysToMonth[] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365}; @@ -411,7 +409,7 @@ CbFieldView::CbFieldView(const void* DataPointer, CbFieldType FieldType) } void -CbFieldView::IterateAttachments(std::function<void(CbFieldView)> Visitor) const +CbFieldView::IterateAttachments(const std::function<void(CbFieldView)>& Visitor) const { switch (CbFieldTypeOps::GetType(Type)) { @@ -1174,7 +1172,7 @@ template class TCbFieldIterator<CbField>; template<typename FieldType> void -TCbFieldIterator<FieldType>::IterateRangeAttachments(std::function<void(CbFieldView)> Visitor) const +TCbFieldIterator<FieldType>::IterateRangeAttachments(const std::function<void(CbFieldView)>& Visitor) const { if (CbFieldTypeOps::HasFieldType(FieldType::GetType())) { @@ -1380,9 +1378,9 @@ TryMeasureCompactBinary(MemoryView View, CbFieldType& OutType, uint64_t& OutSize CbField LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator) { - std::vector<uint8_t> HeaderBytes; - CbFieldType FieldType; - uint64_t FieldSize = 1; + eastl::fixed_vector<uint8_t, 32> HeaderBytes; + CbFieldType FieldType; + uint64_t FieldSize = 1; for (const int64_t StartPos = Ar.CurrentOffset(); FieldSize > 0;) { @@ -1397,7 +1395,7 @@ LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator) HeaderBytes.resize(ReadOffset + ReadSize); Ar.Read(HeaderBytes.data() + ReadOffset, ReadSize); - if (TryMeasureCompactBinary(MakeMemoryView(HeaderBytes), FieldType, FieldSize)) + if (TryMeasureCompactBinary(MakeMemoryView(HeaderBytes.data(), HeaderBytes.size()), FieldType, FieldSize)) { if (FieldSize <= uint64_t(Ar.Size() - StartPos)) { @@ -1425,25 +1423,43 @@ LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator) CbObject LoadCompactBinaryObject(IoBuffer&& Payload) { + if (Payload.GetSize() == 0) + { + return CbObject(); + } return CbObject{SharedBuffer(std::move(Payload))}; } CbObject LoadCompactBinaryObject(const IoBuffer& Payload) { + if (Payload.GetSize() == 0) + { + return CbObject(); + } return CbObject{SharedBuffer(Payload)}; } CbObject LoadCompactBinaryObject(CompressedBuffer&& Payload) { - return CbObject{SharedBuffer(Payload.DecompressToComposite().Flatten())}; + CompositeBuffer Decompressed = std::move(Payload).DecompressToComposite(); + if (Decompressed.GetSize() == 0) + { + return CbObject(); + } + return CbObject{std::move(Decompressed).Flatten()}; } CbObject LoadCompactBinaryObject(const CompressedBuffer& Payload) { - return CbObject{SharedBuffer(Payload.DecompressToComposite().Flatten())}; + CompositeBuffer Decompressed = Payload.DecompressToComposite(); + if (Decompressed.GetSize() == 0) + { + return CbObject(); + } + return CbObject{std::move(Decompressed).Flatten()}; } ////////////////////////////////////////////////////////////////////////// @@ -1468,339 +1484,6 @@ SaveCompactBinary(BinaryWriter& Ar, const CbObjectView& Object) ////////////////////////////////////////////////////////////////////////// -class CbJsonWriter -{ -public: - explicit CbJsonWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) { NewLineAndIndent << LINE_TERMINATOR_ANSI; } - - void BeginObject() - { - Builder << '{'; - NewLineAndIndent << '\t'; - NeedsNewLine = true; - } - - void EndObject() - { - NewLineAndIndent.RemoveSuffix(1); - if (NeedsComma) - { - WriteOptionalNewLine(); - } - Builder << '}'; - } - - void BeginArray() - { - Builder << '['; - NewLineAndIndent << '\t'; - NeedsNewLine = true; - } - - void EndArray() - { - NewLineAndIndent.RemoveSuffix(1); - if (NeedsComma) - { - WriteOptionalNewLine(); - } - Builder << ']'; - } - - void WriteField(CbFieldView Field) - { - using namespace std::literals; - - WriteOptionalComma(); - WriteOptionalNewLine(); - - if (std::u8string_view Name = Field.GetU8Name(); !Name.empty()) - { - AppendQuotedString(Name); - Builder << ": "sv; - } - - switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) - { - case CbFieldType::Null: - Builder << "null"sv; - break; - case CbFieldType::Object: - case CbFieldType::UniformObject: - { - BeginObject(); - for (CbFieldView It : Field) - { - WriteField(It); - } - EndObject(); - } - break; - case CbFieldType::Array: - case CbFieldType::UniformArray: - { - BeginArray(); - for (CbFieldView It : Field) - { - WriteField(It); - } - EndArray(); - } - break; - case CbFieldType::Binary: - AppendBase64String(Accessor.AsBinary()); - break; - case CbFieldType::String: - AppendQuotedString(Accessor.AsU8String()); - break; - case CbFieldType::IntegerPositive: - Builder << Accessor.AsIntegerPositive(); - break; - case CbFieldType::IntegerNegative: - Builder << Accessor.AsIntegerNegative(); - break; - case CbFieldType::Float32: - { - const float Value = Accessor.AsFloat32(); - if (std::isfinite(Value)) - { - Builder.Append(fmt::format("{:.9g}", Value)); - } - else - { - Builder << "null"sv; - } - } - break; - case CbFieldType::Float64: - { - const double Value = Accessor.AsFloat64(); - if (std::isfinite(Value)) - { - Builder.Append(fmt::format("{:.17g}", Value)); - } - else - { - Builder << "null"sv; - } - } - break; - case CbFieldType::BoolFalse: - Builder << "false"sv; - break; - case CbFieldType::BoolTrue: - Builder << "true"sv; - break; - case CbFieldType::ObjectAttachment: - case CbFieldType::BinaryAttachment: - { - Builder << '"'; - Accessor.AsAttachment().ToHexString(Builder); - Builder << '"'; - } - break; - case CbFieldType::Hash: - { - Builder << '"'; - Accessor.AsHash().ToHexString(Builder); - Builder << '"'; - } - break; - case CbFieldType::Uuid: - { - Builder << '"'; - Accessor.AsUuid().ToString(Builder); - Builder << '"'; - } - break; - case CbFieldType::DateTime: - Builder << '"' << DateTime(Accessor.AsDateTimeTicks()).ToIso8601() << '"'; - break; - case CbFieldType::TimeSpan: - { - const TimeSpan Span(Accessor.AsTimeSpanTicks()); - if (Span.GetDays() == 0) - { - Builder << '"' << Span.ToString("%h:%m:%s.%n") << '"'; - } - else - { - Builder << '"' << Span.ToString("%d.%h:%m:%s.%n") << '"'; - } - break; - } - case CbFieldType::ObjectId: - Builder << '"'; - Accessor.AsObjectId().ToString(Builder); - Builder << '"'; - break; - case CbFieldType::CustomById: - { - CbCustomById Custom = Accessor.AsCustomById(); - Builder << "{ \"Id\": "; - Builder << Custom.Id; - Builder << ", \"Data\": "; - AppendBase64String(Custom.Data); - Builder << " }"; - break; - } - case CbFieldType::CustomByName: - { - CbCustomByName Custom = Accessor.AsCustomByName(); - Builder << "{ \"Name\": "; - AppendQuotedString(Custom.Name); - Builder << ", \"Data\": "; - AppendBase64String(Custom.Data); - Builder << " }"; - break; - } - default: - ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType())); - break; - } - - NeedsComma = true; - NeedsNewLine = true; - } - -private: - void WriteOptionalComma() - { - if (NeedsComma) - { - NeedsComma = false; - Builder << ','; - } - } - - void WriteOptionalNewLine() - { - if (NeedsNewLine) - { - NeedsNewLine = false; - Builder << NewLineAndIndent; - } - } - - void AppendQuotedString(std::u8string_view Value) - { - using namespace std::literals; - - const AsciiSet EscapeSet( - "\\\"\b\f\n\r\t" - "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"); - - Builder << '\"'; - while (!Value.empty()) - { - std::u8string_view Verbatim = AsciiSet::FindPrefixWithout(Value, EscapeSet); - Builder << Verbatim; - - Value = Value.substr(Verbatim.size()); - - std::u8string_view Escape = AsciiSet::FindPrefixWith(Value, EscapeSet); - for (char Char : Escape) - { - switch (Char) - { - case '\\': - Builder << "\\\\"sv; - break; - case '\"': - Builder << "\\\""sv; - break; - case '\b': - Builder << "\\b"sv; - break; - case '\f': - Builder << "\\f"sv; - break; - case '\n': - Builder << "\\n"sv; - break; - case '\r': - Builder << "\\r"sv; - break; - case '\t': - Builder << "\\t"sv; - break; - default: - Builder << Char; - break; - } - } - Value = Value.substr(Escape.size()); - } - Builder << '\"'; - } - - void AppendBase64String(MemoryView Value) - { - Builder << '"'; - ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); - } - -private: - StringBuilderBase& Builder; - ExtendableStringBuilder<32> NewLineAndIndent; - bool NeedsComma{false}; - bool NeedsNewLine{false}; -}; - -void -CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder) -{ - CbJsonWriter Writer(Builder); - Writer.WriteField(Object.AsFieldView()); -} - -void -CompactBinaryToJson(const CbArrayView& Array, StringBuilderBase& Builder) -{ - CbJsonWriter Writer(Builder); - Writer.WriteField(Array.AsFieldView()); -} - -void -CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder) -{ - std::vector<CbFieldView> Fields = ReadCompactBinaryStream(Data); - CbJsonWriter Writer(InBuilder); - if (!Fields.empty()) - { - if (Fields.size() == 1) - { - Writer.WriteField(Fields[0]); - return; - } - bool UseTopLevelObject = Fields[0].HasName(); - if (UseTopLevelObject) - { - Writer.BeginObject(); - } - else - { - Writer.BeginArray(); - } - for (const CbFieldView& Field : Fields) - { - Writer.WriteField(Field); - } - if (UseTopLevelObject) - { - Writer.EndObject(); - } - else - { - Writer.EndArray(); - } - } -} - std::vector<CbFieldView> ReadCompactBinaryStream(MemoryView Data) { @@ -1823,225 +1506,6 @@ ReadCompactBinaryStream(MemoryView Data) ////////////////////////////////////////////////////////////////////////// -class CbJsonReader -{ -public: - static CbFieldIterator Read(std::string_view JsonText, std::string& Error) - { - using namespace json11; - - const Json Json = Json::parse(std::string(JsonText), Error); - - if (Error.empty()) - { - CbWriter Writer; - if (ReadField(Writer, Json, std::string_view(), Error)) - { - return Writer.Save(); - } - } - - return CbFieldIterator(); - } - -private: - static bool ReadField(CbWriter& Writer, const json11::Json& Json, const std::string_view FieldName, std::string& Error) - { - using namespace json11; - - switch (Json.type()) - { - case Json::Type::OBJECT: - { - if (FieldName.empty()) - { - Writer.BeginObject(); - } - else - { - Writer.BeginObject(FieldName); - } - - for (const auto& Kv : Json.object_items()) - { - const std::string& Name = Kv.first; - const json11::Json& Item = Kv.second; - - if (ReadField(Writer, Item, Name, Error) == false) - { - return false; - } - } - - Writer.EndObject(); - } - break; - case Json::Type::ARRAY: - { - if (FieldName.empty()) - { - Writer.BeginArray(); - } - else - { - Writer.BeginArray(FieldName); - } - - for (const json11::Json& Item : Json.array_items()) - { - if (ReadField(Writer, Item, std::string_view(), Error) == false) - { - return false; - } - } - - Writer.EndArray(); - } - break; - case Json::Type::NUL: - { - if (FieldName.empty()) - { - Writer.AddNull(); - } - else - { - Writer.AddNull(FieldName); - } - } - break; - case Json::Type::BOOL: - { - if (FieldName.empty()) - { - Writer.AddBool(Json.bool_value()); - } - else - { - Writer.AddBool(FieldName, Json.bool_value()); - } - } - break; - case Json::Type::NUMBER: - { - if (FieldName.empty()) - { - Writer.AddFloat(Json.number_value()); - } - else - { - Writer.AddFloat(FieldName, Json.number_value()); - } - } - break; - case Json::Type::STRING: - { - Oid Id; - if (TryParseObjectId(Json.string_value(), Id)) - { - if (FieldName.empty()) - { - Writer.AddObjectId(Id); - } - else - { - Writer.AddObjectId(FieldName, Id); - } - - return true; - } - - IoHash Hash; - if (TryParseIoHash(Json.string_value(), Hash)) - { - if (FieldName.empty()) - { - Writer.AddHash(Hash); - } - else - { - Writer.AddHash(FieldName, Hash); - } - - return true; - } - - if (FieldName.empty()) - { - Writer.AddString(Json.string_value()); - } - else - { - Writer.AddString(FieldName, Json.string_value()); - } - } - break; - default: - break; - } - - return true; - } - - static constexpr AsciiSet HexCharSet = AsciiSet("0123456789abcdefABCDEF"); - - static bool TryParseObjectId(std::string_view Str, Oid& Id) - { - using namespace std::literals; - - if (Str.size() == Oid::StringLength && AsciiSet::HasOnly(Str, HexCharSet)) - { - Id = Oid::FromHexString(Str); - return true; - } - - if (Str.starts_with("0x"sv)) - { - return TryParseObjectId(Str.substr(2), Id); - } - - return false; - } - - static bool TryParseIoHash(std::string_view Str, IoHash& Hash) - { - using namespace std::literals; - - if (Str.size() == IoHash::StringLength && AsciiSet::HasOnly(Str, HexCharSet)) - { - Hash = IoHash::FromHexString(Str); - return true; - } - - if (Str.starts_with("0x"sv)) - { - return TryParseIoHash(Str.substr(2), Hash); - } - - return false; - } -}; - -CbFieldIterator -LoadCompactBinaryFromJson(std::string_view Json, std::string& Error) -{ - if (Json.empty() == false) - { - return CbJsonReader::Read(Json, Error); - } - - return CbFieldIterator(); -} - -CbFieldIterator -LoadCompactBinaryFromJson(std::string_view Json) -{ - std::string Error; - return LoadCompactBinaryFromJson(Json, Error); -} - -////////////////////////////////////////////////////////////////////////// - #if ZEN_WITH_TESTS void uson_forcelink() @@ -2211,130 +1675,6 @@ TEST_CASE("uson.null") } } -TEST_CASE("uson.json") -{ - using namespace std::literals; - - SUBCASE("string") - { - CbObjectWriter Writer; - Writer << "KeyOne" - << "ValueOne"; - Writer << "KeyTwo" - << "ValueTwo"; - CbObject Obj = Writer.Save(); - - StringBuilder<128> Sb; - const char* JsonText = Obj.ToJson(Sb).Data(); - - std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText, JsonError); - - const std::string ValueOne = Json["KeyOne"].string_value(); - const std::string ValueTwo = Json["KeyTwo"].string_value(); - - CHECK(JsonError.empty()); - CHECK(ValueOne == "ValueOne"); - CHECK(ValueTwo == "ValueTwo"); - } - - SUBCASE("number") - { - const float ExpectedFloatValue = 21.21f; - const double ExpectedDoubleValue = 42.42; - - CbObjectWriter Writer; - Writer << "Float" << ExpectedFloatValue; - Writer << "Double" << ExpectedDoubleValue; - - CbObject Obj = Writer.Save(); - - StringBuilder<128> Sb; - const char* JsonText = Obj.ToJson(Sb).Data(); - - std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText, JsonError); - - const float FloatValue = float(Json["Float"].number_value()); - const double DoubleValue = Json["Double"].number_value(); - - CHECK(JsonError.empty()); - CHECK(FloatValue == Approx(ExpectedFloatValue)); - CHECK(DoubleValue == Approx(ExpectedDoubleValue)); - } - - SUBCASE("number.nan") - { - constexpr float FloatNan = std::numeric_limits<float>::quiet_NaN(); - constexpr double DoubleNan = std::numeric_limits<double>::quiet_NaN(); - - CbObjectWriter Writer; - Writer << "FloatNan" << FloatNan; - Writer << "DoubleNan" << DoubleNan; - - CbObject Obj = Writer.Save(); - - StringBuilder<128> Sb; - const char* JsonText = Obj.ToJson(Sb).Data(); - - std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText, JsonError); - - const double FloatValue = Json["FloatNan"].number_value(); - const double DoubleValue = Json["DoubleNan"].number_value(); - - CHECK(JsonError.empty()); - CHECK(FloatValue == 0); - CHECK(DoubleValue == 0); - } - - SUBCASE("stream") - { - const auto MakeObject = [&](std::string_view Name, const std::vector<int>& Fields) -> CbObject { - CbWriter Writer; - Writer.SetName(Name); - Writer.BeginObject(); - for (const auto& Field : Fields) - { - Writer.AddInteger(fmt::format("{}", Field), Field); - } - Writer.EndObject(); - return Writer.Save().AsObject(); - }; - - std::vector<uint8_t> Buffer; - - auto AppendToBuffer = [&](const void* Data, size_t Count) { - const uint8_t* AppendBytes = reinterpret_cast<const uint8_t*>(Data); - Buffer.insert(Buffer.end(), AppendBytes, AppendBytes + Count); - }; - - auto Append = [&](const CbFieldView& Field) { - Field.WriteToStream([&](const void* Data, size_t Count) { - const uint8_t* AppendBytes = reinterpret_cast<const uint8_t*>(Data); - Buffer.insert(Buffer.end(), AppendBytes, AppendBytes + Count); - }); - }; - - CbObject DataObjects[] = {MakeObject("Empty object"sv, {}), - MakeObject("OneField object"sv, {5}), - MakeObject("TwoField object"sv, {-5, 999}), - MakeObject("ThreeField object"sv, {1, 2, -129})}; - for (const CbObject& Object : DataObjects) - { - Object.AsField().WriteToStream(AppendToBuffer); - } - - ExtendableStringBuilder<128> Sb; - CompactBinaryToJson(MemoryView(Buffer.data(), Buffer.size()), Sb); - std::string JsonText = Sb.ToString().c_str(); - std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText, JsonError); - std::string ParsedJsonString = Json.dump(); - CHECK(!ParsedJsonString.empty()); - } -} - TEST_CASE("uson.datetime") { using namespace std::literals; @@ -2362,107 +1702,6 @@ TEST_CASE("uson.datetime") } } -TEST_CASE("json.uson") -{ - using namespace std::literals; - using namespace json11; - - SUBCASE("empty") - { - CbFieldIterator It = LoadCompactBinaryFromJson(""sv); - CHECK(It.HasValue() == false); - } - - SUBCASE("object") - { - const Json JsonObject = Json::object{{"Null", nullptr}, - {"String", "Value1"}, - {"Bool", true}, - {"Number", 46.2}, - {"Array", Json::array{1, 2, 3}}, - {"Object", - Json::object{ - {"String", "Value2"}, - }}}; - - CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["Null"].IsNull()); - CHECK(Cb["String"].AsString() == "Value1"sv); - CHECK(Cb["Bool"].AsBool()); - CHECK(Cb["Number"].AsDouble() == 46.2); - CHECK(Cb["Object"].IsObject()); - CbObjectView Object = Cb["Object"].AsObjectView(); - CHECK(Object["String"].AsString() == "Value2"sv); - } - - SUBCASE("array") - { - const Json JsonArray = Json::array{42, 43, 44}; - CbArray Cb = LoadCompactBinaryFromJson(JsonArray.dump()).AsArray(); - - auto It = Cb.CreateIterator(); - CHECK((*It).AsDouble() == 42); - It++; - CHECK((*It).AsDouble() == 43); - It++; - CHECK((*It).AsDouble() == 44); - } - - SUBCASE("objectid") - { - const Oid& Id = Oid::NewOid(); - - StringBuilder<64> Sb; - Id.ToString(Sb); - - Json JsonObject = Json::object{{"value", Sb.ToString()}}; - CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["value"sv].IsObjectId()); - CHECK(Cb["value"sv].AsObjectId() == Id); - - Sb.Reset(); - Sb << "0x"; - Id.ToString(Sb); - - JsonObject = Json::object{{"value", Sb.ToString()}}; - Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["value"sv].IsObjectId()); - CHECK(Cb["value"sv].AsObjectId() == Id); - } - - SUBCASE("iohash") - { - const uint8_t Data[] = { - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - }; - - const IoHash Hash = IoHash::HashBuffer(Data, sizeof(Data)); - - Json JsonObject = Json::object{{"value", Hash.ToHexString()}}; - CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["value"sv].IsHash()); - CHECK(Cb["value"sv].AsHash() == Hash); - - JsonObject = Json::object{{"value", "0x" + Hash.ToHexString()}}; - Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); - - CHECK(Cb["value"sv].IsHash()); - CHECK(Cb["value"sv].AsHash() == Hash); - } -} - ////////////////////////////////////////////////////////////////////////// TEST_SUITE_BEGIN("core.datetime"); |