From 59e51bf811a5907983b84f3e2ed14e47a7210e75 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 26 Mar 2024 13:31:10 +0100 Subject: add yaml serialization support (#3) this change adds serialization of payloads as YAML, but not parsing. The implementation is somewhat based on the JSON path, and may be collapsed eventually as it is possible to serialize JSON format using the same code it also separates out the JSON serialization into a separate file for ease of maintenance any HTTP request response may be formatted as yaml by using a `.yaml` suffix or an `Accept: text/yaml` header --- src/zencore/compactbinaryjson.cpp | 805 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 805 insertions(+) create mode 100644 src/zencore/compactbinaryjson.cpp (limited to 'src/zencore/compactbinaryjson.cpp') diff --git a/src/zencore/compactbinaryjson.cpp b/src/zencore/compactbinaryjson.cpp new file mode 100644 index 000000000..58a91c731 --- /dev/null +++ b/src/zencore/compactbinaryjson.cpp @@ -0,0 +1,805 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencore/compactbinary.h" +#include "zencore/compactbinarybuilder.h" +#include "zencore/compactbinaryvalue.h" + +#include +#include +#include +#include +#include + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +class CbJsonWriter +{ +public: + explicit CbJsonWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) { NewLineAndIndent << LINE_TERMINATOR_ANSI; } + + void BeginObject() + { + Builder << '{'; + NewLineAndIndent << '\t'; + NeedsNewLine = true; + } + + void EndObject() + { + NewLineAndIndent.RemoveSuffix(1); + if (NeedsComma) + { + WriteOptionalNewLine(); + } + Builder << '}'; + } + + void BeginArray() + { + Builder << '['; + NewLineAndIndent << '\t'; + NeedsNewLine = true; + } + + void EndArray() + { + NewLineAndIndent.RemoveSuffix(1); + if (NeedsComma) + { + WriteOptionalNewLine(); + } + Builder << ']'; + } + + void WriteField(CbFieldView Field) + { + using namespace std::literals; + + WriteOptionalComma(); + WriteOptionalNewLine(); + + if (std::u8string_view Name = Field.GetU8Name(); !Name.empty()) + { + AppendQuotedString(Name); + Builder << ": "sv; + } + + switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + { + case CbFieldType::Null: + Builder << "null"sv; + break; + case CbFieldType::Object: + case CbFieldType::UniformObject: + { + BeginObject(); + for (CbFieldView It : Field) + { + WriteField(It); + } + EndObject(); + } + break; + case CbFieldType::Array: + case CbFieldType::UniformArray: + { + BeginArray(); + for (CbFieldView It : Field) + { + WriteField(It); + } + EndArray(); + } + break; + case CbFieldType::Binary: + AppendBase64String(Accessor.AsBinary()); + break; + case CbFieldType::String: + AppendQuotedString(Accessor.AsU8String()); + break; + case CbFieldType::IntegerPositive: + Builder << Accessor.AsIntegerPositive(); + break; + case CbFieldType::IntegerNegative: + Builder << Accessor.AsIntegerNegative(); + break; + case CbFieldType::Float32: + { + const float Value = Accessor.AsFloat32(); + if (std::isfinite(Value)) + { + Builder.Append(fmt::format("{:.9g}", Value)); + } + else + { + Builder << "null"sv; + } + } + break; + case CbFieldType::Float64: + { + const double Value = Accessor.AsFloat64(); + if (std::isfinite(Value)) + { + Builder.Append(fmt::format("{:.17g}", Value)); + } + else + { + Builder << "null"sv; + } + } + break; + case CbFieldType::BoolFalse: + Builder << "false"sv; + break; + case CbFieldType::BoolTrue: + Builder << "true"sv; + break; + case CbFieldType::ObjectAttachment: + case CbFieldType::BinaryAttachment: + { + Builder << '"'; + Accessor.AsAttachment().ToHexString(Builder); + Builder << '"'; + } + break; + case CbFieldType::Hash: + { + Builder << '"'; + Accessor.AsHash().ToHexString(Builder); + Builder << '"'; + } + break; + case CbFieldType::Uuid: + { + Builder << '"'; + Accessor.AsUuid().ToString(Builder); + Builder << '"'; + } + break; + case CbFieldType::DateTime: + Builder << '"' << DateTime(Accessor.AsDateTimeTicks()).ToIso8601() << '"'; + break; + case CbFieldType::TimeSpan: + { + const TimeSpan Span(Accessor.AsTimeSpanTicks()); + if (Span.GetDays() == 0) + { + Builder << '"' << Span.ToString("%h:%m:%s.%n") << '"'; + } + else + { + Builder << '"' << Span.ToString("%d.%h:%m:%s.%n") << '"'; + } + break; + } + case CbFieldType::ObjectId: + Builder << '"'; + Accessor.AsObjectId().ToString(Builder); + Builder << '"'; + break; + case CbFieldType::CustomById: + { + CbCustomById Custom = Accessor.AsCustomById(); + Builder << "{ \"Id\": "; + Builder << Custom.Id; + Builder << ", \"Data\": "; + AppendBase64String(Custom.Data); + Builder << " }"; + break; + } + case CbFieldType::CustomByName: + { + CbCustomByName Custom = Accessor.AsCustomByName(); + Builder << "{ \"Name\": "; + AppendQuotedString(Custom.Name); + Builder << ", \"Data\": "; + AppendBase64String(Custom.Data); + Builder << " }"; + break; + } + default: + ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType())); + break; + } + + NeedsComma = true; + NeedsNewLine = true; + } + +private: + void WriteOptionalComma() + { + if (NeedsComma) + { + NeedsComma = false; + Builder << ','; + } + } + + void WriteOptionalNewLine() + { + if (NeedsNewLine) + { + NeedsNewLine = false; + Builder << NewLineAndIndent; + } + } + + void AppendQuotedString(std::u8string_view Value) + { + using namespace std::literals; + + const AsciiSet EscapeSet( + "\\\"\b\f\n\r\t" + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"); + + Builder << '\"'; + while (!Value.empty()) + { + std::u8string_view Verbatim = AsciiSet::FindPrefixWithout(Value, EscapeSet); + Builder << Verbatim; + + Value = Value.substr(Verbatim.size()); + + std::u8string_view Escape = AsciiSet::FindPrefixWith(Value, EscapeSet); + for (char Char : Escape) + { + switch (Char) + { + case '\\': + Builder << "\\\\"sv; + break; + case '\"': + Builder << "\\\""sv; + break; + case '\b': + Builder << "\\b"sv; + break; + case '\f': + Builder << "\\f"sv; + break; + case '\n': + Builder << "\\n"sv; + break; + case '\r': + Builder << "\\r"sv; + break; + case '\t': + Builder << "\\t"sv; + break; + default: + Builder << Char; + break; + } + } + Value = Value.substr(Escape.size()); + } + Builder << '\"'; + } + + void AppendBase64String(MemoryView Value) + { + Builder << '"'; + ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); + const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); + const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); + Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + } + +private: + StringBuilderBase& Builder; + ExtendableStringBuilder<32> NewLineAndIndent; + bool NeedsComma{false}; + bool NeedsNewLine{false}; +}; + +void +CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder) +{ + CbJsonWriter Writer(Builder); + Writer.WriteField(Object.AsFieldView()); +} + +void +CompactBinaryToJson(const CbArrayView& Array, StringBuilderBase& Builder) +{ + CbJsonWriter Writer(Builder); + Writer.WriteField(Array.AsFieldView()); +} + +void +CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder) +{ + std::vector Fields = ReadCompactBinaryStream(Data); + CbJsonWriter Writer(InBuilder); + if (!Fields.empty()) + { + if (Fields.size() == 1) + { + Writer.WriteField(Fields[0]); + return; + } + bool UseTopLevelObject = Fields[0].HasName(); + if (UseTopLevelObject) + { + Writer.BeginObject(); + } + else + { + Writer.BeginArray(); + } + for (const CbFieldView& Field : Fields) + { + Writer.WriteField(Field); + } + if (UseTopLevelObject) + { + Writer.EndObject(); + } + else + { + Writer.EndArray(); + } + } +} + +class CbJsonReader +{ +public: + static CbFieldIterator Read(std::string_view JsonText, std::string& Error) + { + using namespace json11; + + const Json Json = Json::parse(std::string(JsonText), Error); + + if (Error.empty()) + { + CbWriter Writer; + if (ReadField(Writer, Json, std::string_view(), Error)) + { + return Writer.Save(); + } + } + + return CbFieldIterator(); + } + +private: + static bool ReadField(CbWriter& Writer, const json11::Json& Json, const std::string_view FieldName, std::string& Error) + { + using namespace json11; + + switch (Json.type()) + { + case Json::Type::OBJECT: + { + if (FieldName.empty()) + { + Writer.BeginObject(); + } + else + { + Writer.BeginObject(FieldName); + } + + for (const auto& Kv : Json.object_items()) + { + const std::string& Name = Kv.first; + const json11::Json& Item = Kv.second; + + if (ReadField(Writer, Item, Name, Error) == false) + { + return false; + } + } + + Writer.EndObject(); + } + break; + case Json::Type::ARRAY: + { + if (FieldName.empty()) + { + Writer.BeginArray(); + } + else + { + Writer.BeginArray(FieldName); + } + + for (const json11::Json& Item : Json.array_items()) + { + if (ReadField(Writer, Item, std::string_view(), Error) == false) + { + return false; + } + } + + Writer.EndArray(); + } + break; + case Json::Type::NUL: + { + if (FieldName.empty()) + { + Writer.AddNull(); + } + else + { + Writer.AddNull(FieldName); + } + } + break; + case Json::Type::BOOL: + { + if (FieldName.empty()) + { + Writer.AddBool(Json.bool_value()); + } + else + { + Writer.AddBool(FieldName, Json.bool_value()); + } + } + break; + case Json::Type::NUMBER: + { + if (FieldName.empty()) + { + Writer.AddFloat(Json.number_value()); + } + else + { + Writer.AddFloat(FieldName, Json.number_value()); + } + } + break; + case Json::Type::STRING: + { + Oid Id; + if (TryParseObjectId(Json.string_value(), Id)) + { + if (FieldName.empty()) + { + Writer.AddObjectId(Id); + } + else + { + Writer.AddObjectId(FieldName, Id); + } + + return true; + } + + IoHash Hash; + if (TryParseIoHash(Json.string_value(), Hash)) + { + if (FieldName.empty()) + { + Writer.AddHash(Hash); + } + else + { + Writer.AddHash(FieldName, Hash); + } + + return true; + } + + if (FieldName.empty()) + { + Writer.AddString(Json.string_value()); + } + else + { + Writer.AddString(FieldName, Json.string_value()); + } + } + break; + default: + break; + } + + return true; + } + + static constexpr AsciiSet HexCharSet = AsciiSet("0123456789abcdefABCDEF"); + + static bool TryParseObjectId(std::string_view Str, Oid& Id) + { + using namespace std::literals; + + if (Str.size() == Oid::StringLength && AsciiSet::HasOnly(Str, HexCharSet)) + { + Id = Oid::FromHexString(Str); + return true; + } + + if (Str.starts_with("0x"sv)) + { + return TryParseObjectId(Str.substr(2), Id); + } + + return false; + } + + static bool TryParseIoHash(std::string_view Str, IoHash& Hash) + { + using namespace std::literals; + + if (Str.size() == IoHash::StringLength && AsciiSet::HasOnly(Str, HexCharSet)) + { + Hash = IoHash::FromHexString(Str); + return true; + } + + if (Str.starts_with("0x"sv)) + { + return TryParseIoHash(Str.substr(2), Hash); + } + + return false; + } +}; + +CbFieldIterator +LoadCompactBinaryFromJson(std::string_view Json, std::string& Error) +{ + if (Json.empty() == false) + { + return CbJsonReader::Read(Json, Error); + } + + return CbFieldIterator(); +} + +CbFieldIterator +LoadCompactBinaryFromJson(std::string_view Json) +{ + std::string Error; + return LoadCompactBinaryFromJson(Json, Error); +} + +#if ZEN_WITH_TESTS +void +cbjson_forcelink() +{ +} + +TEST_CASE("uson.json") +{ + using namespace std::literals; + + SUBCASE("string") + { + CbObjectWriter Writer; + Writer << "KeyOne" + << "ValueOne"; + Writer << "KeyTwo" + << "ValueTwo"; + CbObject Obj = Writer.Save(); + + StringBuilder<128> Sb; + const char* JsonText = Obj.ToJson(Sb).Data(); + + std::string JsonError; + json11::Json Json = json11::Json::parse(JsonText, JsonError); + + const std::string ValueOne = Json["KeyOne"].string_value(); + const std::string ValueTwo = Json["KeyTwo"].string_value(); + + CHECK(JsonError.empty()); + CHECK(ValueOne == "ValueOne"); + CHECK(ValueTwo == "ValueTwo"); + } + + SUBCASE("number") + { + const float ExpectedFloatValue = 21.21f; + const double ExpectedDoubleValue = 42.42; + + CbObjectWriter Writer; + Writer << "Float" << ExpectedFloatValue; + Writer << "Double" << ExpectedDoubleValue; + + CbObject Obj = Writer.Save(); + + StringBuilder<128> Sb; + const char* JsonText = Obj.ToJson(Sb).Data(); + + std::string JsonError; + json11::Json Json = json11::Json::parse(JsonText, JsonError); + + const float FloatValue = float(Json["Float"].number_value()); + const double DoubleValue = Json["Double"].number_value(); + + CHECK(JsonError.empty()); + CHECK(FloatValue == Approx(ExpectedFloatValue)); + CHECK(DoubleValue == Approx(ExpectedDoubleValue)); + } + + SUBCASE("number.nan") + { + constexpr float FloatNan = std::numeric_limits::quiet_NaN(); + constexpr double DoubleNan = std::numeric_limits::quiet_NaN(); + + CbObjectWriter Writer; + Writer << "FloatNan" << FloatNan; + Writer << "DoubleNan" << DoubleNan; + + CbObject Obj = Writer.Save(); + + StringBuilder<128> Sb; + const char* JsonText = Obj.ToJson(Sb).Data(); + + std::string JsonError; + json11::Json Json = json11::Json::parse(JsonText, JsonError); + + const double FloatValue = Json["FloatNan"].number_value(); + const double DoubleValue = Json["DoubleNan"].number_value(); + + CHECK(JsonError.empty()); + CHECK(FloatValue == 0); + CHECK(DoubleValue == 0); + } + + SUBCASE("stream") + { + const auto MakeObject = [&](std::string_view Name, const std::vector& Fields) -> CbObject { + CbWriter Writer; + Writer.SetName(Name); + Writer.BeginObject(); + for (const auto& Field : Fields) + { + Writer.AddInteger(fmt::format("{}", Field), Field); + } + Writer.EndObject(); + return Writer.Save().AsObject(); + }; + + std::vector Buffer; + + auto AppendToBuffer = [&](const void* Data, size_t Count) { + const uint8_t* AppendBytes = reinterpret_cast(Data); + Buffer.insert(Buffer.end(), AppendBytes, AppendBytes + Count); + }; + + auto Append = [&](const CbFieldView& Field) { + Field.WriteToStream([&](const void* Data, size_t Count) { + const uint8_t* AppendBytes = reinterpret_cast(Data); + Buffer.insert(Buffer.end(), AppendBytes, AppendBytes + Count); + }); + }; + + CbObject DataObjects[] = {MakeObject("Empty object"sv, {}), + MakeObject("OneField object"sv, {5}), + MakeObject("TwoField object"sv, {-5, 999}), + MakeObject("ThreeField object"sv, {1, 2, -129})}; + for (const CbObject& Object : DataObjects) + { + Object.AsField().WriteToStream(AppendToBuffer); + } + + ExtendableStringBuilder<128> Sb; + CompactBinaryToJson(MemoryView(Buffer.data(), Buffer.size()), Sb); + std::string JsonText = Sb.ToString().c_str(); + std::string JsonError; + json11::Json Json = json11::Json::parse(JsonText, JsonError); + std::string ParsedJsonString = Json.dump(); + CHECK(!ParsedJsonString.empty()); + } +} + +TEST_CASE("json.uson") +{ + using namespace std::literals; + using namespace json11; + + SUBCASE("empty") + { + CbFieldIterator It = LoadCompactBinaryFromJson(""sv); + CHECK(It.HasValue() == false); + } + + SUBCASE("object") + { + const Json JsonObject = Json::object{{"Null", nullptr}, + {"String", "Value1"}, + {"Bool", true}, + {"Number", 46.2}, + {"Array", Json::array{1, 2, 3}}, + {"Object", + Json::object{ + {"String", "Value2"}, + }}}; + + CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); + + CHECK(Cb["Null"].IsNull()); + CHECK(Cb["String"].AsString() == "Value1"sv); + CHECK(Cb["Bool"].AsBool()); + CHECK(Cb["Number"].AsDouble() == 46.2); + CHECK(Cb["Object"].IsObject()); + CbObjectView Object = Cb["Object"].AsObjectView(); + CHECK(Object["String"].AsString() == "Value2"sv); + } + + SUBCASE("array") + { + const Json JsonArray = Json::array{42, 43, 44}; + CbArray Cb = LoadCompactBinaryFromJson(JsonArray.dump()).AsArray(); + + auto It = Cb.CreateIterator(); + CHECK((*It).AsDouble() == 42); + It++; + CHECK((*It).AsDouble() == 43); + It++; + CHECK((*It).AsDouble() == 44); + } + + SUBCASE("objectid") + { + const Oid& Id = Oid::NewOid(); + + StringBuilder<64> Sb; + Id.ToString(Sb); + + Json JsonObject = Json::object{{"value", Sb.ToString()}}; + CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); + + CHECK(Cb["value"sv].IsObjectId()); + CHECK(Cb["value"sv].AsObjectId() == Id); + + Sb.Reset(); + Sb << "0x"; + Id.ToString(Sb); + + JsonObject = Json::object{{"value", Sb.ToString()}}; + Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); + + CHECK(Cb["value"sv].IsObjectId()); + CHECK(Cb["value"sv].AsObjectId() == Id); + } + + SUBCASE("iohash") + { + const uint8_t Data[] = { + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + }; + + const IoHash Hash = IoHash::HashBuffer(Data, sizeof(Data)); + + Json JsonObject = Json::object{{"value", Hash.ToHexString()}}; + CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); + + CHECK(Cb["value"sv].IsHash()); + CHECK(Cb["value"sv].AsHash() == Hash); + + JsonObject = Json::object{{"value", "0x" + Hash.ToHexString()}}; + Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); + + CHECK(Cb["value"sv].IsHash()); + CHECK(Cb["value"sv].AsHash() == Hash); + } +} + +#endif // ZEN_WITH_TESTS + +} // namespace zen -- cgit v1.2.3