From 59e51bf811a5907983b84f3e2ed14e47a7210e75 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 26 Mar 2024 13:31:10 +0100 Subject: add yaml serialization support (#3) this change adds serialization of payloads as YAML, but not parsing. The implementation is somewhat based on the JSON path, and may be collapsed eventually as it is possible to serialize JSON format using the same code it also separates out the JSON serialization into a separate file for ease of maintenance any HTTP request response may be formatted as yaml by using a `.yaml` suffix or an `Accept: text/yaml` header --- src/zencore/compactbinaryyaml.cpp | 352 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 src/zencore/compactbinaryyaml.cpp (limited to 'src/zencore/compactbinaryyaml.cpp') diff --git a/src/zencore/compactbinaryyaml.cpp b/src/zencore/compactbinaryyaml.cpp new file mode 100644 index 000000000..3a9705684 --- /dev/null +++ b/src/zencore/compactbinaryyaml.cpp @@ -0,0 +1,352 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencore/compactbinary.h" +#include "zencore/compactbinarybuilder.h" +#include "zencore/compactbinaryvalue.h" + +#include +#include +#include +#include +#include + +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +class CbYamlWriter +{ +public: + explicit CbYamlWriter(StringBuilderBase& InBuilder) : m_StrBuilder(InBuilder) { m_NodeStack.push_back(m_Tree.rootref()); } + + void WriteField(CbFieldView Field) + { + ryml::NodeRef Node; + + if (m_IsFirst) + { + Node = Top(); + + m_IsFirst = false; + } + else + { + Node = Top().append_child(); + } + + if (std::u8string_view Name = Field.GetU8Name(); !Name.empty()) + { + Node.set_key_serialized(ryml::csubstr((const char*)Name.data(), Name.size())); + } + + switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + { + case CbFieldType::Null: + Node.set_val("null"); + break; + case CbFieldType::Object: + case CbFieldType::UniformObject: + Node |= ryml::MAP; + m_NodeStack.push_back(Node); + for (CbFieldView It : Field) + { + WriteField(It); + } + m_NodeStack.pop_back(); + break; + case CbFieldType::Array: + case CbFieldType::UniformArray: + Node |= ryml::SEQ; + m_NodeStack.push_back(Node); + for (CbFieldView It : Field) + { + WriteField(It); + } + m_NodeStack.pop_back(); + break; + case CbFieldType::Binary: + { + ExtendableStringBuilder<256> Builder; + const MemoryView Value = Accessor.AsBinary(); + ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); + const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); + const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); + Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + + Node.set_key_serialized(Builder.c_str()); + } + break; + case CbFieldType::String: + { + const std::u8string_view U8String = Accessor.AsU8String(); + Node.set_val(ryml::csubstr((const char*)U8String.data(), U8String.size())); + } + break; + case CbFieldType::IntegerPositive: + Node << Accessor.AsIntegerPositive(); + break; + case CbFieldType::IntegerNegative: + Node << Accessor.AsIntegerNegative(); + break; + case CbFieldType::Float32: + if (const float Value = Accessor.AsFloat32(); std::isfinite(Value)) + { + Node << Value; + } + else + { + Node << "null"; + } + break; + case CbFieldType::Float64: + if (const double Value = Accessor.AsFloat64(); std::isfinite(Value)) + { + Node << Value; + } + else + { + Node << "null"; + } + break; + case CbFieldType::BoolFalse: + Node << "false"; + break; + case CbFieldType::BoolTrue: + Node << "true"; + break; + case CbFieldType::ObjectAttachment: + case CbFieldType::BinaryAttachment: + Node << Accessor.AsAttachment().ToHexString(); + break; + case CbFieldType::Hash: + Node << Accessor.AsHash().ToHexString(); + break; + case CbFieldType::Uuid: + Node << fmt::format("{}", Accessor.AsUuid()); + break; + case CbFieldType::DateTime: + Node << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); + break; + case CbFieldType::TimeSpan: + if (const TimeSpan Span(Accessor.AsTimeSpanTicks()); Span.GetDays() == 0) + { + Node << Span.ToString("%h:%m:%s.%n"); + } + else + { + Node << Span.ToString("%d.%h:%m:%s.%n"); + } + break; + case CbFieldType::ObjectId: + Node << fmt::format("{}", Accessor.AsObjectId()); + break; + case CbFieldType::CustomById: + { + CbCustomById Custom = Accessor.AsCustomById(); + + Node |= ryml::MAP; + + ryml::NodeRef IdNode = Node.append_child(); + IdNode.set_key("Id"); + IdNode.set_val_serialized(fmt::format("{}", Custom.Id)); + + ryml::NodeRef DataNode = Node.append_child(); + DataNode.set_key("Data"); + + ExtendableStringBuilder<256> Builder; + const MemoryView& Value = Custom.Data; + const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); + const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); + Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + + DataNode.set_val_serialized(Builder.c_str()); + } + break; + case CbFieldType::CustomByName: + { + CbCustomByName Custom = Accessor.AsCustomByName(); + + Node |= ryml::MAP; + + ryml::NodeRef NameNode = Node.append_child(); + NameNode.set_key("Name"); + std::string_view Name = std::string_view((const char*)Custom.Name.data(), Custom.Name.size()); + NameNode.set_val_serialized(std::string(Name)); + + ryml::NodeRef DataNode = Node.append_child(); + DataNode.set_key("Data"); + + ExtendableStringBuilder<256> Builder; + const MemoryView& Value = Custom.Data; + const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); + const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); + Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + + DataNode.set_val_serialized(Builder.c_str()); + } + break; + default: + ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType())); + break; + } + + if (m_NodeStack.size() == 1) + { + std::string Yaml = ryml::emitrs_yaml(m_Tree); + m_StrBuilder << Yaml; + } + } + +private: + StringBuilderBase& m_StrBuilder; + bool m_IsFirst = true; + + ryml::Tree m_Tree; + std::vector m_NodeStack; + ryml::NodeRef& Top() { return m_NodeStack.back(); } +}; + +void +CompactBinaryToYaml(const CbObjectView& Object, StringBuilderBase& Builder) +{ + CbYamlWriter Writer(Builder); + Writer.WriteField(Object.AsFieldView()); +} + +void +CompactBinaryToYaml(const CbArrayView& Array, StringBuilderBase& Builder) +{ + CbYamlWriter Writer(Builder); + Writer.WriteField(Array.AsFieldView()); +} + +#if ZEN_WITH_TESTS +void +cbyaml_forcelink() +{ +} + +TEST_CASE("uson.yaml") +{ + using namespace std::literals; + + SUBCASE("simple") + { + CbObjectWriter Writer; + Writer << "KeyOne" + << "ValueOne"; + Writer << "KeyTwo" + << "ValueTwo"; + CbObject Obj = Writer.Save(); + + StringBuilder<128> Sb; + CbYamlWriter YamlWriter(Sb); + YamlWriter.WriteField(Obj.AsFieldView()); + + CHECK_EQ(Sb.ToView(), "KeyOne: ValueOne\nKeyTwo: ValueTwo\n"sv); + } + + SUBCASE("scalar_fields") + { + CbObjectWriter Writer; + Writer << "small_int"sv << 10; + Writer << "neg_small_int"sv << -10; + Writer << "small_real"sv << 10.5f; + Writer << "neg_small_real"sv << -10.5f; + Writer.AddNull("null_val"sv); + Writer.AddDateTimeTicks("date"sv, 622'033'000'000'000'000ull); + Writer.AddHash("hash"sv, IoHash::FromHexString("0011223344556677889900112233445566778899"sv)); + Writer.AddObjectId("oid"sv, Oid::FromHexString("112233445566778899001122"sv)); + Writer.AddTimeSpanTicks("dt"sv, 3'000'000'000'000ull); + Writer.AddUuid("guid"sv, Guid::FromString("E0596ADC-996A-4BA4-ACA3-A2A378AB2796")); + Writer.AddBool("yes"sv, true); + Writer.AddBool("no"sv, false); + CbObject Obj = Writer.Save(); + + ExtendableStringBuilder<128> Sb; + CbYamlWriter YamlWriter(Sb); + YamlWriter.WriteField(Obj.AsFieldView()); + + CHECK_EQ(Sb.ToView(), + "small_int: 10\n" + "neg_small_int: -10\n" + "small_real: 10.5\n" + "neg_small_real: -10.5\n" + "null_val: null\n" + "date: '1972-02-23T14:26:40.000Z'\n" + "hash: 0011223344556677889900112233445566778899\n" + "oid: 112233445566778899001122\n" + "dt: '+3.11:20:00.000000000'\n" + "guid: 'e0596adc-996a-4ba4-aca3-a2a378ab2796'\n" + "yes: true\n" + "no: false\n"sv); + } + + SUBCASE("complex_fields") + { + CbObjectWriter Writer; + Writer.BeginObject("sub"); + Writer.AddBool("no"sv, false); + Writer.BeginObject("sub"); + Writer.AddBool("yes"sv, true); + Writer.BeginArray("seq"); + Writer.AddInteger(1); + Writer.AddInteger(2); + Writer.AddInteger(3); + Writer.EndArray(); + Writer.EndObject(); + Writer.EndObject(); + Writer.BeginArray("seq"); + Writer.AddInteger(1); + Writer.AddInteger(2); + Writer.AddInteger(3); + Writer.EndArray(); + Writer.BeginArray("mixed_seq"); + Writer.AddInteger(1); + Writer.AddString("hello"sv); + Writer.AddFloat(44.4f); + Writer.BeginObject(); + Writer.AddBool("yes"sv, true); + Writer.AddBool("no"sv, false); + Writer.EndObject(); + Writer.EndArray(); + CbObject Obj = Writer.Save(); + + ExtendableStringBuilder<128> Sb; + CbYamlWriter YamlWriter(Sb); + YamlWriter.WriteField(Obj.AsFieldView()); + + CHECK_EQ(Sb.ToView(), + R"(sub: + no: false + sub: + yes: true + seq: + - 1 + - 2 + - 3 +seq: + - 1 + - 2 + - 3 +mixed_seq: + - 1 + - hello + - 44.4 + - yes: true + no: false +)"sv); + } +} +#endif + +} // namespace zen -- cgit v1.2.3