add yaml serialization support (#3)

this change adds serialization of payloads as YAML, but not parsing. The implementation is somewhat based on the JSON path, and may be collapsed eventually as it is possible to serialize JSON format using the same code

it also separates out the JSON serialization into a separate file for ease of maintenance

any HTTP request response may be formatted as yaml by using a `.yaml` suffix or an `Accept: text/yaml` header

1 files changed, 352 insertions, 0 deletions

diff --git a/src/zencore/compactbinaryyaml.cpp b/src/zencore/compactbinaryyaml.cpp
new file mode 100644
index 000000000..3a9705684
--- /dev/null
+++ b/src/zencore/compactbinaryyaml.cpp
@@ -0,0 +1,352 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zencore/compactbinary.h"
+#include "zencore/compactbinarybuilder.h"
+#include "zencore/compactbinaryvalue.h"
+
+#include <zencore/assertfmt.h>
+#include <zencore/base64.h>
+#include <zencore/fmtutils.h>
+#include <zencore/string.h>
+#include <zencore/testing.h>
+
+#include <fmt/format.h>
+#include <string_view>
+#include <vector>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <ryml/ryml.hpp>
+#include <ryml/ryml_std.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+//////////////////////////////////////////////////////////////////////////
+
+class CbYamlWriter
+{
+public:
+	explicit CbYamlWriter(StringBuilderBase& InBuilder) : m_StrBuilder(InBuilder) { m_NodeStack.push_back(m_Tree.rootref()); }
+
+	void WriteField(CbFieldView Field)
+	{
+		ryml::NodeRef Node;
+
+		if (m_IsFirst)
+		{
+			Node = Top();
+
+			m_IsFirst = false;
+		}
+		else
+		{
+			Node = Top().append_child();
+		}
+
+		if (std::u8string_view Name = Field.GetU8Name(); !Name.empty())
+		{
+			Node.set_key_serialized(ryml::csubstr((const char*)Name.data(), Name.size()));
+		}
+
+		switch (CbValue Accessor = Field.GetValue(); Accessor.GetType())
+		{
+			case CbFieldType::Null:
+				Node.set_val("null");
+				break;
+			case CbFieldType::Object:
+			case CbFieldType::UniformObject:
+				Node |= ryml::MAP;
+				m_NodeStack.push_back(Node);
+				for (CbFieldView It : Field)
+				{
+					WriteField(It);
+				}
+				m_NodeStack.pop_back();
+				break;
+			case CbFieldType::Array:
+			case CbFieldType::UniformArray:
+				Node |= ryml::SEQ;
+				m_NodeStack.push_back(Node);
+				for (CbFieldView It : Field)
+				{
+					WriteField(It);
+				}
+				m_NodeStack.pop_back();
+				break;
+			case CbFieldType::Binary:
+				{
+					ExtendableStringBuilder<256> Builder;
+					const MemoryView			 Value = Accessor.AsBinary();
+					ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024);
+					const uint32_t EncodedSize	= Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
+					const size_t   EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize));
+					Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex);
+
+					Node.set_key_serialized(Builder.c_str());
+				}
+				break;
+			case CbFieldType::String:
+				{
+					const std::u8string_view U8String = Accessor.AsU8String();
+					Node.set_val(ryml::csubstr((const char*)U8String.data(), U8String.size()));
+				}
+				break;
+			case CbFieldType::IntegerPositive:
+				Node << Accessor.AsIntegerPositive();
+				break;
+			case CbFieldType::IntegerNegative:
+				Node << Accessor.AsIntegerNegative();
+				break;
+			case CbFieldType::Float32:
+				if (const float Value = Accessor.AsFloat32(); std::isfinite(Value))
+				{
+					Node << Value;
+				}
+				else
+				{
+					Node << "null";
+				}
+				break;
+			case CbFieldType::Float64:
+				if (const double Value = Accessor.AsFloat64(); std::isfinite(Value))
+				{
+					Node << Value;
+				}
+				else
+				{
+					Node << "null";
+				}
+				break;
+			case CbFieldType::BoolFalse:
+				Node << "false";
+				break;
+			case CbFieldType::BoolTrue:
+				Node << "true";
+				break;
+			case CbFieldType::ObjectAttachment:
+			case CbFieldType::BinaryAttachment:
+				Node << Accessor.AsAttachment().ToHexString();
+				break;
+			case CbFieldType::Hash:
+				Node << Accessor.AsHash().ToHexString();
+				break;
+			case CbFieldType::Uuid:
+				Node << fmt::format("{}", Accessor.AsUuid());
+				break;
+			case CbFieldType::DateTime:
+				Node << DateTime(Accessor.AsDateTimeTicks()).ToIso8601();
+				break;
+			case CbFieldType::TimeSpan:
+				if (const TimeSpan Span(Accessor.AsTimeSpanTicks()); Span.GetDays() == 0)
+				{
+					Node << Span.ToString("%h:%m:%s.%n");
+				}
+				else
+				{
+					Node << Span.ToString("%d.%h:%m:%s.%n");
+				}
+				break;
+			case CbFieldType::ObjectId:
+				Node << fmt::format("{}", Accessor.AsObjectId());
+				break;
+			case CbFieldType::CustomById:
+				{
+					CbCustomById Custom = Accessor.AsCustomById();
+
+					Node |= ryml::MAP;
+
+					ryml::NodeRef IdNode = Node.append_child();
+					IdNode.set_key("Id");
+					IdNode.set_val_serialized(fmt::format("{}", Custom.Id));
+
+					ryml::NodeRef DataNode = Node.append_child();
+					DataNode.set_key("Data");
+
+					ExtendableStringBuilder<256> Builder;
+					const MemoryView&			 Value		  = Custom.Data;
+					const uint32_t				 EncodedSize  = Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
+					const size_t				 EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize));
+					Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex);
+
+					DataNode.set_val_serialized(Builder.c_str());
+				}
+				break;
+			case CbFieldType::CustomByName:
+				{
+					CbCustomByName Custom = Accessor.AsCustomByName();
+
+					Node |= ryml::MAP;
+
+					ryml::NodeRef NameNode = Node.append_child();
+					NameNode.set_key("Name");
+					std::string_view Name = std::string_view((const char*)Custom.Name.data(), Custom.Name.size());
+					NameNode.set_val_serialized(std::string(Name));
+
+					ryml::NodeRef DataNode = Node.append_child();
+					DataNode.set_key("Data");
+
+					ExtendableStringBuilder<256> Builder;
+					const MemoryView&			 Value		  = Custom.Data;
+					const uint32_t				 EncodedSize  = Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
+					const size_t				 EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize));
+					Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex);
+
+					DataNode.set_val_serialized(Builder.c_str());
+				}
+				break;
+			default:
+				ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType()));
+				break;
+		}
+
+		if (m_NodeStack.size() == 1)
+		{
+			std::string Yaml = ryml::emitrs_yaml<std::string>(m_Tree);
+			m_StrBuilder << Yaml;
+		}
+	}
+
+private:
+	StringBuilderBase& m_StrBuilder;
+	bool			   m_IsFirst = true;
+
+	ryml::Tree				   m_Tree;
+	std::vector<ryml::NodeRef> m_NodeStack;
+	ryml::NodeRef&			   Top() { return m_NodeStack.back(); }
+};
+
+void
+CompactBinaryToYaml(const CbObjectView& Object, StringBuilderBase& Builder)
+{
+	CbYamlWriter Writer(Builder);
+	Writer.WriteField(Object.AsFieldView());
+}
+
+void
+CompactBinaryToYaml(const CbArrayView& Array, StringBuilderBase& Builder)
+{
+	CbYamlWriter Writer(Builder);
+	Writer.WriteField(Array.AsFieldView());
+}
+
+#if ZEN_WITH_TESTS
+void
+cbyaml_forcelink()
+{
+}
+
+TEST_CASE("uson.yaml")
+{
+	using namespace std::literals;
+
+	SUBCASE("simple")
+	{
+		CbObjectWriter Writer;
+		Writer << "KeyOne"
+			   << "ValueOne";
+		Writer << "KeyTwo"
+			   << "ValueTwo";
+		CbObject Obj = Writer.Save();
+
+		StringBuilder<128> Sb;
+		CbYamlWriter	   YamlWriter(Sb);
+		YamlWriter.WriteField(Obj.AsFieldView());
+
+		CHECK_EQ(Sb.ToView(), "KeyOne: ValueOne\nKeyTwo: ValueTwo\n"sv);
+	}
+
+	SUBCASE("scalar_fields")
+	{
+		CbObjectWriter Writer;
+		Writer << "small_int"sv << 10;
+		Writer << "neg_small_int"sv << -10;
+		Writer << "small_real"sv << 10.5f;
+		Writer << "neg_small_real"sv << -10.5f;
+		Writer.AddNull("null_val"sv);
+		Writer.AddDateTimeTicks("date"sv, 622'033'000'000'000'000ull);
+		Writer.AddHash("hash"sv, IoHash::FromHexString("0011223344556677889900112233445566778899"sv));
+		Writer.AddObjectId("oid"sv, Oid::FromHexString("112233445566778899001122"sv));
+		Writer.AddTimeSpanTicks("dt"sv, 3'000'000'000'000ull);
+		Writer.AddUuid("guid"sv, Guid::FromString("E0596ADC-996A-4BA4-ACA3-A2A378AB2796"));
+		Writer.AddBool("yes"sv, true);
+		Writer.AddBool("no"sv, false);
+		CbObject Obj = Writer.Save();
+
+		ExtendableStringBuilder<128> Sb;
+		CbYamlWriter				 YamlWriter(Sb);
+		YamlWriter.WriteField(Obj.AsFieldView());
+
+		CHECK_EQ(Sb.ToView(),
+				 "small_int: 10\n"
+				 "neg_small_int: -10\n"
+				 "small_real: 10.5\n"
+				 "neg_small_real: -10.5\n"
+				 "null_val: null\n"
+				 "date: '1972-02-23T14:26:40.000Z'\n"
+				 "hash: 0011223344556677889900112233445566778899\n"
+				 "oid: 112233445566778899001122\n"
+				 "dt: '+3.11:20:00.000000000'\n"
+				 "guid: 'e0596adc-996a-4ba4-aca3-a2a378ab2796'\n"
+				 "yes: true\n"
+				 "no: false\n"sv);
+	}
+
+	SUBCASE("complex_fields")
+	{
+		CbObjectWriter Writer;
+		Writer.BeginObject("sub");
+		Writer.AddBool("no"sv, false);
+		Writer.BeginObject("sub");
+		Writer.AddBool("yes"sv, true);
+		Writer.BeginArray("seq");
+		Writer.AddInteger(1);
+		Writer.AddInteger(2);
+		Writer.AddInteger(3);
+		Writer.EndArray();
+		Writer.EndObject();
+		Writer.EndObject();
+		Writer.BeginArray("seq");
+		Writer.AddInteger(1);
+		Writer.AddInteger(2);
+		Writer.AddInteger(3);
+		Writer.EndArray();
+		Writer.BeginArray("mixed_seq");
+		Writer.AddInteger(1);
+		Writer.AddString("hello"sv);
+		Writer.AddFloat(44.4f);
+		Writer.BeginObject();
+		Writer.AddBool("yes"sv, true);
+		Writer.AddBool("no"sv, false);
+		Writer.EndObject();
+		Writer.EndArray();
+		CbObject Obj = Writer.Save();
+
+		ExtendableStringBuilder<128> Sb;
+		CbYamlWriter				 YamlWriter(Sb);
+		YamlWriter.WriteField(Obj.AsFieldView());
+
+		CHECK_EQ(Sb.ToView(),
+				 R"(sub:
+  no: false
+  sub:
+    yes: true
+    seq:
+      - 1
+      - 2
+      - 3
+seq:
+  - 1
+  - 2
+  - 3
+mixed_seq:
+  - 1
+  - hello
+  - 44.4
+  - yes: true
+    no: false
+)"sv);
+	}
+}
+#endif
+
+}  // namespace zen