diff options
Diffstat (limited to 'src/zencore/compactbinaryjson.cpp')
| -rw-r--r-- | src/zencore/compactbinaryjson.cpp | 805 |
1 files changed, 805 insertions, 0 deletions
diff --git a/src/zencore/compactbinaryjson.cpp b/src/zencore/compactbinaryjson.cpp new file mode 100644 index 000000000..58a91c731 --- /dev/null +++ b/src/zencore/compactbinaryjson.cpp @@ -0,0 +1,805 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencore/compactbinary.h" +#include "zencore/compactbinarybuilder.h" +#include "zencore/compactbinaryvalue.h" + +#include <zencore/assertfmt.h> +#include <zencore/base64.h> +#include <zencore/fmtutils.h> +#include <zencore/string.h> +#include <zencore/testing.h> + +#include <fmt/format.h> +#include <vector> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <json11.hpp> +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +class CbJsonWriter +{ +public: + explicit CbJsonWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) { NewLineAndIndent << LINE_TERMINATOR_ANSI; } + + void BeginObject() + { + Builder << '{'; + NewLineAndIndent << '\t'; + NeedsNewLine = true; + } + + void EndObject() + { + NewLineAndIndent.RemoveSuffix(1); + if (NeedsComma) + { + WriteOptionalNewLine(); + } + Builder << '}'; + } + + void BeginArray() + { + Builder << '['; + NewLineAndIndent << '\t'; + NeedsNewLine = true; + } + + void EndArray() + { + NewLineAndIndent.RemoveSuffix(1); + if (NeedsComma) + { + WriteOptionalNewLine(); + } + Builder << ']'; + } + + void WriteField(CbFieldView Field) + { + using namespace std::literals; + + WriteOptionalComma(); + WriteOptionalNewLine(); + + if (std::u8string_view Name = Field.GetU8Name(); !Name.empty()) + { + AppendQuotedString(Name); + Builder << ": "sv; + } + + switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + { + case CbFieldType::Null: + Builder << "null"sv; + break; + case CbFieldType::Object: + case CbFieldType::UniformObject: + { + BeginObject(); + for (CbFieldView It : Field) + { + WriteField(It); + } + EndObject(); + } + break; + case CbFieldType::Array: + case CbFieldType::UniformArray: + { + BeginArray(); + for (CbFieldView It : Field) + { + WriteField(It); + } + EndArray(); + } + break; + case CbFieldType::Binary: + AppendBase64String(Accessor.AsBinary()); + break; + case CbFieldType::String: + AppendQuotedString(Accessor.AsU8String()); + break; + case CbFieldType::IntegerPositive: + Builder << Accessor.AsIntegerPositive(); + break; + case CbFieldType::IntegerNegative: + Builder << Accessor.AsIntegerNegative(); + break; + case CbFieldType::Float32: + { + const float Value = Accessor.AsFloat32(); + if (std::isfinite(Value)) + { + Builder.Append(fmt::format("{:.9g}", Value)); + } + else + { + Builder << "null"sv; + } + } + break; + case CbFieldType::Float64: + { + const double Value = Accessor.AsFloat64(); + if (std::isfinite(Value)) + { + Builder.Append(fmt::format("{:.17g}", Value)); + } + else + { + Builder << "null"sv; + } + } + break; + case CbFieldType::BoolFalse: + Builder << "false"sv; + break; + case CbFieldType::BoolTrue: + Builder << "true"sv; + break; + case CbFieldType::ObjectAttachment: + case CbFieldType::BinaryAttachment: + { + Builder << '"'; + Accessor.AsAttachment().ToHexString(Builder); + Builder << '"'; + } + break; + case CbFieldType::Hash: + { + Builder << '"'; + Accessor.AsHash().ToHexString(Builder); + Builder << '"'; + } + break; + case CbFieldType::Uuid: + { + Builder << '"'; + Accessor.AsUuid().ToString(Builder); + Builder << '"'; + } + break; + case CbFieldType::DateTime: + Builder << '"' << DateTime(Accessor.AsDateTimeTicks()).ToIso8601() << '"'; + break; + case CbFieldType::TimeSpan: + { + const TimeSpan Span(Accessor.AsTimeSpanTicks()); + if (Span.GetDays() == 0) + { + Builder << '"' << Span.ToString("%h:%m:%s.%n") << '"'; + } + else + { + Builder << '"' << Span.ToString("%d.%h:%m:%s.%n") << '"'; + } + break; + } + case CbFieldType::ObjectId: + Builder << '"'; + Accessor.AsObjectId().ToString(Builder); + Builder << '"'; + break; + case CbFieldType::CustomById: + { + CbCustomById Custom = Accessor.AsCustomById(); + Builder << "{ \"Id\": "; + Builder << Custom.Id; + Builder << ", \"Data\": "; + AppendBase64String(Custom.Data); + Builder << " }"; + break; + } + case CbFieldType::CustomByName: + { + CbCustomByName Custom = Accessor.AsCustomByName(); + Builder << "{ \"Name\": "; + AppendQuotedString(Custom.Name); + Builder << ", \"Data\": "; + AppendBase64String(Custom.Data); + Builder << " }"; + break; + } + default: + ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType())); + break; + } + + NeedsComma = true; + NeedsNewLine = true; + } + +private: + void WriteOptionalComma() + { + if (NeedsComma) + { + NeedsComma = false; + Builder << ','; + } + } + + void WriteOptionalNewLine() + { + if (NeedsNewLine) + { + NeedsNewLine = false; + Builder << NewLineAndIndent; + } + } + + void AppendQuotedString(std::u8string_view Value) + { + using namespace std::literals; + + const AsciiSet EscapeSet( + "\\\"\b\f\n\r\t" + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"); + + Builder << '\"'; + while (!Value.empty()) + { + std::u8string_view Verbatim = AsciiSet::FindPrefixWithout(Value, EscapeSet); + Builder << Verbatim; + + Value = Value.substr(Verbatim.size()); + + std::u8string_view Escape = AsciiSet::FindPrefixWith(Value, EscapeSet); + for (char Char : Escape) + { + switch (Char) + { + case '\\': + Builder << "\\\\"sv; + break; + case '\"': + Builder << "\\\""sv; + break; + case '\b': + Builder << "\\b"sv; + break; + case '\f': + Builder << "\\f"sv; + break; + case '\n': + Builder << "\\n"sv; + break; + case '\r': + Builder << "\\r"sv; + break; + case '\t': + Builder << "\\t"sv; + break; + default: + Builder << Char; + break; + } + } + Value = Value.substr(Escape.size()); + } + Builder << '\"'; + } + + void AppendBase64String(MemoryView Value) + { + Builder << '"'; + ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); + const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); + const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); + Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + } + +private: + StringBuilderBase& Builder; + ExtendableStringBuilder<32> NewLineAndIndent; + bool NeedsComma{false}; + bool NeedsNewLine{false}; +}; + +void +CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder) +{ + CbJsonWriter Writer(Builder); + Writer.WriteField(Object.AsFieldView()); +} + +void +CompactBinaryToJson(const CbArrayView& Array, StringBuilderBase& Builder) +{ + CbJsonWriter Writer(Builder); + Writer.WriteField(Array.AsFieldView()); +} + +void +CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder) +{ + std::vector<CbFieldView> Fields = ReadCompactBinaryStream(Data); + CbJsonWriter Writer(InBuilder); + if (!Fields.empty()) + { + if (Fields.size() == 1) + { + Writer.WriteField(Fields[0]); + return; + } + bool UseTopLevelObject = Fields[0].HasName(); + if (UseTopLevelObject) + { + Writer.BeginObject(); + } + else + { + Writer.BeginArray(); + } + for (const CbFieldView& Field : Fields) + { + Writer.WriteField(Field); + } + if (UseTopLevelObject) + { + Writer.EndObject(); + } + else + { + Writer.EndArray(); + } + } +} + +class CbJsonReader +{ +public: + static CbFieldIterator Read(std::string_view JsonText, std::string& Error) + { + using namespace json11; + + const Json Json = Json::parse(std::string(JsonText), Error); + + if (Error.empty()) + { + CbWriter Writer; + if (ReadField(Writer, Json, std::string_view(), Error)) + { + return Writer.Save(); + } + } + + return CbFieldIterator(); + } + +private: + static bool ReadField(CbWriter& Writer, const json11::Json& Json, const std::string_view FieldName, std::string& Error) + { + using namespace json11; + + switch (Json.type()) + { + case Json::Type::OBJECT: + { + if (FieldName.empty()) + { + Writer.BeginObject(); + } + else + { + Writer.BeginObject(FieldName); + } + + for (const auto& Kv : Json.object_items()) + { + const std::string& Name = Kv.first; + const json11::Json& Item = Kv.second; + + if (ReadField(Writer, Item, Name, Error) == false) + { + return false; + } + } + + Writer.EndObject(); + } + break; + case Json::Type::ARRAY: + { + if (FieldName.empty()) + { + Writer.BeginArray(); + } + else + { + Writer.BeginArray(FieldName); + } + + for (const json11::Json& Item : Json.array_items()) + { + if (ReadField(Writer, Item, std::string_view(), Error) == false) + { + return false; + } + } + + Writer.EndArray(); + } + break; + case Json::Type::NUL: + { + if (FieldName.empty()) + { + Writer.AddNull(); + } + else + { + Writer.AddNull(FieldName); + } + } + break; + case Json::Type::BOOL: + { + if (FieldName.empty()) + { + Writer.AddBool(Json.bool_value()); + } + else + { + Writer.AddBool(FieldName, Json.bool_value()); + } + } + break; + case Json::Type::NUMBER: + { + if (FieldName.empty()) + { + Writer.AddFloat(Json.number_value()); + } + else + { + Writer.AddFloat(FieldName, Json.number_value()); + } + } + break; + case Json::Type::STRING: + { + Oid Id; + if (TryParseObjectId(Json.string_value(), Id)) + { + if (FieldName.empty()) + { + Writer.AddObjectId(Id); + } + else + { + Writer.AddObjectId(FieldName, Id); + } + + return true; + } + + IoHash Hash; + if (TryParseIoHash(Json.string_value(), Hash)) + { + if (FieldName.empty()) + { + Writer.AddHash(Hash); + } + else + { + Writer.AddHash(FieldName, Hash); + } + + return true; + } + + if (FieldName.empty()) + { + Writer.AddString(Json.string_value()); + } + else + { + Writer.AddString(FieldName, Json.string_value()); + } + } + break; + default: + break; + } + + return true; + } + + static constexpr AsciiSet HexCharSet = AsciiSet("0123456789abcdefABCDEF"); + + static bool TryParseObjectId(std::string_view Str, Oid& Id) + { + using namespace std::literals; + + if (Str.size() == Oid::StringLength && AsciiSet::HasOnly(Str, HexCharSet)) + { + Id = Oid::FromHexString(Str); + return true; + } + + if (Str.starts_with("0x"sv)) + { + return TryParseObjectId(Str.substr(2), Id); + } + + return false; + } + + static bool TryParseIoHash(std::string_view Str, IoHash& Hash) + { + using namespace std::literals; + + if (Str.size() == IoHash::StringLength && AsciiSet::HasOnly(Str, HexCharSet)) + { + Hash = IoHash::FromHexString(Str); + return true; + } + + if (Str.starts_with("0x"sv)) + { + return TryParseIoHash(Str.substr(2), Hash); + } + + return false; + } +}; + +CbFieldIterator +LoadCompactBinaryFromJson(std::string_view Json, std::string& Error) +{ + if (Json.empty() == false) + { + return CbJsonReader::Read(Json, Error); + } + + return CbFieldIterator(); +} + +CbFieldIterator +LoadCompactBinaryFromJson(std::string_view Json) +{ + std::string Error; + return LoadCompactBinaryFromJson(Json, Error); +} + +#if ZEN_WITH_TESTS +void +cbjson_forcelink() +{ +} + +TEST_CASE("uson.json") +{ + using namespace std::literals; + + SUBCASE("string") + { + CbObjectWriter Writer; + Writer << "KeyOne" + << "ValueOne"; + Writer << "KeyTwo" + << "ValueTwo"; + CbObject Obj = Writer.Save(); + + StringBuilder<128> Sb; + const char* JsonText = Obj.ToJson(Sb).Data(); + + std::string JsonError; + json11::Json Json = json11::Json::parse(JsonText, JsonError); + + const std::string ValueOne = Json["KeyOne"].string_value(); + const std::string ValueTwo = Json["KeyTwo"].string_value(); + + CHECK(JsonError.empty()); + CHECK(ValueOne == "ValueOne"); + CHECK(ValueTwo == "ValueTwo"); + } + + SUBCASE("number") + { + const float ExpectedFloatValue = 21.21f; + const double ExpectedDoubleValue = 42.42; + + CbObjectWriter Writer; + Writer << "Float" << ExpectedFloatValue; + Writer << "Double" << ExpectedDoubleValue; + + CbObject Obj = Writer.Save(); + + StringBuilder<128> Sb; + const char* JsonText = Obj.ToJson(Sb).Data(); + + std::string JsonError; + json11::Json Json = json11::Json::parse(JsonText, JsonError); + + const float FloatValue = float(Json["Float"].number_value()); + const double DoubleValue = Json["Double"].number_value(); + + CHECK(JsonError.empty()); + CHECK(FloatValue == Approx(ExpectedFloatValue)); + CHECK(DoubleValue == Approx(ExpectedDoubleValue)); + } + + SUBCASE("number.nan") + { + constexpr float FloatNan = std::numeric_limits<float>::quiet_NaN(); + constexpr double DoubleNan = std::numeric_limits<double>::quiet_NaN(); + + CbObjectWriter Writer; + Writer << "FloatNan" << FloatNan; + Writer << "DoubleNan" << DoubleNan; + + CbObject Obj = Writer.Save(); + + StringBuilder<128> Sb; + const char* JsonText = Obj.ToJson(Sb).Data(); + + std::string JsonError; + json11::Json Json = json11::Json::parse(JsonText, JsonError); + + const double FloatValue = Json["FloatNan"].number_value(); + const double DoubleValue = Json["DoubleNan"].number_value(); + + CHECK(JsonError.empty()); + CHECK(FloatValue == 0); + CHECK(DoubleValue == 0); + } + + SUBCASE("stream") + { + const auto MakeObject = [&](std::string_view Name, const std::vector<int>& Fields) -> CbObject { + CbWriter Writer; + Writer.SetName(Name); + Writer.BeginObject(); + for (const auto& Field : Fields) + { + Writer.AddInteger(fmt::format("{}", Field), Field); + } + Writer.EndObject(); + return Writer.Save().AsObject(); + }; + + std::vector<uint8_t> Buffer; + + auto AppendToBuffer = [&](const void* Data, size_t Count) { + const uint8_t* AppendBytes = reinterpret_cast<const uint8_t*>(Data); + Buffer.insert(Buffer.end(), AppendBytes, AppendBytes + Count); + }; + + auto Append = [&](const CbFieldView& Field) { + Field.WriteToStream([&](const void* Data, size_t Count) { + const uint8_t* AppendBytes = reinterpret_cast<const uint8_t*>(Data); + Buffer.insert(Buffer.end(), AppendBytes, AppendBytes + Count); + }); + }; + + CbObject DataObjects[] = {MakeObject("Empty object"sv, {}), + MakeObject("OneField object"sv, {5}), + MakeObject("TwoField object"sv, {-5, 999}), + MakeObject("ThreeField object"sv, {1, 2, -129})}; + for (const CbObject& Object : DataObjects) + { + Object.AsField().WriteToStream(AppendToBuffer); + } + + ExtendableStringBuilder<128> Sb; + CompactBinaryToJson(MemoryView(Buffer.data(), Buffer.size()), Sb); + std::string JsonText = Sb.ToString().c_str(); + std::string JsonError; + json11::Json Json = json11::Json::parse(JsonText, JsonError); + std::string ParsedJsonString = Json.dump(); + CHECK(!ParsedJsonString.empty()); + } +} + +TEST_CASE("json.uson") +{ + using namespace std::literals; + using namespace json11; + + SUBCASE("empty") + { + CbFieldIterator It = LoadCompactBinaryFromJson(""sv); + CHECK(It.HasValue() == false); + } + + SUBCASE("object") + { + const Json JsonObject = Json::object{{"Null", nullptr}, + {"String", "Value1"}, + {"Bool", true}, + {"Number", 46.2}, + {"Array", Json::array{1, 2, 3}}, + {"Object", + Json::object{ + {"String", "Value2"}, + }}}; + + CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); + + CHECK(Cb["Null"].IsNull()); + CHECK(Cb["String"].AsString() == "Value1"sv); + CHECK(Cb["Bool"].AsBool()); + CHECK(Cb["Number"].AsDouble() == 46.2); + CHECK(Cb["Object"].IsObject()); + CbObjectView Object = Cb["Object"].AsObjectView(); + CHECK(Object["String"].AsString() == "Value2"sv); + } + + SUBCASE("array") + { + const Json JsonArray = Json::array{42, 43, 44}; + CbArray Cb = LoadCompactBinaryFromJson(JsonArray.dump()).AsArray(); + + auto It = Cb.CreateIterator(); + CHECK((*It).AsDouble() == 42); + It++; + CHECK((*It).AsDouble() == 43); + It++; + CHECK((*It).AsDouble() == 44); + } + + SUBCASE("objectid") + { + const Oid& Id = Oid::NewOid(); + + StringBuilder<64> Sb; + Id.ToString(Sb); + + Json JsonObject = Json::object{{"value", Sb.ToString()}}; + CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); + + CHECK(Cb["value"sv].IsObjectId()); + CHECK(Cb["value"sv].AsObjectId() == Id); + + Sb.Reset(); + Sb << "0x"; + Id.ToString(Sb); + + JsonObject = Json::object{{"value", Sb.ToString()}}; + Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); + + CHECK(Cb["value"sv].IsObjectId()); + CHECK(Cb["value"sv].AsObjectId() == Id); + } + + SUBCASE("iohash") + { + const uint8_t Data[] = { + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + }; + + const IoHash Hash = IoHash::HashBuffer(Data, sizeof(Data)); + + Json JsonObject = Json::object{{"value", Hash.ToHexString()}}; + CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); + + CHECK(Cb["value"sv].IsHash()); + CHECK(Cb["value"sv].AsHash() == Hash); + + JsonObject = Json::object{{"value", "0x" + Hash.ToHexString()}}; + Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); + + CHECK(Cb["value"sv].IsHash()); + CHECK(Cb["value"sv].AsHash() == Hash); + } +} + +#endif // ZEN_WITH_TESTS + +} // namespace zen |