// Copyright Epic Games, Inc. All Rights Reserved. #include "zencore/compactbinary.h" #include "zencore/compactbinarybuilder.h" #include "zencore/compactbinaryvalue.h" #include #include #include #include #include #include #include ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END namespace zen { class CbJsonWriter { public: explicit CbJsonWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) { NewLineAndIndent << LINE_TERMINATOR_ANSI; } void BeginObject() { Builder << '{'; NewLineAndIndent << '\t'; NeedsNewLine = true; } void EndObject() { NewLineAndIndent.RemoveSuffix(1); if (NeedsComma) { WriteOptionalNewLine(); } Builder << '}'; } void BeginArray() { Builder << '['; NewLineAndIndent << '\t'; NeedsNewLine = true; } void EndArray() { NewLineAndIndent.RemoveSuffix(1); if (NeedsComma) { WriteOptionalNewLine(); } Builder << ']'; } void WriteField(CbFieldView Field) { using namespace std::literals; WriteOptionalComma(); WriteOptionalNewLine(); if (std::u8string_view Name = Field.GetU8Name(); !Name.empty()) { AppendQuotedString(Name); Builder << ": "sv; } switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) { case CbFieldType::Null: Builder << "null"sv; break; case CbFieldType::Object: case CbFieldType::UniformObject: { BeginObject(); for (CbFieldView It : Field) { WriteField(It); } EndObject(); } break; case CbFieldType::Array: case CbFieldType::UniformArray: { BeginArray(); for (CbFieldView It : Field) { WriteField(It); } EndArray(); } break; case CbFieldType::Binary: AppendBase64String(Accessor.AsBinary()); break; case CbFieldType::String: AppendQuotedString(Accessor.AsU8String()); break; case CbFieldType::IntegerPositive: Builder << Accessor.AsIntegerPositive(); break; case CbFieldType::IntegerNegative: Builder << Accessor.AsIntegerNegative(); break; case CbFieldType::Float32: { const float Value = Accessor.AsFloat32(); if (std::isfinite(Value)) { Builder.Append(fmt::format("{:.9g}", Value)); } else { Builder << "null"sv; } } break; case CbFieldType::Float64: { const double Value = Accessor.AsFloat64(); if (std::isfinite(Value)) { Builder.Append(fmt::format("{:.17g}", Value)); } else { Builder << "null"sv; } } break; case CbFieldType::BoolFalse: Builder << "false"sv; break; case CbFieldType::BoolTrue: Builder << "true"sv; break; case CbFieldType::ObjectAttachment: case CbFieldType::BinaryAttachment: { Builder << '"'; Accessor.AsAttachment().ToHexString(Builder); Builder << '"'; } break; case CbFieldType::Hash: { Builder << '"'; Accessor.AsHash().ToHexString(Builder); Builder << '"'; } break; case CbFieldType::Uuid: { Builder << '"'; Accessor.AsUuid().ToString(Builder); Builder << '"'; } break; case CbFieldType::DateTime: Builder << '"' << DateTime(Accessor.AsDateTimeTicks()).ToIso8601() << '"'; break; case CbFieldType::TimeSpan: { const TimeSpan Span(Accessor.AsTimeSpanTicks()); if (Span.GetDays() == 0) { Builder << '"' << Span.ToString("%h:%m:%s.%n") << '"'; } else { Builder << '"' << Span.ToString("%d.%h:%m:%s.%n") << '"'; } break; } case CbFieldType::ObjectId: Builder << '"'; Accessor.AsObjectId().ToString(Builder); Builder << '"'; break; case CbFieldType::CustomById: { CbCustomById Custom = Accessor.AsCustomById(); Builder << "{ \"Id\": "; Builder << Custom.Id; Builder << ", \"Data\": "; AppendBase64String(Custom.Data); Builder << " }"; break; } case CbFieldType::CustomByName: { CbCustomByName Custom = Accessor.AsCustomByName(); Builder << "{ \"Name\": "; AppendQuotedString(Custom.Name); Builder << ", \"Data\": "; AppendBase64String(Custom.Data); Builder << " }"; break; } default: ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType())); break; } NeedsComma = true; NeedsNewLine = true; } private: void WriteOptionalComma() { if (NeedsComma) { NeedsComma = false; Builder << ','; } } void WriteOptionalNewLine() { if (NeedsNewLine) { NeedsNewLine = false; Builder << NewLineAndIndent; } } void AppendQuotedString(std::u8string_view Value) { using namespace std::literals; const AsciiSet EscapeSet( "\\\"\b\f\n\r\t" "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"); Builder << '\"'; while (!Value.empty()) { std::u8string_view Verbatim = AsciiSet::FindPrefixWithout(Value, EscapeSet); Builder << Verbatim; Value = Value.substr(Verbatim.size()); std::u8string_view Escape = AsciiSet::FindPrefixWith(Value, EscapeSet); for (char Char : Escape) { switch (Char) { case '\\': Builder << "\\\\"sv; break; case '\"': Builder << "\\\""sv; break; case '\b': Builder << "\\b"sv; break; case '\f': Builder << "\\f"sv; break; case '\n': Builder << "\\n"sv; break; case '\r': Builder << "\\r"sv; break; case '\t': Builder << "\\t"sv; break; default: Builder << Char; break; } } Value = Value.substr(Escape.size()); } Builder << '\"'; } void AppendBase64String(MemoryView Value) { Builder << '"'; ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); } private: StringBuilderBase& Builder; ExtendableStringBuilder<32> NewLineAndIndent; bool NeedsComma{false}; bool NeedsNewLine{false}; }; void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder) { CbJsonWriter Writer(Builder); Writer.WriteField(Object.AsFieldView()); } void CompactBinaryToJson(const CbArrayView& Array, StringBuilderBase& Builder) { CbJsonWriter Writer(Builder); Writer.WriteField(Array.AsFieldView()); } void CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder) { std::vector Fields = ReadCompactBinaryStream(Data); CbJsonWriter Writer(InBuilder); if (!Fields.empty()) { if (Fields.size() == 1) { Writer.WriteField(Fields[0]); return; } bool UseTopLevelObject = Fields[0].HasName(); if (UseTopLevelObject) { Writer.BeginObject(); } else { Writer.BeginArray(); } for (const CbFieldView& Field : Fields) { Writer.WriteField(Field); } if (UseTopLevelObject) { Writer.EndObject(); } else { Writer.EndArray(); } } } class CbJsonReader { public: static CbFieldIterator Read(std::string_view JsonText, std::string& Error) { using namespace json11; const Json Json = Json::parse(std::string(JsonText), Error); if (Error.empty()) { CbWriter Writer; if (ReadField(Writer, Json, std::string_view(), Error)) { return Writer.Save(); } } return CbFieldIterator(); } private: static bool ReadField(CbWriter& Writer, const json11::Json& Json, const std::string_view FieldName, std::string& Error) { using namespace json11; switch (Json.type()) { case Json::Type::OBJECT: { if (FieldName.empty()) { Writer.BeginObject(); } else { Writer.BeginObject(FieldName); } for (const auto& Kv : Json.object_items()) { const std::string& Name = Kv.first; const json11::Json& Item = Kv.second; if (ReadField(Writer, Item, Name, Error) == false) { return false; } } Writer.EndObject(); } break; case Json::Type::ARRAY: { if (FieldName.empty()) { Writer.BeginArray(); } else { Writer.BeginArray(FieldName); } for (const json11::Json& Item : Json.array_items()) { if (ReadField(Writer, Item, std::string_view(), Error) == false) { return false; } } Writer.EndArray(); } break; case Json::Type::NUL: { if (FieldName.empty()) { Writer.AddNull(); } else { Writer.AddNull(FieldName); } } break; case Json::Type::BOOL: { if (FieldName.empty()) { Writer.AddBool(Json.bool_value()); } else { Writer.AddBool(FieldName, Json.bool_value()); } } break; case Json::Type::NUMBER: { if (FieldName.empty()) { Writer.AddFloat(Json.number_value()); } else { Writer.AddFloat(FieldName, Json.number_value()); } } break; case Json::Type::STRING: { Oid Id; if (TryParseObjectId(Json.string_value(), Id)) { if (FieldName.empty()) { Writer.AddObjectId(Id); } else { Writer.AddObjectId(FieldName, Id); } return true; } IoHash Hash; if (TryParseIoHash(Json.string_value(), Hash)) { if (FieldName.empty()) { Writer.AddHash(Hash); } else { Writer.AddHash(FieldName, Hash); } return true; } if (FieldName.empty()) { Writer.AddString(Json.string_value()); } else { Writer.AddString(FieldName, Json.string_value()); } } break; default: break; } return true; } static constexpr AsciiSet HexCharSet = AsciiSet("0123456789abcdefABCDEF"); static bool TryParseObjectId(std::string_view Str, Oid& Id) { using namespace std::literals; if (Str.size() == Oid::StringLength && AsciiSet::HasOnly(Str, HexCharSet)) { Id = Oid::FromHexString(Str); return true; } if (Str.starts_with("0x"sv)) { return TryParseObjectId(Str.substr(2), Id); } return false; } static bool TryParseIoHash(std::string_view Str, IoHash& Hash) { using namespace std::literals; if (Str.size() == IoHash::StringLength && AsciiSet::HasOnly(Str, HexCharSet)) { Hash = IoHash::FromHexString(Str); return true; } if (Str.starts_with("0x"sv)) { return TryParseIoHash(Str.substr(2), Hash); } return false; } }; CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json, std::string& Error) { if (Json.empty() == false) { return CbJsonReader::Read(Json, Error); } return CbFieldIterator(); } CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json) { std::string Error; return LoadCompactBinaryFromJson(Json, Error); } #if ZEN_WITH_TESTS void cbjson_forcelink() { } TEST_CASE("uson.json") { using namespace std::literals; SUBCASE("string") { CbObjectWriter Writer; Writer << "KeyOne" << "ValueOne"; Writer << "KeyTwo" << "ValueTwo"; CbObject Obj = Writer.Save(); StringBuilder<128> Sb; const char* JsonText = Obj.ToJson(Sb).Data(); std::string JsonError; json11::Json Json = json11::Json::parse(JsonText, JsonError); const std::string ValueOne = Json["KeyOne"].string_value(); const std::string ValueTwo = Json["KeyTwo"].string_value(); CHECK(JsonError.empty()); CHECK(ValueOne == "ValueOne"); CHECK(ValueTwo == "ValueTwo"); } SUBCASE("number") { const float ExpectedFloatValue = 21.21f; const double ExpectedDoubleValue = 42.42; CbObjectWriter Writer; Writer << "Float" << ExpectedFloatValue; Writer << "Double" << ExpectedDoubleValue; CbObject Obj = Writer.Save(); StringBuilder<128> Sb; const char* JsonText = Obj.ToJson(Sb).Data(); std::string JsonError; json11::Json Json = json11::Json::parse(JsonText, JsonError); const float FloatValue = float(Json["Float"].number_value()); const double DoubleValue = Json["Double"].number_value(); CHECK(JsonError.empty()); CHECK(FloatValue == Approx(ExpectedFloatValue)); CHECK(DoubleValue == Approx(ExpectedDoubleValue)); } SUBCASE("number.nan") { constexpr float FloatNan = std::numeric_limits::quiet_NaN(); constexpr double DoubleNan = std::numeric_limits::quiet_NaN(); CbObjectWriter Writer; Writer << "FloatNan" << FloatNan; Writer << "DoubleNan" << DoubleNan; CbObject Obj = Writer.Save(); StringBuilder<128> Sb; const char* JsonText = Obj.ToJson(Sb).Data(); std::string JsonError; json11::Json Json = json11::Json::parse(JsonText, JsonError); const double FloatValue = Json["FloatNan"].number_value(); const double DoubleValue = Json["DoubleNan"].number_value(); CHECK(JsonError.empty()); CHECK(FloatValue == 0); CHECK(DoubleValue == 0); } SUBCASE("stream") { const auto MakeObject = [&](std::string_view Name, const std::vector& Fields) -> CbObject { CbWriter Writer; Writer.SetName(Name); Writer.BeginObject(); for (const auto& Field : Fields) { Writer.AddInteger(fmt::format("{}", Field), Field); } Writer.EndObject(); return Writer.Save().AsObject(); }; std::vector Buffer; auto AppendToBuffer = [&](const void* Data, size_t Count) { const uint8_t* AppendBytes = reinterpret_cast(Data); Buffer.insert(Buffer.end(), AppendBytes, AppendBytes + Count); }; auto Append = [&](const CbFieldView& Field) { Field.WriteToStream([&](const void* Data, size_t Count) { const uint8_t* AppendBytes = reinterpret_cast(Data); Buffer.insert(Buffer.end(), AppendBytes, AppendBytes + Count); }); }; CbObject DataObjects[] = {MakeObject("Empty object"sv, {}), MakeObject("OneField object"sv, {5}), MakeObject("TwoField object"sv, {-5, 999}), MakeObject("ThreeField object"sv, {1, 2, -129})}; for (const CbObject& Object : DataObjects) { Object.AsField().WriteToStream(AppendToBuffer); } ExtendableStringBuilder<128> Sb; CompactBinaryToJson(MemoryView(Buffer.data(), Buffer.size()), Sb); std::string JsonText = Sb.ToString().c_str(); std::string JsonError; json11::Json Json = json11::Json::parse(JsonText, JsonError); std::string ParsedJsonString = Json.dump(); CHECK(!ParsedJsonString.empty()); } } TEST_CASE("json.uson") { using namespace std::literals; using namespace json11; SUBCASE("empty") { CbFieldIterator It = LoadCompactBinaryFromJson(""sv); CHECK(It.HasValue() == false); } SUBCASE("object") { const Json JsonObject = Json::object{{"Null", nullptr}, {"String", "Value1"}, {"Bool", true}, {"Number", 46.2}, {"Array", Json::array{1, 2, 3}}, {"Object", Json::object{ {"String", "Value2"}, }}}; CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); CHECK(Cb["Null"].IsNull()); CHECK(Cb["String"].AsString() == "Value1"sv); CHECK(Cb["Bool"].AsBool()); CHECK(Cb["Number"].AsDouble() == 46.2); CHECK(Cb["Object"].IsObject()); CbObjectView Object = Cb["Object"].AsObjectView(); CHECK(Object["String"].AsString() == "Value2"sv); } SUBCASE("array") { const Json JsonArray = Json::array{42, 43, 44}; CbArray Cb = LoadCompactBinaryFromJson(JsonArray.dump()).AsArray(); auto It = Cb.CreateIterator(); CHECK((*It).AsDouble() == 42); It++; CHECK((*It).AsDouble() == 43); It++; CHECK((*It).AsDouble() == 44); } SUBCASE("objectid") { const Oid& Id = Oid::NewOid(); StringBuilder<64> Sb; Id.ToString(Sb); Json JsonObject = Json::object{{"value", Sb.ToString()}}; CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); CHECK(Cb["value"sv].IsObjectId()); CHECK(Cb["value"sv].AsObjectId() == Id); Sb.Reset(); Sb << "0x"; Id.ToString(Sb); JsonObject = Json::object{{"value", Sb.ToString()}}; Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); CHECK(Cb["value"sv].IsObjectId()); CHECK(Cb["value"sv].AsObjectId() == Id); } SUBCASE("iohash") { const uint8_t Data[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, }; const IoHash Hash = IoHash::HashBuffer(Data, sizeof(Data)); Json JsonObject = Json::object{{"value", Hash.ToHexString()}}; CbObject Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); CHECK(Cb["value"sv].IsHash()); CHECK(Cb["value"sv].AsHash() == Hash); JsonObject = Json::object{{"value", "0x" + Hash.ToHexString()}}; Cb = LoadCompactBinaryFromJson(JsonObject.dump()).AsObject(); CHECK(Cb["value"sv].IsHash()); CHECK(Cb["value"sv].AsHash() == Hash); } } #endif // ZEN_WITH_TESTS } // namespace zen