// Copyright Epic Games, Inc. All Rights Reserved. #include "zencore/compactbinarypackage.h" #include #include #include #include #include namespace zen { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// CbAttachment::CbAttachment(const CompressedBuffer& InValue, const IoHash& Hash) : CbAttachment(InValue.MakeOwned(), Hash) { } CbAttachment::CbAttachment(const SharedBuffer& InValue) : CbAttachment(CompositeBuffer(InValue)) { } CbAttachment::CbAttachment(const SharedBuffer& InValue, const IoHash& InHash) : CbAttachment(CompositeBuffer(InValue), InHash) { } CbAttachment::CbAttachment(const CompositeBuffer& InValue) : Hash(InValue.IsNull() ? IoHash::Zero : IoHash::HashBuffer(InValue)) , Value(InValue) { if (std::get(Value).IsNull()) { Value.emplace(); } } CbAttachment::CbAttachment(CompositeBuffer&& InValue) : Hash(InValue.IsNull() ? IoHash::Zero : IoHash::HashBuffer(InValue)) , Value(std::move(InValue)) { if (std::get(Value).IsNull()) { Value.emplace(); } } CbAttachment::CbAttachment(CompositeBuffer&& InValue, const IoHash& InHash) : Hash(InHash), Value(InValue) { if (std::get(Value).IsNull()) { Value.emplace(); } } CbAttachment::CbAttachment(CompressedBuffer&& InValue, const IoHash& InHash) : Hash(InHash), Value(InValue) { if (std::get(Value).IsNull()) { Value.emplace(); } } CbAttachment::CbAttachment(const CbObject& InValue, const IoHash* const InHash) { auto SetValue = [&](const CbObject& ValueToSet) { if (InHash) { Value.emplace(ValueToSet); Hash = *InHash; } else { Value.emplace(ValueToSet); Hash = ValueToSet.GetHash(); } }; MemoryView View; if (!InValue.IsOwned() || !InValue.TryGetSerializedView(View)) { SetValue(CbObject::Clone(InValue)); } else { SetValue(InValue); } } bool CbAttachment::TryLoad(IoBuffer& InBuffer, BufferAllocator Allocator) { BinaryReader Reader(InBuffer.Data(), InBuffer.Size()); return TryLoad(Reader, Allocator); } bool CbAttachment::TryLoad(CbFieldIterator& Fields) { if (const CbObjectView ObjectView = Fields.AsObjectView(); !Fields.HasError()) { // Is a null object or object not prefixed with a precomputed hash value Value.emplace(CbObject(ObjectView, Fields.GetOuterBuffer())); Hash = ObjectView.GetHash(); ++Fields; } else if (const IoHash ObjectAttachmentHash = Fields.AsObjectAttachment(); !Fields.HasError()) { // Is an object ++Fields; const CbObjectView InnerObjectView = Fields.AsObjectView(); if (Fields.HasError()) { return false; } Value.emplace(CbObject(InnerObjectView, Fields.GetOuterBuffer())); Hash = ObjectAttachmentHash; ++Fields; } else if (const IoHash BinaryAttachmentHash = Fields.AsBinaryAttachment(); !Fields.HasError()) { // Is an uncompressed binary blob ++Fields; MemoryView BinaryView = Fields.AsBinaryView(); if (Fields.HasError()) { return false; } Value.emplace(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())); Hash = BinaryAttachmentHash; ++Fields; } else if (MemoryView BinaryView = Fields.AsBinaryView(); !Fields.HasError()) { if (BinaryView.GetSize() > 0) { // Is a compressed binary blob IoHash RawHash; uint64_t RawSize; CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer()), RawHash, RawSize).MakeOwned(); Value.emplace(Compressed); Hash = RawHash; ++Fields; } else { // Is an uncompressed empty binary blob Value.emplace(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())); Hash = IoHash::HashBuffer(nullptr, 0); ++Fields; } } else { return false; } return true; } static bool TryLoad_ArchiveFieldIntoAttachment(CbAttachment& TargetAttachment, CbField&& Field, BinaryReader& Reader, BufferAllocator Allocator) { if (const CbObjectView ObjectView = Field.AsObjectView(); !Field.HasError()) { // Is a null object or object not prefixed with a precomputed hash value TargetAttachment = CbAttachment(CbObject(ObjectView, std::move(Field)), ObjectView.GetHash()); } else if (const IoHash ObjectAttachmentHash = Field.AsObjectAttachment(); !Field.HasError()) { // Is an object Field = LoadCompactBinary(Reader, Allocator); if (!Field.IsObject()) { return false; } TargetAttachment = CbAttachment(std::move(Field).AsObject(), ObjectAttachmentHash); } else if (const IoHash BinaryAttachmentHash = Field.AsBinaryAttachment(); !Field.HasError()) { // Is an uncompressed binary blob Field = LoadCompactBinary(Reader, Allocator); SharedBuffer Buffer = Field.AsBinary(); if (Field.HasError()) { return false; } TargetAttachment = CbAttachment(CompositeBuffer(Buffer), BinaryAttachmentHash); } else if (SharedBuffer Buffer = Field.AsBinary(); !Field.HasError()) { if (Buffer.GetSize() > 0) { // Is a compressed binary blob IoHash RawHash; uint64_t RawSize; CompressedBuffer Compressed = CompressedBuffer::FromCompressed(std::move(Buffer), RawHash, RawSize); TargetAttachment = CbAttachment(Compressed, RawHash); } else { // Is an uncompressed empty binary blob TargetAttachment = CbAttachment(CompositeBuffer(Buffer), IoHash::HashBuffer(nullptr, 0)); } } else { return false; } return true; } bool CbAttachment::TryLoad(BinaryReader& Reader, BufferAllocator Allocator) { CbField Field = LoadCompactBinary(Reader, Allocator); return TryLoad_ArchiveFieldIntoAttachment(*this, std::move(Field), Reader, Allocator); } void CbAttachment::Save(CbWriter& Writer) const { if (const CbObject* Object = std::get_if(&Value)) { if (*Object) { Writer.AddObjectAttachment(Hash); } Writer.AddObject(*Object); } else if (const CompositeBuffer* Binary = std::get_if(&Value)) { if (Binary->GetSize() > 0) { Writer.AddBinaryAttachment(Hash); } Writer.AddBinary(*Binary); } else if (const CompressedBuffer* Compressed = std::get_if(&Value)) { Writer.AddBinary(Compressed->GetCompressed()); } } void CbAttachment::Save(BinaryWriter& Writer) const { CbWriter TempWriter; Save(TempWriter); TempWriter.Save(Writer); } bool CbAttachment::IsNull() const { return std::holds_alternative(Value); } bool CbAttachment::IsBinary() const { return std::holds_alternative(Value); } bool CbAttachment::IsCompressedBinary() const { return std::holds_alternative(Value); } bool CbAttachment::IsObject() const { return std::holds_alternative(Value); } IoHash CbAttachment::GetHash() const { return Hash; } CompositeBuffer CbAttachment::AsCompositeBinary() const { if (const CompositeBuffer* BinValue = std::get_if(&Value)) { return *BinValue; } return CompositeBuffer::Null; } SharedBuffer CbAttachment::AsBinary() const { if (const CompositeBuffer* BinValue = std::get_if(&Value)) { return BinValue->Flatten(); } return {}; } CompressedBuffer CbAttachment::AsCompressedBinary() const { if (const CompressedBuffer* CompValue = std::get_if(&Value)) { return *CompValue; } return CompressedBuffer::Null; } /** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */ CbObject CbAttachment::AsObject() const { if (const CbObject* ObjectValue = std::get_if(&Value)) { return *ObjectValue; } return {}; } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void CbPackage::SetObject(CbObject InObject, const IoHash* InObjectHash, AttachmentResolver* InResolver) { if (InObject) { Object = InObject.IsOwned() ? std::move(InObject) : CbObject::Clone(InObject); if (InObjectHash) { ObjectHash = *InObjectHash; ZEN_ASSERT_SLOW(ObjectHash == Object.GetHash()); } else { ObjectHash = Object.GetHash(); } if (InResolver) { GatherAttachments(Object, *InResolver); } } else { Object.Reset(); ObjectHash = IoHash::Zero; } } void CbPackage::AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver) { if (!Attachment.IsNull()) { auto It = std::lower_bound(begin(Attachments), end(Attachments), Attachment); if (It != Attachments.end() && *It == Attachment) { CbAttachment& Existing = *It; Existing = Attachment; } else { Attachments.insert(It, Attachment); } if (Attachment.IsObject() && Resolver) { GatherAttachments(Attachment.AsObject(), *Resolver); } } } void CbPackage::AddAttachments(std::span InAttachments) { if (InAttachments.empty()) { return; } // Assume we have no duplicates! Attachments.insert(Attachments.end(), InAttachments.begin(), InAttachments.end()); std::sort(Attachments.begin(), Attachments.end()); ZEN_ASSERT_SLOW(std::unique(Attachments.begin(), Attachments.end()) == Attachments.end()); } int32_t CbPackage::RemoveAttachment(const IoHash& Hash) { return gsl::narrow_cast( std::erase_if(Attachments, [&Hash](const CbAttachment& Attachment) -> bool { return Attachment.GetHash() == Hash; })); } bool CbPackage::Equals(const CbPackage& Package) const { return ObjectHash == Package.ObjectHash && Attachments == Package.Attachments; } const CbAttachment* CbPackage::FindAttachment(const IoHash& Hash) const { auto It = std::find_if(begin(Attachments), end(Attachments), [&Hash](const CbAttachment& Attachment) -> bool { return Attachment.GetHash() == Hash; }); if (It == end(Attachments)) return nullptr; return &*It; } void CbPackage::GatherAttachments(const CbObject& Value, AttachmentResolver Resolver) { Value.IterateAttachments([this, &Resolver](CbFieldView Field) { const IoHash& Hash = Field.AsAttachment(); if (SharedBuffer Buffer = Resolver(Hash)) { if (Field.IsObjectAttachment()) { AddAttachment(CbAttachment(CbObject(std::move(Buffer)), Hash), &Resolver); } else { AddAttachment(CbAttachment(std::move(Buffer))); } } }); } bool CbPackage::TryLoad(IoBuffer InBuffer, BufferAllocator Allocator, AttachmentResolver* Mapper) { BinaryReader Reader(InBuffer.Data(), InBuffer.Size()); return TryLoad(Reader, Allocator, Mapper); } bool CbPackage::TryLoad(CbFieldIterator& Fields) { *this = CbPackage(); while (Fields) { if (Fields.IsNull()) { ++Fields; break; } else if (IoHash Hash = Fields.AsHash(); !Fields.HasError() && !Fields.IsAttachment()) { ++Fields; CbObjectView ObjectView = Fields.AsObjectView(); if (Fields.HasError() || Hash != ObjectView.GetHash()) { return false; } Object = CbObject(ObjectView, Fields.GetOuterBuffer()); Object.MakeOwned(); ObjectHash = Hash; ++Fields; } else { CbAttachment Attachment; if (!Attachment.TryLoad(Fields)) { return false; } AddAttachment(Attachment); } } return true; } bool CbPackage::TryLoad(BinaryReader& Reader, BufferAllocator Allocator, AttachmentResolver* Mapper) { // TODO: this needs to re-grow the ability to accept a reference to an attachment which is // not embedded ZEN_UNUSED(Mapper); #if 1 *this = CbPackage(); for (;;) { CbField Field = LoadCompactBinary(Reader, Allocator); if (!Field) { return false; } if (Field.IsNull()) { return true; } else if (IoHash Hash = Field.AsHash(); !Field.HasError() && !Field.IsAttachment()) { Field = LoadCompactBinary(Reader, Allocator); CbObjectView ObjectView = Field.AsObjectView(); if (Field.HasError() || Hash != ObjectView.GetHash()) { return false; } Object = CbObject(ObjectView, Field.GetOuterBuffer()); ObjectHash = Hash; } else { CbAttachment Attachment; if (!TryLoad_ArchiveFieldIntoAttachment(Attachment, std::move(Field), Reader, Allocator)) { return false; } AddAttachment(Attachment); } } #else uint8_t StackBuffer[64]; const auto StackAllocator = [&Allocator, &StackBuffer](uint64_t Size) -> UniqueBuffer { if (Size <= sizeof(StackBuffer)) { return UniqueBuffer::MakeMutableView(StackBuffer, Size); } return Allocator(Size); }; *this = CbPackage(); for (;;) { CbField ValueField = LoadCompactBinary(Reader, StackAllocator); if (!ValueField) { return false; } if (ValueField.IsNull()) { return true; } else if (ValueField.IsBinary()) { const MemoryView View = ValueField.AsBinaryView(); if (View.GetSize() > 0) { SharedBuffer Buffer = SharedBuffer::MakeView(View, ValueField.GetOuterBuffer()).MakeOwned(); CbField HashField = LoadCompactBinary(Reader, StackAllocator); const IoHash& Hash = HashField.AsAttachment(); ZEN_ASSERT(!HashField.HasError(), "Attachments must be a non-empty binary value with a content hash."); if (HashField.IsObjectAttachment()) { AddAttachment(CbAttachment(CbObject(std::move(Buffer)), Hash)); } else { AddAttachment(CbAttachment(std::move(Buffer), Hash)); } } } else if (ValueField.IsHash()) { const IoHash Hash = ValueField.AsHash(); ZEN_ASSERT(Mapper); AddAttachment(CbAttachment((*Mapper)(Hash), Hash)); } else { Object = ValueField.AsObject(); if (ValueField.HasError()) { return false; } Object.MakeOwned(); if (Object) { CbField HashField = LoadCompactBinary(Reader, StackAllocator); ObjectHash = HashField.AsObjectAttachment(); if (HashField.HasError() || Object.GetHash() != ObjectHash) { return false; } } else { Object.Reset(); } } } #endif } void CbPackage::Save(CbWriter& Writer) const { if (Object) { Writer.AddHash(ObjectHash); Writer.AddObject(Object); } for (const CbAttachment& Attachment : Attachments) { Attachment.Save(Writer); } Writer.AddNull(); } void CbPackage::Save(BinaryWriter& StreamWriter) const { CbWriter Writer; Save(Writer); Writer.Save(StreamWriter); } ////////////////////////////////////////////////////////////////////////// // // Legacy package serialization support // namespace legacy { void SaveCbAttachment(const CbAttachment& Attachment, CbWriter& Writer) { if (Attachment.IsObject()) { CbObject Object = Attachment.AsObject(); Writer.AddBinary(Object.GetBuffer()); if (Object) { Writer.AddObjectAttachment(Attachment.GetHash()); } } else if (Attachment.IsBinary()) { Writer.AddBinary(Attachment.AsBinary()); Writer.AddBinaryAttachment(Attachment.GetHash()); } else if (Attachment.IsCompressedBinary()) { Writer.AddBinary(Attachment.AsCompressedBinary().GetCompressed()); Writer.AddBinaryAttachment(Attachment.GetHash()); } else if (Attachment.IsNull()) { Writer.AddBinary(MemoryView()); } else { ZEN_NOT_IMPLEMENTED("Compressed binary is not supported in this serialization format"); } } void SaveCbPackage(const CbPackage& Package, CbWriter& Writer) { if (const CbObject& RootObject = Package.GetObject()) { Writer.AddObject(RootObject); Writer.AddObjectAttachment(Package.GetObjectHash()); } for (const CbAttachment& Attachment : Package.GetAttachments()) { SaveCbAttachment(Attachment, Writer); } Writer.AddNull(); } void SaveCbPackage(const CbPackage& Package, BinaryWriter& Ar) { CbWriter Writer; SaveCbPackage(Package, Writer); Writer.Save(Ar); } bool TryLoadCbPackage(CbPackage& Package, IoBuffer InBuffer, BufferAllocator Allocator, CbPackage::AttachmentResolver* Mapper) { BinaryReader Reader(InBuffer.Data(), InBuffer.Size()); return TryLoadCbPackage(Package, Reader, Allocator, Mapper); } bool TryLoadCbPackage(CbPackage& Package, BinaryReader& Reader, BufferAllocator Allocator, CbPackage::AttachmentResolver* Mapper) { Package = CbPackage(); for (;;) { CbField ValueField = LoadCompactBinary(Reader, Allocator); if (!ValueField) { return false; } if (ValueField.IsNull()) { return true; } if (ValueField.IsBinary()) { const MemoryView View = ValueField.AsBinaryView(); if (View.GetSize() > 0) { SharedBuffer Buffer = SharedBuffer::MakeView(View, ValueField.GetOuterBuffer()).MakeOwned(); CbField HashField = LoadCompactBinary(Reader, Allocator); const IoHash& Hash = HashField.AsAttachment(); if (HashField.HasError()) { return false; } IoHash RawHash; uint64_t RawSize; if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Buffer, RawHash, RawSize)) { if (RawHash != Hash) { return false; } Package.AddAttachment(CbAttachment(Compressed, Hash)); } else { if (IoHash::HashBuffer(Buffer) != Hash) { return false; } if (HashField.IsObjectAttachment()) { Package.AddAttachment(CbAttachment(CbObject(std::move(Buffer)), Hash)); } else { Package.AddAttachment(CbAttachment(CompositeBuffer(std::move(Buffer)), Hash)); } } } } else if (ValueField.IsHash()) { const IoHash Hash = ValueField.AsHash(); ZEN_ASSERT(Mapper); if (SharedBuffer AttachmentData = (*Mapper)(Hash)) { IoHash RawHash; uint64_t RawSize; if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(AttachmentData, RawHash, RawSize)) { if (RawHash != Hash) { return false; } Package.AddAttachment(CbAttachment(Compressed, Hash)); } else { const CbValidateError ValidationResult = ValidateCompactBinary(AttachmentData.GetView(), CbValidateMode::All); if (ValidationResult != CbValidateError::None) { return false; } Package.AddAttachment(CbAttachment(CbObject(std::move(AttachmentData)), Hash)); } } } else { CbObject Object = ValueField.AsObject(); if (ValueField.HasError()) { return false; } if (Object) { CbField HashField = LoadCompactBinary(Reader, Allocator); IoHash ObjectHash = HashField.AsObjectAttachment(); if (HashField.HasError() || Object.GetHash() != ObjectHash) { return false; } Package.SetObject(Object, ObjectHash); } } } } } // namespace legacy /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #if ZEN_WITH_TESTS void usonpackage_forcelink() { } TEST_CASE("usonpackage") { using namespace std::literals; const auto TestSaveLoadValidate = [&](const char* Test, const CbAttachment& Attachment) { ZEN_UNUSED(Test); CbWriter Writer; Attachment.Save(Writer); CbFieldIterator Fields = Writer.Save(); BinaryWriter StreamWriter; Attachment.Save(StreamWriter); CHECK(MakeMemoryView(StreamWriter).EqualBytes(Fields.GetRangeBuffer().GetView())); CHECK(ValidateCompactBinaryRange(MakeMemoryView(StreamWriter), CbValidateMode::All) == CbValidateError::None); CHECK(ValidateObjectAttachment(MakeMemoryView(StreamWriter), CbValidateMode::All) == CbValidateError::None); CbAttachment FromFields; FromFields.TryLoad(Fields); CHECK(!bool(Fields)); CHECK(FromFields == Attachment); CbAttachment FromArchive; BinaryReader Reader(MakeMemoryView(StreamWriter)); FromArchive.TryLoad(Reader); CHECK(Reader.CurrentOffset() == Reader.Size()); CHECK(FromArchive == Attachment); }; SUBCASE("Empty Attachment") { CbAttachment Attachment; CHECK(Attachment.IsNull()); CHECK_FALSE(bool(Attachment)); CHECK_FALSE(bool(Attachment.AsBinary())); CHECK_FALSE(bool(Attachment.AsObject())); CHECK_FALSE(Attachment.IsBinary()); CHECK_FALSE(Attachment.IsObject()); CHECK(Attachment.GetHash() == IoHash::Zero); } SUBCASE("Binary Attachment") { const SharedBuffer Buffer = SharedBuffer::Clone(MakeMemoryView({0, 1, 2, 3})); CbAttachment Attachment(Buffer); CHECK_FALSE(Attachment.IsNull()); CHECK(bool(Attachment)); CHECK(Attachment.AsBinary() == Buffer); CHECK_FALSE(bool(Attachment.AsObject())); CHECK(Attachment.IsBinary()); CHECK_FALSE(Attachment.IsObject()); CHECK(Attachment.GetHash() == IoHash::HashBuffer(Buffer)); TestSaveLoadValidate("Binary", Attachment); } SUBCASE("Object Attachment") { CbWriter Writer; Writer.BeginObject(); Writer << "Name"sv << 42; Writer.EndObject(); CbObject Object = Writer.Save().AsObject(); CbAttachment Attachment(Object); CHECK_FALSE(Attachment.IsNull()); CHECK(bool(Attachment)); CHECK(Attachment.AsBinary() == SharedBuffer()); CHECK(Attachment.AsObject().Equals(Object)); CHECK_FALSE(Attachment.IsBinary()); CHECK(Attachment.IsObject()); CHECK(Attachment.GetHash() == Object.GetHash()); TestSaveLoadValidate("Object", Attachment); } SUBCASE("Binary View") { const uint8_t Value[]{0, 1, 2, 3}; SharedBuffer Buffer = SharedBuffer::MakeView(MakeMemoryView(Value)); CbAttachment Attachment(Buffer); CHECK_FALSE(Attachment.IsNull()); CHECK(bool(Attachment)); CHECK(Attachment.AsBinary().GetView().EqualBytes(Buffer.GetView())); CHECK_FALSE(bool(Attachment.AsObject())); CHECK(Attachment.IsBinary()); CHECK_FALSE(Attachment.IsObject()); CHECK(Attachment.GetHash() == IoHash::HashBuffer(Buffer)); } SUBCASE("Object View") { CbWriter Writer; Writer.BeginObject(); Writer << "Name"sv << 42; Writer.EndObject(); CbObject Object = Writer.Save().AsObject(); CbObject ObjectView = CbObject::MakeView(Object); CbAttachment Attachment(ObjectView); CHECK_FALSE(Attachment.IsNull()); CHECK(bool(Attachment)); CHECK(Attachment.AsBinary() != ObjectView.GetBuffer()); CHECK(Attachment.AsObject().Equals(Object)); CHECK_FALSE(Attachment.IsBinary()); CHECK(Attachment.IsObject()); CHECK(Attachment.GetHash() == IoHash(Object.GetHash())); } SUBCASE("Binary Load from View") { const uint8_t Value[]{0, 1, 2, 3}; const SharedBuffer Buffer = SharedBuffer::MakeView(MakeMemoryView(Value)); CbAttachment Attachment(Buffer); CbWriter Writer; Attachment.Save(Writer); CbFieldIterator Fields = Writer.Save(); CbFieldIterator FieldsView = CbFieldIterator::MakeRangeView(CbFieldViewIterator(Fields)); Attachment.TryLoad(FieldsView); CHECK_FALSE(Attachment.IsNull()); CHECK(bool(Attachment)); CHECK_FALSE(FieldsView.GetRangeBuffer().GetView().Contains(Attachment.AsBinary().GetView())); CHECK(Attachment.AsBinary().GetView().EqualBytes(Buffer.GetView())); CHECK_FALSE(Attachment.AsObject()); CHECK(Attachment.IsBinary()); CHECK_FALSE(Attachment.IsObject()); CHECK(Attachment.GetHash() == IoHash::HashBuffer(MakeMemoryView(Value))); } SUBCASE("Object Load from View") { CbWriter ValueWriter; ValueWriter.BeginObject(); ValueWriter << "Name"sv << 42; ValueWriter.EndObject(); const CbObject Value = ValueWriter.Save().AsObject(); CHECK(ValidateCompactBinaryRange(Value.GetView(), CbValidateMode::All) == CbValidateError::None); CbAttachment Attachment(Value); CbWriter Writer; Attachment.Save(Writer); CbFieldIterator Fields = Writer.Save(); CbFieldIterator FieldsView = CbFieldIterator::MakeRangeView(CbFieldViewIterator(Fields)); Attachment.TryLoad(FieldsView); MemoryView View; CHECK_FALSE(Attachment.IsNull()); CHECK(bool(Attachment)); CHECK(Attachment.AsBinary().GetView().EqualBytes(MemoryView())); CHECK_FALSE((!Attachment.AsObject().TryGetSerializedView(View) || FieldsView.GetOuterBuffer().GetView().Contains(View))); CHECK_FALSE(Attachment.IsBinary()); CHECK(Attachment.IsObject()); CHECK(Attachment.GetHash() == Value.GetHash()); } SUBCASE("Binary Null") { const CbAttachment Attachment(SharedBuffer{}); CHECK(Attachment.IsNull()); CHECK_FALSE(Attachment.IsBinary()); CHECK_FALSE(Attachment.IsObject()); CHECK(Attachment.GetHash() == IoHash::Zero); } SUBCASE("Binary Empty") { const CbAttachment Attachment(UniqueBuffer::Alloc(0).MoveToShared()); CHECK_FALSE(Attachment.IsNull()); CHECK(Attachment.IsBinary()); CHECK_FALSE(Attachment.IsObject()); CHECK(Attachment.GetHash() == IoHash::HashBuffer(SharedBuffer{})); } SUBCASE("Object Empty") { const CbAttachment Attachment(CbObject{}); CHECK_FALSE(Attachment.IsNull()); CHECK_FALSE(Attachment.IsBinary()); CHECK(Attachment.IsObject()); CHECK(Attachment.GetHash() == CbObject().GetHash()); } } TEST_CASE("usonpackage.serialization") { using namespace std::literals; const auto TestSaveLoadValidate = [&](const char* Test, CbPackage& InOutPackage) { ZEN_UNUSED(Test); CbWriter Writer; InOutPackage.Save(Writer); CbFieldIterator Fields = Writer.Save(); BinaryWriter MemStream; InOutPackage.Save(MemStream); CHECK(MakeMemoryView(MemStream).EqualBytes(Fields.GetRangeBuffer().GetView())); CHECK(ValidateCompactBinaryRange(MakeMemoryView(MemStream), CbValidateMode::All) == CbValidateError::None); CHECK(ValidateCompactBinaryPackage(MakeMemoryView(MemStream), CbValidateMode::All) == CbValidateError::None); CbPackage FromFields; FromFields.TryLoad(Fields); CHECK_FALSE(bool(Fields)); CHECK(FromFields == InOutPackage); CbPackage FromArchive; BinaryReader ReadAr(MakeMemoryView(MemStream)); FromArchive.TryLoad(ReadAr); CHECK(ReadAr.CurrentOffset() == ReadAr.Size()); CHECK(FromArchive == InOutPackage); InOutPackage = FromArchive; }; SUBCASE("Empty") { CbPackage Package; CHECK(Package.IsNull()); CHECK_FALSE(bool(Package)); CHECK(Package.GetAttachments().size() == 0); TestSaveLoadValidate("Empty", Package); } SUBCASE("Object Only") { CbWriter Writer; Writer.BeginObject(); Writer << "Field" << 42; Writer.EndObject(); const CbObject Object = Writer.Save().AsObject(); CbPackage Package(Object); CHECK_FALSE(Package.IsNull()); CHECK(bool(Package)); CHECK(Package.GetAttachments().size() == 0); CHECK(Package.GetObject().GetOuterBuffer() == Object.GetOuterBuffer()); CHECK(Package.GetObject()["Field"].AsInt32() == 42); CHECK(Package.GetObjectHash() == Package.GetObject().GetHash()); TestSaveLoadValidate("Object", Package); } // Object View Only { CbWriter Writer; Writer.BeginObject(); Writer << "Field" << 42; Writer.EndObject(); const CbObject Object = Writer.Save().AsObject(); CbPackage Package(CbObject::MakeView(Object)); CHECK_FALSE(Package.IsNull()); CHECK(bool(Package)); CHECK(Package.GetAttachments().size() == 0); CHECK(Package.GetObject().GetOuterBuffer() != Object.GetOuterBuffer()); CHECK(Package.GetObject()["Field"].AsInt32() == 42); CHECK(Package.GetObjectHash() == Package.GetObject().GetHash()); TestSaveLoadValidate("Object", Package); } // Attachment Only { CbObject Object1; { CbWriter Writer; Writer.BeginObject(); Writer << "Field1" << 42; Writer.EndObject(); Object1 = Writer.Save().AsObject(); } CbObject Object2; { CbWriter Writer; Writer.BeginObject(); Writer << "Field2" << 42; Writer.EndObject(); Object2 = Writer.Save().AsObject(); } CbPackage Package; Package.AddAttachment(CbAttachment(Object1)); Package.AddAttachment(CbAttachment(Object2.GetBuffer())); CHECK_FALSE(Package.IsNull()); CHECK(bool(Package)); CHECK(Package.GetAttachments().size() == 2); CHECK(Package.GetObject().Equals(CbObject())); CHECK(Package.GetObjectHash() == IoHash::Zero); TestSaveLoadValidate("Attachments", Package); const CbAttachment* const Object1Attachment = Package.FindAttachment(Object1.GetHash()); const CbAttachment* const Object2Attachment = Package.FindAttachment(Object2.GetHash()); CHECK((Object1Attachment && Object1Attachment->AsObject().Equals(Object1))); CHECK((Object2Attachment && Object2Attachment->AsBinary().GetView().EqualBytes(Object2.GetBuffer().GetView()))); SharedBuffer Object1ClonedBuffer = SharedBuffer::Clone(Object1.GetOuterBuffer()); Package.AddAttachment(CbAttachment(Object1ClonedBuffer)); Package.AddAttachment(CbAttachment(CbObject::Clone(Object2))); CHECK(Package.GetAttachments().size() == 2); CHECK(Package.FindAttachment(Object1.GetHash()) == Object1Attachment); CHECK(Package.FindAttachment(Object2.GetHash()) == Object2Attachment); CHECK((Object1Attachment && Object1Attachment->AsBinary() == Object1ClonedBuffer)); CHECK((Object2Attachment && Object2Attachment->AsObject().Equals(Object2))); CHECK(std::is_sorted(begin(Package.GetAttachments()), end(Package.GetAttachments()))); } // Shared Values const uint8_t Level4Values[]{0, 1, 2, 3}; SharedBuffer Level4 = SharedBuffer::MakeView(MakeMemoryView(Level4Values)); const IoHash Level4Hash = IoHash::HashBuffer(Level4); CbObject Level3; { CbWriter Writer; Writer.BeginObject(); Writer.AddBinaryAttachment("Level4", Level4Hash); Writer.EndObject(); Level3 = Writer.Save().AsObject(); } const IoHash Level3Hash = Level3.GetHash(); CbObject Level2; { CbWriter Writer; Writer.BeginObject(); Writer.AddObjectAttachment("Level3", Level3Hash); Writer.EndObject(); Level2 = Writer.Save().AsObject(); } const IoHash Level2Hash = Level2.GetHash(); CbObject Level1; { CbWriter Writer; Writer.BeginObject(); Writer.AddObjectAttachment("Level2", Level2Hash); Writer.EndObject(); Level1 = Writer.Save().AsObject(); } const IoHash Level1Hash = Level1.GetHash(); const auto Resolver = [&Level2, &Level2Hash, &Level3, &Level3Hash, &Level4, &Level4Hash](const IoHash& Hash) -> SharedBuffer { return Hash == Level2Hash ? Level2.GetOuterBuffer() : Hash == Level3Hash ? Level3.GetOuterBuffer() : Hash == Level4Hash ? Level4 : SharedBuffer(); }; // Object + Attachments { CbPackage Package; Package.SetObject(Level1, Level1Hash, Resolver); CHECK_FALSE(Package.IsNull()); CHECK(bool(Package)); CHECK(Package.GetAttachments().size() == 3); CHECK(Package.GetObject().GetBuffer() == Level1.GetBuffer()); CHECK(Package.GetObjectHash() == Level1Hash); TestSaveLoadValidate("Object+Attachments", Package); const CbAttachment* const Level2Attachment = Package.FindAttachment(Level2Hash); const CbAttachment* const Level3Attachment = Package.FindAttachment(Level3Hash); const CbAttachment* const Level4Attachment = Package.FindAttachment(Level4Hash); CHECK((Level2Attachment && Level2Attachment->AsObject().Equals(Level2))); CHECK((Level3Attachment && Level3Attachment->AsObject().Equals(Level3))); REQUIRE(Level4Attachment); CHECK(Level4Attachment->AsBinary() != Level4); CHECK(Level4Attachment->AsBinary().GetView().EqualBytes(Level4.GetView())); CHECK(std::is_sorted(begin(Package.GetAttachments()), end(Package.GetAttachments()))); const CbPackage PackageCopy = Package; CHECK(PackageCopy == Package); CHECK(Package.RemoveAttachment(Level1Hash) == 0); CHECK(Package.RemoveAttachment(Level2Hash) == 1); CHECK(Package.RemoveAttachment(Level3Hash) == 1); CHECK(Package.RemoveAttachment(Level4Hash) == 1); CHECK(Package.RemoveAttachment(Level4Hash) == 0); CHECK(Package.GetAttachments().size() == 0); CHECK(PackageCopy != Package); Package = PackageCopy; CHECK(PackageCopy == Package); Package.SetObject(CbObject()); CHECK(PackageCopy != Package); CHECK(Package.GetObjectHash() == IoHash()); } // Out of Order { CbWriter Writer; CbAttachment Attachment2(Level2, Level2Hash); Attachment2.Save(Writer); CbAttachment Attachment4(Level4); Attachment4.Save(Writer); Writer.AddHash(Level1Hash); Writer.AddObject(Level1); CbAttachment Attachment3(Level3, Level3Hash); Attachment3.Save(Writer); Writer.AddNull(); CbFieldIterator Fields = Writer.Save(); CbPackage FromFields; FromFields.TryLoad(Fields); const CbAttachment* const Level2Attachment = FromFields.FindAttachment(Level2Hash); REQUIRE(Level2Attachment); const CbAttachment* const Level3Attachment = FromFields.FindAttachment(Level3Hash); REQUIRE(Level3Attachment); const CbAttachment* const Level4Attachment = FromFields.FindAttachment(Level4Hash); REQUIRE(Level4Attachment); CHECK(FromFields.GetObject().Equals(Level1)); CHECK(FromFields.GetObject().GetOuterBuffer() == Fields.GetOuterBuffer()); CHECK(FromFields.GetObjectHash() == Level1Hash); const MemoryView FieldsOuterBufferView = Fields.GetOuterBuffer().GetView(); CHECK(Level2Attachment->AsObject().Equals(Level2)); CHECK(Level2Attachment->GetHash() == Level2Hash); CHECK(Level3Attachment->AsObject().Equals(Level3)); CHECK(Level3Attachment->GetHash() == Level3Hash); CHECK(Level4Attachment->AsBinary().GetView().EqualBytes(Level4.GetView())); CHECK(FieldsOuterBufferView.Contains(Level4Attachment->AsBinary().GetView())); CHECK(Level4Attachment->GetHash() == Level4Hash); BinaryWriter WriteStream; Writer.Save(WriteStream); CbPackage FromArchive; BinaryReader ReadAr(MakeMemoryView(WriteStream)); FromArchive.TryLoad(ReadAr); Writer.Reset(); FromArchive.Save(Writer); CbFieldIterator Saved = Writer.Save(); CHECK(Saved.AsHash() == Level1Hash); ++Saved; CHECK(Saved.AsObject().Equals(Level1)); ++Saved; CHECK_EQ(Saved.AsObjectAttachment(), Level2Hash); ++Saved; CHECK(Saved.AsObject().Equals(Level2)); ++Saved; CHECK_EQ(Saved.AsObjectAttachment(), Level3Hash); ++Saved; CHECK(Saved.AsObject().Equals(Level3)); ++Saved; CHECK_EQ(Saved.AsBinaryAttachment(), Level4Hash); ++Saved; SharedBuffer SavedLevel4Buffer = SharedBuffer::MakeView(Saved.AsBinaryView()); CHECK(SavedLevel4Buffer.GetView().EqualBytes(Level4.GetView())); ++Saved; CHECK(Saved.IsNull()); ++Saved; CHECK(!Saved); } // Null Attachment { CbAttachment NullAttachment; CbPackage Package; Package.AddAttachment(NullAttachment); CHECK(Package.IsNull()); CHECK_FALSE(bool(Package)); CHECK(Package.GetAttachments().size() == 0); CHECK_FALSE(Package.FindAttachment(NullAttachment)); } // Resolve After Merge { bool bResolved = false; CbPackage Package; Package.AddAttachment(CbAttachment(Level3.GetBuffer())); Package.AddAttachment(CbAttachment(Level3), [&bResolved](const IoHash& Hash) -> SharedBuffer { ZEN_UNUSED(Hash); bResolved = true; return SharedBuffer(); }); CHECK(bResolved); } } TEST_CASE("usonpackage.invalidpackage") { const auto TestLoad = [](std::initializer_list RawData, BufferAllocator Allocator = UniqueBuffer::Alloc) { const MemoryView RawView = MakeMemoryView(RawData); CbPackage FromArchive; BinaryReader ReadAr(RawView); CHECK_FALSE(FromArchive.TryLoad(ReadAr, Allocator)); }; const auto AllocFail = [](uint64_t) -> UniqueBuffer { FAIL_CHECK("Allocation is not expected"); return UniqueBuffer(); }; SUBCASE("Empty") { TestLoad({}, AllocFail); } SUBCASE("Invalid Initial Field") { TestLoad({uint8_t(CbFieldType::None)}); TestLoad({uint8_t(CbFieldType::Array)}); TestLoad({uint8_t(CbFieldType::UniformArray)}); TestLoad({uint8_t(CbFieldType::Binary)}); TestLoad({uint8_t(CbFieldType::String)}); TestLoad({uint8_t(CbFieldType::IntegerPositive)}); TestLoad({uint8_t(CbFieldType::IntegerNegative)}); TestLoad({uint8_t(CbFieldType::Float32)}); TestLoad({uint8_t(CbFieldType::Float64)}); TestLoad({uint8_t(CbFieldType::BoolFalse)}); TestLoad({uint8_t(CbFieldType::BoolTrue)}); TestLoad({uint8_t(CbFieldType::ObjectAttachment)}); TestLoad({uint8_t(CbFieldType::BinaryAttachment)}); TestLoad({uint8_t(CbFieldType::Uuid)}); TestLoad({uint8_t(CbFieldType::DateTime)}); TestLoad({uint8_t(CbFieldType::TimeSpan)}); TestLoad({uint8_t(CbFieldType::ObjectId)}); TestLoad({uint8_t(CbFieldType::CustomById)}); TestLoad({uint8_t(CbFieldType::CustomByName)}); } SUBCASE("Size Out Of Bounds") { TestLoad({uint8_t(CbFieldType::Object), 1}, AllocFail); TestLoad({uint8_t(CbFieldType::Object), 0xff, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0}, AllocFail); } } #endif } // namespace zen