diff options
Diffstat (limited to 'src/zencore/compactbinarypackage.cpp')
| -rw-r--r-- | src/zencore/compactbinarypackage.cpp | 1350 |
1 files changed, 1350 insertions, 0 deletions
diff --git a/src/zencore/compactbinarypackage.cpp b/src/zencore/compactbinarypackage.cpp new file mode 100644 index 000000000..a4fa38a1d --- /dev/null +++ b/src/zencore/compactbinarypackage.cpp @@ -0,0 +1,1350 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencore/compactbinarypackage.h" +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryvalidation.h> +#include <zencore/endian.h> +#include <zencore/stream.h> +#include <zencore/testing.h> + +namespace zen { + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +CbAttachment::CbAttachment(const CompressedBuffer& InValue, const IoHash& Hash) : CbAttachment(InValue.MakeOwned(), Hash) +{ +} + +CbAttachment::CbAttachment(const SharedBuffer& InValue) : CbAttachment(CompositeBuffer(InValue)) +{ +} + +CbAttachment::CbAttachment(const SharedBuffer& InValue, const IoHash& InHash) : CbAttachment(CompositeBuffer(InValue), InHash) +{ +} + +CbAttachment::CbAttachment(const CompositeBuffer& InValue) +: Hash(InValue.IsNull() ? IoHash::Zero : IoHash::HashBuffer(InValue)) +, Value(InValue) +{ + if (std::get<CompositeBuffer>(Value).IsNull()) + { + Value.emplace<std::nullptr_t>(); + } +} + +CbAttachment::CbAttachment(CompositeBuffer&& InValue) +: Hash(InValue.IsNull() ? IoHash::Zero : IoHash::HashBuffer(InValue)) +, Value(std::move(InValue)) + +{ + if (std::get<CompositeBuffer>(Value).IsNull()) + { + Value.emplace<std::nullptr_t>(); + } +} + +CbAttachment::CbAttachment(CompositeBuffer&& InValue, const IoHash& InHash) : Hash(InHash), Value(InValue) +{ + if (std::get<CompositeBuffer>(Value).IsNull()) + { + Value.emplace<std::nullptr_t>(); + } +} + +CbAttachment::CbAttachment(CompressedBuffer&& InValue, const IoHash& InHash) : Hash(InHash), Value(InValue) +{ + if (std::get<CompressedBuffer>(Value).IsNull()) + { + Value.emplace<std::nullptr_t>(); + } +} + +CbAttachment::CbAttachment(const CbObject& InValue, const IoHash* const InHash) +{ + auto SetValue = [&](const CbObject& ValueToSet) { + if (InHash) + { + Value.emplace<CbObject>(ValueToSet); + Hash = *InHash; + } + else + { + Value.emplace<CbObject>(ValueToSet); + Hash = ValueToSet.GetHash(); + } + }; + + MemoryView View; + if (!InValue.IsOwned() || !InValue.TryGetSerializedView(View)) + { + SetValue(CbObject::Clone(InValue)); + } + else + { + SetValue(InValue); + } +} + +bool +CbAttachment::TryLoad(IoBuffer& InBuffer, BufferAllocator Allocator) +{ + BinaryReader Reader(InBuffer.Data(), InBuffer.Size()); + + return TryLoad(Reader, Allocator); +} + +bool +CbAttachment::TryLoad(CbFieldIterator& Fields) +{ + if (const CbObjectView ObjectView = Fields.AsObjectView(); !Fields.HasError()) + { + // Is a null object or object not prefixed with a precomputed hash value + Value.emplace<CbObject>(CbObject(ObjectView, Fields.GetOuterBuffer())); + Hash = ObjectView.GetHash(); + ++Fields; + } + else if (const IoHash ObjectAttachmentHash = Fields.AsObjectAttachment(); !Fields.HasError()) + { + // Is an object + ++Fields; + const CbObjectView InnerObjectView = Fields.AsObjectView(); + if (Fields.HasError()) + { + return false; + } + Value.emplace<CbObject>(CbObject(InnerObjectView, Fields.GetOuterBuffer())); + Hash = ObjectAttachmentHash; + ++Fields; + } + else if (const IoHash BinaryAttachmentHash = Fields.AsBinaryAttachment(); !Fields.HasError()) + { + // Is an uncompressed binary blob + ++Fields; + MemoryView BinaryView = Fields.AsBinaryView(); + if (Fields.HasError()) + { + return false; + } + Value.emplace<CompositeBuffer>(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())); + Hash = BinaryAttachmentHash; + ++Fields; + } + else if (MemoryView BinaryView = Fields.AsBinaryView(); !Fields.HasError()) + { + if (BinaryView.GetSize() > 0) + { + // Is a compressed binary blob + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = + CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer()), RawHash, RawSize).MakeOwned(); + Value.emplace<CompressedBuffer>(Compressed); + Hash = RawHash; + ++Fields; + } + else + { + // Is an uncompressed empty binary blob + Value.emplace<CompositeBuffer>(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())); + Hash = IoHash::HashBuffer(nullptr, 0); + ++Fields; + } + } + else + { + return false; + } + + return true; +} + +static bool +TryLoad_ArchiveFieldIntoAttachment(CbAttachment& TargetAttachment, CbField&& Field, BinaryReader& Reader, BufferAllocator Allocator) +{ + if (const CbObjectView ObjectView = Field.AsObjectView(); !Field.HasError()) + { + // Is a null object or object not prefixed with a precomputed hash value + TargetAttachment = CbAttachment(CbObject(ObjectView, std::move(Field)), ObjectView.GetHash()); + } + else if (const IoHash ObjectAttachmentHash = Field.AsObjectAttachment(); !Field.HasError()) + { + // Is an object + Field = LoadCompactBinary(Reader, Allocator); + if (!Field.IsObject()) + { + return false; + } + TargetAttachment = CbAttachment(std::move(Field).AsObject(), ObjectAttachmentHash); + } + else if (const IoHash BinaryAttachmentHash = Field.AsBinaryAttachment(); !Field.HasError()) + { + // Is an uncompressed binary blob + Field = LoadCompactBinary(Reader, Allocator); + SharedBuffer Buffer = Field.AsBinary(); + if (Field.HasError()) + { + return false; + } + TargetAttachment = CbAttachment(CompositeBuffer(Buffer), BinaryAttachmentHash); + } + else if (SharedBuffer Buffer = Field.AsBinary(); !Field.HasError()) + { + if (Buffer.GetSize() > 0) + { + // Is a compressed binary blob + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(std::move(Buffer), RawHash, RawSize); + TargetAttachment = CbAttachment(Compressed, RawHash); + } + else + { + // Is an uncompressed empty binary blob + TargetAttachment = CbAttachment(CompositeBuffer(Buffer), IoHash::HashBuffer(nullptr, 0)); + } + } + else + { + return false; + } + + return true; +} + +bool +CbAttachment::TryLoad(BinaryReader& Reader, BufferAllocator Allocator) +{ + CbField Field = LoadCompactBinary(Reader, Allocator); + return TryLoad_ArchiveFieldIntoAttachment(*this, std::move(Field), Reader, Allocator); +} + +void +CbAttachment::Save(CbWriter& Writer) const +{ + if (const CbObject* Object = std::get_if<CbObject>(&Value)) + { + if (*Object) + { + Writer.AddObjectAttachment(Hash); + } + Writer.AddObject(*Object); + } + else if (const CompositeBuffer* Binary = std::get_if<CompositeBuffer>(&Value)) + { + if (Binary->GetSize() > 0) + { + Writer.AddBinaryAttachment(Hash); + } + Writer.AddBinary(*Binary); + } + else if (const CompressedBuffer* Compressed = std::get_if<CompressedBuffer>(&Value)) + { + Writer.AddBinary(Compressed->GetCompressed()); + } +} + +void +CbAttachment::Save(BinaryWriter& Writer) const +{ + CbWriter TempWriter; + Save(TempWriter); + TempWriter.Save(Writer); +} + +bool +CbAttachment::IsNull() const +{ + return std::holds_alternative<std::nullptr_t>(Value); +} + +bool +CbAttachment::IsBinary() const +{ + return std::holds_alternative<CompositeBuffer>(Value); +} + +bool +CbAttachment::IsCompressedBinary() const +{ + return std::holds_alternative<CompressedBuffer>(Value); +} + +bool +CbAttachment::IsObject() const +{ + return std::holds_alternative<CbObject>(Value); +} + +IoHash +CbAttachment::GetHash() const +{ + return Hash; +} + +CompositeBuffer +CbAttachment::AsCompositeBinary() const +{ + if (const CompositeBuffer* BinValue = std::get_if<CompositeBuffer>(&Value)) + { + return *BinValue; + } + + return CompositeBuffer::Null; +} + +SharedBuffer +CbAttachment::AsBinary() const +{ + if (const CompositeBuffer* BinValue = std::get_if<CompositeBuffer>(&Value)) + { + return BinValue->Flatten(); + } + + return {}; +} + +CompressedBuffer +CbAttachment::AsCompressedBinary() const +{ + if (const CompressedBuffer* CompValue = std::get_if<CompressedBuffer>(&Value)) + { + return *CompValue; + } + + return CompressedBuffer::Null; +} + +/** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */ +CbObject +CbAttachment::AsObject() const +{ + if (const CbObject* ObjectValue = std::get_if<CbObject>(&Value)) + { + return *ObjectValue; + } + + return {}; +} + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +void +CbPackage::SetObject(CbObject InObject, const IoHash* InObjectHash, AttachmentResolver* InResolver) +{ + if (InObject) + { + Object = InObject.IsOwned() ? std::move(InObject) : CbObject::Clone(InObject); + if (InObjectHash) + { + ObjectHash = *InObjectHash; + ZEN_ASSERT_SLOW(ObjectHash == Object.GetHash()); + } + else + { + ObjectHash = Object.GetHash(); + } + if (InResolver) + { + GatherAttachments(Object, *InResolver); + } + } + else + { + Object.Reset(); + ObjectHash = IoHash::Zero; + } +} + +void +CbPackage::AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver) +{ + if (!Attachment.IsNull()) + { + auto It = std::lower_bound(begin(Attachments), end(Attachments), Attachment); + if (It != Attachments.end() && *It == Attachment) + { + CbAttachment& Existing = *It; + Existing = Attachment; + } + else + { + Attachments.insert(It, Attachment); + } + + if (Attachment.IsObject() && Resolver) + { + GatherAttachments(Attachment.AsObject(), *Resolver); + } + } +} + +void +CbPackage::AddAttachments(std::span<const CbAttachment> InAttachments) +{ + if (InAttachments.empty()) + { + return; + } + // Assume we have no duplicates! + Attachments.insert(Attachments.end(), InAttachments.begin(), InAttachments.end()); + std::sort(Attachments.begin(), Attachments.end()); + ZEN_ASSERT_SLOW(std::unique(Attachments.begin(), Attachments.end()) == Attachments.end()); +} + +int32_t +CbPackage::RemoveAttachment(const IoHash& Hash) +{ + return gsl::narrow_cast<int32_t>( + std::erase_if(Attachments, [&Hash](const CbAttachment& Attachment) -> bool { return Attachment.GetHash() == Hash; })); +} + +bool +CbPackage::Equals(const CbPackage& Package) const +{ + return ObjectHash == Package.ObjectHash && Attachments == Package.Attachments; +} + +const CbAttachment* +CbPackage::FindAttachment(const IoHash& Hash) const +{ + auto It = std::find_if(begin(Attachments), end(Attachments), [&Hash](const CbAttachment& Attachment) -> bool { + return Attachment.GetHash() == Hash; + }); + + if (It == end(Attachments)) + return nullptr; + + return &*It; +} + +void +CbPackage::GatherAttachments(const CbObject& Value, AttachmentResolver Resolver) +{ + Value.IterateAttachments([this, &Resolver](CbFieldView Field) { + const IoHash& Hash = Field.AsAttachment(); + + if (SharedBuffer Buffer = Resolver(Hash)) + { + if (Field.IsObjectAttachment()) + { + AddAttachment(CbAttachment(CbObject(std::move(Buffer)), Hash), &Resolver); + } + else + { + AddAttachment(CbAttachment(std::move(Buffer))); + } + } + }); +} + +bool +CbPackage::TryLoad(IoBuffer InBuffer, BufferAllocator Allocator, AttachmentResolver* Mapper) +{ + BinaryReader Reader(InBuffer.Data(), InBuffer.Size()); + + return TryLoad(Reader, Allocator, Mapper); +} + +bool +CbPackage::TryLoad(CbFieldIterator& Fields) +{ + *this = CbPackage(); + + while (Fields) + { + if (Fields.IsNull()) + { + ++Fields; + break; + } + else if (IoHash Hash = Fields.AsHash(); !Fields.HasError() && !Fields.IsAttachment()) + { + ++Fields; + CbObjectView ObjectView = Fields.AsObjectView(); + if (Fields.HasError() || Hash != ObjectView.GetHash()) + { + return false; + } + Object = CbObject(ObjectView, Fields.GetOuterBuffer()); + Object.MakeOwned(); + ObjectHash = Hash; + ++Fields; + } + else + { + CbAttachment Attachment; + if (!Attachment.TryLoad(Fields)) + { + return false; + } + AddAttachment(Attachment); + } + } + return true; +} + +bool +CbPackage::TryLoad(BinaryReader& Reader, BufferAllocator Allocator, AttachmentResolver* Mapper) +{ + // TODO: this needs to re-grow the ability to accept a reference to an attachment which is + // not embedded + + ZEN_UNUSED(Mapper); + +#if 1 + *this = CbPackage(); + for (;;) + { + CbField Field = LoadCompactBinary(Reader, Allocator); + if (!Field) + { + return false; + } + + if (Field.IsNull()) + { + return true; + } + else if (IoHash Hash = Field.AsHash(); !Field.HasError() && !Field.IsAttachment()) + { + Field = LoadCompactBinary(Reader, Allocator); + CbObjectView ObjectView = Field.AsObjectView(); + if (Field.HasError() || Hash != ObjectView.GetHash()) + { + return false; + } + Object = CbObject(ObjectView, Field.GetOuterBuffer()); + ObjectHash = Hash; + } + else + { + CbAttachment Attachment; + if (!TryLoad_ArchiveFieldIntoAttachment(Attachment, std::move(Field), Reader, Allocator)) + { + return false; + } + AddAttachment(Attachment); + } + } +#else + uint8_t StackBuffer[64]; + const auto StackAllocator = [&Allocator, &StackBuffer](uint64_t Size) -> UniqueBuffer { + if (Size <= sizeof(StackBuffer)) + { + return UniqueBuffer::MakeMutableView(StackBuffer, Size); + } + + return Allocator(Size); + }; + + *this = CbPackage(); + + for (;;) + { + CbField ValueField = LoadCompactBinary(Reader, StackAllocator); + if (!ValueField) + { + return false; + } + if (ValueField.IsNull()) + { + return true; + } + else if (ValueField.IsBinary()) + { + const MemoryView View = ValueField.AsBinaryView(); + if (View.GetSize() > 0) + { + SharedBuffer Buffer = SharedBuffer::MakeView(View, ValueField.GetOuterBuffer()).MakeOwned(); + CbField HashField = LoadCompactBinary(Reader, StackAllocator); + const IoHash& Hash = HashField.AsAttachment(); + ZEN_ASSERT(!HashField.HasError(), "Attachments must be a non-empty binary value with a content hash."); + if (HashField.IsObjectAttachment()) + { + AddAttachment(CbAttachment(CbObject(std::move(Buffer)), Hash)); + } + else + { + AddAttachment(CbAttachment(std::move(Buffer), Hash)); + } + } + } + else if (ValueField.IsHash()) + { + const IoHash Hash = ValueField.AsHash(); + + ZEN_ASSERT(Mapper); + + AddAttachment(CbAttachment((*Mapper)(Hash), Hash)); + } + else + { + Object = ValueField.AsObject(); + if (ValueField.HasError()) + { + return false; + } + Object.MakeOwned(); + if (Object) + { + CbField HashField = LoadCompactBinary(Reader, StackAllocator); + ObjectHash = HashField.AsObjectAttachment(); + if (HashField.HasError() || Object.GetHash() != ObjectHash) + { + return false; + } + } + else + { + Object.Reset(); + } + } + } +#endif +} + +void +CbPackage::Save(CbWriter& Writer) const +{ + if (Object) + { + Writer.AddHash(ObjectHash); + Writer.AddObject(Object); + } + for (const CbAttachment& Attachment : Attachments) + { + Attachment.Save(Writer); + } + Writer.AddNull(); +} + +void +CbPackage::Save(BinaryWriter& StreamWriter) const +{ + CbWriter Writer; + Save(Writer); + Writer.Save(StreamWriter); +} + +////////////////////////////////////////////////////////////////////////// +// +// Legacy package serialization support +// + +namespace legacy { + + void SaveCbAttachment(const CbAttachment& Attachment, CbWriter& Writer) + { + if (Attachment.IsObject()) + { + CbObject Object = Attachment.AsObject(); + Writer.AddBinary(Object.GetBuffer()); + if (Object) + { + Writer.AddObjectAttachment(Attachment.GetHash()); + } + } + else if (Attachment.IsBinary()) + { + Writer.AddBinary(Attachment.AsBinary()); + Writer.AddBinaryAttachment(Attachment.GetHash()); + } + else if (Attachment.IsCompressedBinary()) + { + Writer.AddBinary(Attachment.AsCompressedBinary().GetCompressed()); + Writer.AddBinaryAttachment(Attachment.GetHash()); + } + else if (Attachment.IsNull()) + { + Writer.AddBinary(MemoryView()); + } + else + { + ZEN_NOT_IMPLEMENTED("Compressed binary is not supported in this serialization format"); + } + } + + void SaveCbPackage(const CbPackage& Package, CbWriter& Writer) + { + if (const CbObject& RootObject = Package.GetObject()) + { + Writer.AddObject(RootObject); + Writer.AddObjectAttachment(Package.GetObjectHash()); + } + for (const CbAttachment& Attachment : Package.GetAttachments()) + { + SaveCbAttachment(Attachment, Writer); + } + Writer.AddNull(); + } + + void SaveCbPackage(const CbPackage& Package, BinaryWriter& Ar) + { + CbWriter Writer; + SaveCbPackage(Package, Writer); + Writer.Save(Ar); + } + + bool TryLoadCbPackage(CbPackage& Package, IoBuffer InBuffer, BufferAllocator Allocator, CbPackage::AttachmentResolver* Mapper) + { + BinaryReader Reader(InBuffer.Data(), InBuffer.Size()); + + return TryLoadCbPackage(Package, Reader, Allocator, Mapper); + } + + bool TryLoadCbPackage(CbPackage& Package, BinaryReader& Reader, BufferAllocator Allocator, CbPackage::AttachmentResolver* Mapper) + { + Package = CbPackage(); + for (;;) + { + CbField ValueField = LoadCompactBinary(Reader, Allocator); + if (!ValueField) + { + return false; + } + if (ValueField.IsNull()) + { + return true; + } + if (ValueField.IsBinary()) + { + const MemoryView View = ValueField.AsBinaryView(); + if (View.GetSize() > 0) + { + SharedBuffer Buffer = SharedBuffer::MakeView(View, ValueField.GetOuterBuffer()).MakeOwned(); + CbField HashField = LoadCompactBinary(Reader, Allocator); + const IoHash& Hash = HashField.AsAttachment(); + if (HashField.HasError()) + { + return false; + } + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Buffer, RawHash, RawSize)) + { + if (RawHash != Hash) + { + return false; + } + Package.AddAttachment(CbAttachment(Compressed, Hash)); + } + else + { + if (IoHash::HashBuffer(Buffer) != Hash) + { + return false; + } + if (HashField.IsObjectAttachment()) + { + Package.AddAttachment(CbAttachment(CbObject(std::move(Buffer)), Hash)); + } + else + { + Package.AddAttachment(CbAttachment(CompositeBuffer(std::move(Buffer)), Hash)); + } + } + } + } + else if (ValueField.IsHash()) + { + const IoHash Hash = ValueField.AsHash(); + + ZEN_ASSERT(Mapper); + if (SharedBuffer AttachmentData = (*Mapper)(Hash)) + { + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(AttachmentData, RawHash, RawSize)) + { + if (RawHash != Hash) + { + return false; + } + Package.AddAttachment(CbAttachment(Compressed, Hash)); + } + else + { + const CbValidateError ValidationResult = ValidateCompactBinary(AttachmentData.GetView(), CbValidateMode::All); + if (ValidationResult != CbValidateError::None) + { + return false; + } + Package.AddAttachment(CbAttachment(CbObject(std::move(AttachmentData)), Hash)); + } + } + } + else + { + CbObject Object = ValueField.AsObject(); + if (ValueField.HasError()) + { + return false; + } + + if (Object) + { + CbField HashField = LoadCompactBinary(Reader, Allocator); + IoHash ObjectHash = HashField.AsObjectAttachment(); + if (HashField.HasError() || Object.GetHash() != ObjectHash) + { + return false; + } + Package.SetObject(Object, ObjectHash); + } + } + } + } + +} // namespace legacy + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#if ZEN_WITH_TESTS + +void +usonpackage_forcelink() +{ +} + +TEST_CASE("usonpackage") +{ + using namespace std::literals; + + const auto TestSaveLoadValidate = [&](const char* Test, const CbAttachment& Attachment) { + ZEN_UNUSED(Test); + + CbWriter Writer; + Attachment.Save(Writer); + CbFieldIterator Fields = Writer.Save(); + + BinaryWriter StreamWriter; + Attachment.Save(StreamWriter); + + CHECK(MakeMemoryView(StreamWriter).EqualBytes(Fields.GetRangeBuffer().GetView())); + CHECK(ValidateCompactBinaryRange(MakeMemoryView(StreamWriter), CbValidateMode::All) == CbValidateError::None); + CHECK(ValidateObjectAttachment(MakeMemoryView(StreamWriter), CbValidateMode::All) == CbValidateError::None); + + CbAttachment FromFields; + FromFields.TryLoad(Fields); + CHECK(!bool(Fields)); + CHECK(FromFields == Attachment); + + CbAttachment FromArchive; + BinaryReader Reader(MakeMemoryView(StreamWriter)); + FromArchive.TryLoad(Reader); + CHECK(Reader.CurrentOffset() == Reader.Size()); + CHECK(FromArchive == Attachment); + }; + + SUBCASE("Empty Attachment") + { + CbAttachment Attachment; + CHECK(Attachment.IsNull()); + CHECK_FALSE(bool(Attachment)); + CHECK_FALSE(bool(Attachment.AsBinary())); + CHECK_FALSE(bool(Attachment.AsObject())); + CHECK_FALSE(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsObject()); + CHECK(Attachment.GetHash() == IoHash::Zero); + } + + SUBCASE("Binary Attachment") + { + const SharedBuffer Buffer = SharedBuffer::Clone(MakeMemoryView<uint8_t>({0, 1, 2, 3})); + CbAttachment Attachment(Buffer); + CHECK_FALSE(Attachment.IsNull()); + CHECK(bool(Attachment)); + CHECK(Attachment.AsBinary() == Buffer); + CHECK_FALSE(bool(Attachment.AsObject())); + CHECK(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsObject()); + CHECK(Attachment.GetHash() == IoHash::HashBuffer(Buffer)); + TestSaveLoadValidate("Binary", Attachment); + } + + SUBCASE("Object Attachment") + { + CbWriter Writer; + Writer.BeginObject(); + Writer << "Name"sv << 42; + Writer.EndObject(); + CbObject Object = Writer.Save().AsObject(); + CbAttachment Attachment(Object); + + CHECK_FALSE(Attachment.IsNull()); + CHECK(bool(Attachment)); + CHECK(Attachment.AsBinary() == SharedBuffer()); + CHECK(Attachment.AsObject().Equals(Object)); + CHECK_FALSE(Attachment.IsBinary()); + CHECK(Attachment.IsObject()); + CHECK(Attachment.GetHash() == Object.GetHash()); + TestSaveLoadValidate("Object", Attachment); + } + + SUBCASE("Binary View") + { + const uint8_t Value[]{0, 1, 2, 3}; + SharedBuffer Buffer = SharedBuffer::MakeView(MakeMemoryView(Value)); + CbAttachment Attachment(Buffer); + CHECK_FALSE(Attachment.IsNull()); + CHECK(bool(Attachment)); + CHECK(Attachment.AsBinary().GetView().EqualBytes(Buffer.GetView())); + CHECK_FALSE(bool(Attachment.AsObject())); + CHECK(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsObject()); + CHECK(Attachment.GetHash() == IoHash::HashBuffer(Buffer)); + } + + SUBCASE("Object View") + { + CbWriter Writer; + Writer.BeginObject(); + Writer << "Name"sv << 42; + Writer.EndObject(); + CbObject Object = Writer.Save().AsObject(); + CbObject ObjectView = CbObject::MakeView(Object); + CbAttachment Attachment(ObjectView); + + CHECK_FALSE(Attachment.IsNull()); + CHECK(bool(Attachment)); + + CHECK(Attachment.AsBinary() != ObjectView.GetBuffer()); + CHECK(Attachment.AsObject().Equals(Object)); + CHECK_FALSE(Attachment.IsBinary()); + CHECK(Attachment.IsObject()); + CHECK(Attachment.GetHash() == IoHash(Object.GetHash())); + } + + SUBCASE("Binary Load from View") + { + const uint8_t Value[]{0, 1, 2, 3}; + const SharedBuffer Buffer = SharedBuffer::MakeView(MakeMemoryView(Value)); + CbAttachment Attachment(Buffer); + + CbWriter Writer; + Attachment.Save(Writer); + CbFieldIterator Fields = Writer.Save(); + CbFieldIterator FieldsView = CbFieldIterator::MakeRangeView(CbFieldViewIterator(Fields)); + Attachment.TryLoad(FieldsView); + + CHECK_FALSE(Attachment.IsNull()); + CHECK(bool(Attachment)); + CHECK_FALSE(FieldsView.GetRangeBuffer().GetView().Contains(Attachment.AsBinary().GetView())); + CHECK(Attachment.AsBinary().GetView().EqualBytes(Buffer.GetView())); + CHECK_FALSE(Attachment.AsObject()); + CHECK(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsObject()); + CHECK(Attachment.GetHash() == IoHash::HashBuffer(MakeMemoryView(Value))); + } + + SUBCASE("Object Load from View") + { + CbWriter ValueWriter; + ValueWriter.BeginObject(); + ValueWriter << "Name"sv << 42; + ValueWriter.EndObject(); + const CbObject Value = ValueWriter.Save().AsObject(); + + CHECK(ValidateCompactBinaryRange(Value.GetView(), CbValidateMode::All) == CbValidateError::None); + CbAttachment Attachment(Value); + + CbWriter Writer; + Attachment.Save(Writer); + CbFieldIterator Fields = Writer.Save(); + CbFieldIterator FieldsView = CbFieldIterator::MakeRangeView(CbFieldViewIterator(Fields)); + + Attachment.TryLoad(FieldsView); + MemoryView View; + + CHECK_FALSE(Attachment.IsNull()); + CHECK(bool(Attachment)); + CHECK(Attachment.AsBinary().GetView().EqualBytes(MemoryView())); + CHECK_FALSE((!Attachment.AsObject().TryGetSerializedView(View) || FieldsView.GetOuterBuffer().GetView().Contains(View))); + CHECK_FALSE(Attachment.IsBinary()); + CHECK(Attachment.IsObject()); + CHECK(Attachment.GetHash() == Value.GetHash()); + } + + SUBCASE("Binary Null") + { + const CbAttachment Attachment(SharedBuffer{}); + + CHECK(Attachment.IsNull()); + CHECK_FALSE(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsObject()); + CHECK(Attachment.GetHash() == IoHash::Zero); + } + + SUBCASE("Binary Empty") + { + const CbAttachment Attachment(UniqueBuffer::Alloc(0).MoveToShared()); + + CHECK_FALSE(Attachment.IsNull()); + CHECK(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsObject()); + CHECK(Attachment.GetHash() == IoHash::HashBuffer(SharedBuffer{})); + } + + SUBCASE("Object Empty") + { + const CbAttachment Attachment(CbObject{}); + + CHECK_FALSE(Attachment.IsNull()); + CHECK_FALSE(Attachment.IsBinary()); + CHECK(Attachment.IsObject()); + CHECK(Attachment.GetHash() == CbObject().GetHash()); + } +} + +TEST_CASE("usonpackage.serialization") +{ + using namespace std::literals; + + const auto TestSaveLoadValidate = [&](const char* Test, CbPackage& InOutPackage) { + ZEN_UNUSED(Test); + + CbWriter Writer; + InOutPackage.Save(Writer); + CbFieldIterator Fields = Writer.Save(); + + BinaryWriter MemStream; + InOutPackage.Save(MemStream); + + CHECK(MakeMemoryView(MemStream).EqualBytes(Fields.GetRangeBuffer().GetView())); + CHECK(ValidateCompactBinaryRange(MakeMemoryView(MemStream), CbValidateMode::All) == CbValidateError::None); + CHECK(ValidateCompactBinaryPackage(MakeMemoryView(MemStream), CbValidateMode::All) == CbValidateError::None); + + CbPackage FromFields; + FromFields.TryLoad(Fields); + CHECK_FALSE(bool(Fields)); + CHECK(FromFields == InOutPackage); + + CbPackage FromArchive; + BinaryReader ReadAr(MakeMemoryView(MemStream)); + FromArchive.TryLoad(ReadAr); + CHECK(ReadAr.CurrentOffset() == ReadAr.Size()); + CHECK(FromArchive == InOutPackage); + InOutPackage = FromArchive; + }; + + SUBCASE("Empty") + { + CbPackage Package; + CHECK(Package.IsNull()); + CHECK_FALSE(bool(Package)); + CHECK(Package.GetAttachments().size() == 0); + TestSaveLoadValidate("Empty", Package); + } + + SUBCASE("Object Only") + { + CbWriter Writer; + Writer.BeginObject(); + Writer << "Field" << 42; + Writer.EndObject(); + + const CbObject Object = Writer.Save().AsObject(); + CbPackage Package(Object); + CHECK_FALSE(Package.IsNull()); + CHECK(bool(Package)); + CHECK(Package.GetAttachments().size() == 0); + CHECK(Package.GetObject().GetOuterBuffer() == Object.GetOuterBuffer()); + CHECK(Package.GetObject()["Field"].AsInt32() == 42); + CHECK(Package.GetObjectHash() == Package.GetObject().GetHash()); + TestSaveLoadValidate("Object", Package); + } + + // Object View Only + { + CbWriter Writer; + Writer.BeginObject(); + Writer << "Field" << 42; + Writer.EndObject(); + + const CbObject Object = Writer.Save().AsObject(); + CbPackage Package(CbObject::MakeView(Object)); + CHECK_FALSE(Package.IsNull()); + CHECK(bool(Package)); + CHECK(Package.GetAttachments().size() == 0); + CHECK(Package.GetObject().GetOuterBuffer() != Object.GetOuterBuffer()); + CHECK(Package.GetObject()["Field"].AsInt32() == 42); + CHECK(Package.GetObjectHash() == Package.GetObject().GetHash()); + TestSaveLoadValidate("Object", Package); + } + + // Attachment Only + { + CbObject Object1; + { + CbWriter Writer; + Writer.BeginObject(); + Writer << "Field1" << 42; + Writer.EndObject(); + Object1 = Writer.Save().AsObject(); + } + CbObject Object2; + { + CbWriter Writer; + Writer.BeginObject(); + Writer << "Field2" << 42; + Writer.EndObject(); + Object2 = Writer.Save().AsObject(); + } + + CbPackage Package; + Package.AddAttachment(CbAttachment(Object1)); + Package.AddAttachment(CbAttachment(Object2.GetBuffer())); + + CHECK_FALSE(Package.IsNull()); + CHECK(bool(Package)); + CHECK(Package.GetAttachments().size() == 2); + CHECK(Package.GetObject().Equals(CbObject())); + CHECK(Package.GetObjectHash() == IoHash::Zero); + TestSaveLoadValidate("Attachments", Package); + + const CbAttachment* const Object1Attachment = Package.FindAttachment(Object1.GetHash()); + const CbAttachment* const Object2Attachment = Package.FindAttachment(Object2.GetHash()); + + CHECK((Object1Attachment && Object1Attachment->AsObject().Equals(Object1))); + CHECK((Object2Attachment && Object2Attachment->AsBinary().GetView().EqualBytes(Object2.GetBuffer().GetView()))); + + SharedBuffer Object1ClonedBuffer = SharedBuffer::Clone(Object1.GetOuterBuffer()); + Package.AddAttachment(CbAttachment(Object1ClonedBuffer)); + Package.AddAttachment(CbAttachment(CbObject::Clone(Object2))); + + CHECK(Package.GetAttachments().size() == 2); + CHECK(Package.FindAttachment(Object1.GetHash()) == Object1Attachment); + CHECK(Package.FindAttachment(Object2.GetHash()) == Object2Attachment); + + CHECK((Object1Attachment && Object1Attachment->AsBinary() == Object1ClonedBuffer)); + CHECK((Object2Attachment && Object2Attachment->AsObject().Equals(Object2))); + + CHECK(std::is_sorted(begin(Package.GetAttachments()), end(Package.GetAttachments()))); + } + + // Shared Values + const uint8_t Level4Values[]{0, 1, 2, 3}; + SharedBuffer Level4 = SharedBuffer::MakeView(MakeMemoryView(Level4Values)); + const IoHash Level4Hash = IoHash::HashBuffer(Level4); + + CbObject Level3; + { + CbWriter Writer; + Writer.BeginObject(); + Writer.AddBinaryAttachment("Level4", Level4Hash); + Writer.EndObject(); + Level3 = Writer.Save().AsObject(); + } + const IoHash Level3Hash = Level3.GetHash(); + + CbObject Level2; + { + CbWriter Writer; + Writer.BeginObject(); + Writer.AddObjectAttachment("Level3", Level3Hash); + Writer.EndObject(); + Level2 = Writer.Save().AsObject(); + } + const IoHash Level2Hash = Level2.GetHash(); + + CbObject Level1; + { + CbWriter Writer; + Writer.BeginObject(); + Writer.AddObjectAttachment("Level2", Level2Hash); + Writer.EndObject(); + Level1 = Writer.Save().AsObject(); + } + const IoHash Level1Hash = Level1.GetHash(); + + const auto Resolver = [&Level2, &Level2Hash, &Level3, &Level3Hash, &Level4, &Level4Hash](const IoHash& Hash) -> SharedBuffer { + return Hash == Level2Hash ? Level2.GetOuterBuffer() + : Hash == Level3Hash ? Level3.GetOuterBuffer() + : Hash == Level4Hash ? Level4 + : SharedBuffer(); + }; + + // Object + Attachments + { + CbPackage Package; + Package.SetObject(Level1, Level1Hash, Resolver); + + CHECK_FALSE(Package.IsNull()); + CHECK(bool(Package)); + CHECK(Package.GetAttachments().size() == 3); + CHECK(Package.GetObject().GetBuffer() == Level1.GetBuffer()); + CHECK(Package.GetObjectHash() == Level1Hash); + TestSaveLoadValidate("Object+Attachments", Package); + + const CbAttachment* const Level2Attachment = Package.FindAttachment(Level2Hash); + const CbAttachment* const Level3Attachment = Package.FindAttachment(Level3Hash); + const CbAttachment* const Level4Attachment = Package.FindAttachment(Level4Hash); + CHECK((Level2Attachment && Level2Attachment->AsObject().Equals(Level2))); + CHECK((Level3Attachment && Level3Attachment->AsObject().Equals(Level3))); + REQUIRE(Level4Attachment); + CHECK(Level4Attachment->AsBinary() != Level4); + CHECK(Level4Attachment->AsBinary().GetView().EqualBytes(Level4.GetView())); + + CHECK(std::is_sorted(begin(Package.GetAttachments()), end(Package.GetAttachments()))); + + const CbPackage PackageCopy = Package; + CHECK(PackageCopy == Package); + + CHECK(Package.RemoveAttachment(Level1Hash) == 0); + CHECK(Package.RemoveAttachment(Level2Hash) == 1); + CHECK(Package.RemoveAttachment(Level3Hash) == 1); + CHECK(Package.RemoveAttachment(Level4Hash) == 1); + CHECK(Package.RemoveAttachment(Level4Hash) == 0); + CHECK(Package.GetAttachments().size() == 0); + + CHECK(PackageCopy != Package); + Package = PackageCopy; + CHECK(PackageCopy == Package); + Package.SetObject(CbObject()); + CHECK(PackageCopy != Package); + CHECK(Package.GetObjectHash() == IoHash()); + } + + // Out of Order + { + CbWriter Writer; + CbAttachment Attachment2(Level2, Level2Hash); + Attachment2.Save(Writer); + CbAttachment Attachment4(Level4); + Attachment4.Save(Writer); + Writer.AddHash(Level1Hash); + Writer.AddObject(Level1); + CbAttachment Attachment3(Level3, Level3Hash); + Attachment3.Save(Writer); + Writer.AddNull(); + + CbFieldIterator Fields = Writer.Save(); + CbPackage FromFields; + FromFields.TryLoad(Fields); + + const CbAttachment* const Level2Attachment = FromFields.FindAttachment(Level2Hash); + REQUIRE(Level2Attachment); + const CbAttachment* const Level3Attachment = FromFields.FindAttachment(Level3Hash); + REQUIRE(Level3Attachment); + const CbAttachment* const Level4Attachment = FromFields.FindAttachment(Level4Hash); + REQUIRE(Level4Attachment); + + CHECK(FromFields.GetObject().Equals(Level1)); + CHECK(FromFields.GetObject().GetOuterBuffer() == Fields.GetOuterBuffer()); + CHECK(FromFields.GetObjectHash() == Level1Hash); + + const MemoryView FieldsOuterBufferView = Fields.GetOuterBuffer().GetView(); + + CHECK(Level2Attachment->AsObject().Equals(Level2)); + CHECK(Level2Attachment->GetHash() == Level2Hash); + + CHECK(Level3Attachment->AsObject().Equals(Level3)); + CHECK(Level3Attachment->GetHash() == Level3Hash); + + CHECK(Level4Attachment->AsBinary().GetView().EqualBytes(Level4.GetView())); + CHECK(FieldsOuterBufferView.Contains(Level4Attachment->AsBinary().GetView())); + CHECK(Level4Attachment->GetHash() == Level4Hash); + + BinaryWriter WriteStream; + Writer.Save(WriteStream); + CbPackage FromArchive; + BinaryReader ReadAr(MakeMemoryView(WriteStream)); + FromArchive.TryLoad(ReadAr); + + Writer.Reset(); + FromArchive.Save(Writer); + CbFieldIterator Saved = Writer.Save(); + + CHECK(Saved.AsHash() == Level1Hash); + ++Saved; + CHECK(Saved.AsObject().Equals(Level1)); + ++Saved; + CHECK_EQ(Saved.AsObjectAttachment(), Level2Hash); + ++Saved; + CHECK(Saved.AsObject().Equals(Level2)); + ++Saved; + CHECK_EQ(Saved.AsObjectAttachment(), Level3Hash); + ++Saved; + CHECK(Saved.AsObject().Equals(Level3)); + ++Saved; + CHECK_EQ(Saved.AsBinaryAttachment(), Level4Hash); + ++Saved; + SharedBuffer SavedLevel4Buffer = SharedBuffer::MakeView(Saved.AsBinaryView()); + CHECK(SavedLevel4Buffer.GetView().EqualBytes(Level4.GetView())); + ++Saved; + CHECK(Saved.IsNull()); + ++Saved; + CHECK(!Saved); + } + + // Null Attachment + { + CbAttachment NullAttachment; + CbPackage Package; + Package.AddAttachment(NullAttachment); + CHECK(Package.IsNull()); + CHECK_FALSE(bool(Package)); + CHECK(Package.GetAttachments().size() == 0); + CHECK_FALSE(Package.FindAttachment(NullAttachment)); + } + + // Resolve After Merge + { + bool bResolved = false; + CbPackage Package; + Package.AddAttachment(CbAttachment(Level3.GetBuffer())); + Package.AddAttachment(CbAttachment(Level3), [&bResolved](const IoHash& Hash) -> SharedBuffer { + ZEN_UNUSED(Hash); + bResolved = true; + return SharedBuffer(); + }); + CHECK(bResolved); + } +} + +TEST_CASE("usonpackage.invalidpackage") +{ + const auto TestLoad = [](std::initializer_list<uint8_t> RawData, BufferAllocator Allocator = UniqueBuffer::Alloc) { + const MemoryView RawView = MakeMemoryView(RawData); + CbPackage FromArchive; + BinaryReader ReadAr(RawView); + CHECK_FALSE(FromArchive.TryLoad(ReadAr, Allocator)); + }; + const auto AllocFail = [](uint64_t) -> UniqueBuffer { + FAIL_CHECK("Allocation is not expected"); + return UniqueBuffer(); + }; + SUBCASE("Empty") { TestLoad({}, AllocFail); } + SUBCASE("Invalid Initial Field") + { + TestLoad({uint8_t(CbFieldType::None)}); + TestLoad({uint8_t(CbFieldType::Array)}); + TestLoad({uint8_t(CbFieldType::UniformArray)}); + TestLoad({uint8_t(CbFieldType::Binary)}); + TestLoad({uint8_t(CbFieldType::String)}); + TestLoad({uint8_t(CbFieldType::IntegerPositive)}); + TestLoad({uint8_t(CbFieldType::IntegerNegative)}); + TestLoad({uint8_t(CbFieldType::Float32)}); + TestLoad({uint8_t(CbFieldType::Float64)}); + TestLoad({uint8_t(CbFieldType::BoolFalse)}); + TestLoad({uint8_t(CbFieldType::BoolTrue)}); + TestLoad({uint8_t(CbFieldType::ObjectAttachment)}); + TestLoad({uint8_t(CbFieldType::BinaryAttachment)}); + TestLoad({uint8_t(CbFieldType::Uuid)}); + TestLoad({uint8_t(CbFieldType::DateTime)}); + TestLoad({uint8_t(CbFieldType::TimeSpan)}); + TestLoad({uint8_t(CbFieldType::ObjectId)}); + TestLoad({uint8_t(CbFieldType::CustomById)}); + TestLoad({uint8_t(CbFieldType::CustomByName)}); + } + SUBCASE("Size Out Of Bounds") + { + TestLoad({uint8_t(CbFieldType::Object), 1}, AllocFail); + TestLoad({uint8_t(CbFieldType::Object), 0xff, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0}, AllocFail); + } +} + +#endif + +} // namespace zen |