From 709c2f8afebe9151aa947ec53ab801c2e5be4746 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 2 Sep 2021 13:01:52 +0200 Subject: Introduced support for compressed buffer attachments --- zencore/compactbinarypackage.cpp | 251 +++++++++++++++++++++++++++------------ 1 file changed, 173 insertions(+), 78 deletions(-) (limited to 'zencore/compactbinarypackage.cpp') diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp index 92bb5d411..c4df54616 100644 --- a/zencore/compactbinarypackage.cpp +++ b/zencore/compactbinarypackage.cpp @@ -12,47 +12,49 @@ namespace zen { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CbAttachment::CbAttachment(CbObject InValue, const IoHash* const InHash) +CbAttachment::CbAttachment(const CompressedBuffer& InValue) : CbAttachment(InValue.MakeOwned()) { - MemoryView View; - if (!InValue.IsOwned() || !InValue.TryGetSerializedView(View)) - { - InValue = CbObject::Clone(InValue); - } +} - Object = InValue.AsFieldView(); - Buffer = std::move(InValue).GetBuffer(); +CbAttachment::CbAttachment(CompressedBuffer&& InValue) +{ + Value.emplace(std::move(InValue).MakeOwned()); +} - if (InHash) - { - Hash = *InHash; - } - else - { - Hash = IoHash::HashMemory(Buffer); - } +CbAttachment::CbAttachment(const SharedBuffer& InValue) +: CbAttachment(InValue.IsNull() ? CompressedBuffer() + : CompressedBuffer::Compress(InValue, OodleCompressor::NotSet, OodleCompressionLevel::None)) +{ } -CbAttachment::CbAttachment(SharedBuffer InBuffer, const IoHash* const InHash) : Buffer(std::move(InBuffer).MakeOwned()) +CbAttachment::CbAttachment(const SharedBuffer& InValue, [[maybe_unused]] const IoHash& InHash) +: CbAttachment(InValue.IsNull() ? CompressedBuffer() + : CompressedBuffer::Compress(InValue, OodleCompressor::NotSet, OodleCompressionLevel::None)) { - if (InHash) - { - Hash = *InHash; - if (Buffer.GetSize()) - { - // This is disabled for now as it forces disk-based attachments to get mapped which - // then prevents us from making them delete themselves on close + // This could be more efficient, and should at the very least try to validate the hash +} - // ZEN_ASSERT_SLOW(Hash == IoHash::HashMemory(Buffer.GetData(), Buffer.GetSize())); +CbAttachment::CbAttachment(const CbObject& InValue, const IoHash* const InHash) +{ + auto SetValue = [&](const CbObject& ValueToSet) { + if (InHash) + { + Value.emplace(ValueToSet, *InHash); } else { - ZEN_ASSERT_SLOW(Hash == IoHash::Zero, TEXT("A null or empty buffer must use a hash of zero.")); + Value.emplace(ValueToSet, ValueToSet.GetHash()); } + }; + + MemoryView View; + if (!InValue.IsOwned() || !InValue.TryGetSerializedView(View)) + { + SetValue(CbObject::Clone(InValue)); } else { - Hash = IoHash::HashMemory(Buffer.GetData(), Buffer.GetSize()); + SetValue(InValue); } } @@ -68,33 +70,39 @@ CbAttachment::TryLoad(IoBuffer& InBuffer, BufferAllocator Allocator) bool CbAttachment::TryLoad(CbFieldIterator& Fields) { - const MemoryView View = Fields.AsBinaryView(); + const CbObjectView ObjectView = Fields.AsObjectView(); if (Fields.HasError()) { - return false; - } - - if (View.GetSize() > 0) - { - Buffer = SharedBuffer::MakeView(View, Fields.GetOuterBuffer()).MakeOwned(); - ++Fields; - Hash = Fields.AsAttachment(); - if (Fields.HasError()) + // Is a buffer + const MemoryView BinaryView = Fields.AsBinaryView(); + if (BinaryView.GetSize() > 0) { - return false; + Value.emplace( + CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())).MakeOwned()); + + ++Fields; } - if (Fields.IsObjectAttachment()) + else { - Object = CbFieldView(Buffer.GetData()); + ++Fields; + Value.emplace(); } - ++Fields; } else { + // It's an object ++Fields; - Buffer.Reset(); - Object = CbFieldView(); - Hash = IoHash::Zero; + IoHash Hash; + if (ObjectView) + { + Hash = Fields.AsObjectAttachment(); + ++Fields; + } + else + { + Hash = IoHash::HashMemory(MemoryView{}); + } + Value.emplace(CbObject(ObjectView, Fields->GetOuterBuffer()), Hash); } return true; @@ -103,60 +111,73 @@ CbAttachment::TryLoad(CbFieldIterator& Fields) bool CbAttachment::TryLoad(BinaryReader& Reader, BufferAllocator Allocator) { - CbField BufferField = LoadCompactBinary(Reader, Allocator); - const MemoryView View = BufferField.AsBinaryView(); - if (BufferField.HasError()) - { - return false; - } - if (View.GetSize() > 0) - { - Buffer = SharedBuffer::MakeView(View, BufferField.GetOuterBuffer()).MakeOwned(); - Object = CbFieldView(); + CbField Field = LoadCompactBinary(Reader, Allocator); + const CbObjectView ObjectView = Field.AsObjectView(); - std::vector HashBuffer; - CbField HashField = LoadCompactBinary(Reader, [&HashBuffer](uint64_t Size) -> UniqueBuffer { - HashBuffer.resize(Size); - return UniqueBuffer::MakeMutableView(HashBuffer.data(), Size); - }); - Hash = HashField.AsAttachment(); - if (HashField.HasError() || IoHash::HashMemory(Buffer) != Hash) + if (Field.HasError()) + { + // It's a buffer + const MemoryView BinaryView = Field.AsBinaryView(); + if (BinaryView.GetSize() > 0) { - return false; + Value.emplace( + CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Field.GetOuterBuffer())).MakeOwned()); } - if (HashField.IsObjectAttachment()) + else { - Object = CbFieldView(Buffer.GetData()); + Value.emplace(); } } else { - Buffer.Reset(); - Object = CbFieldView(); - Hash = IoHash::Zero; + // It's an object + IoHash Hash; + if (ObjectView) + { + std::vector HashBuffer; + CbField HashField = LoadCompactBinary(Reader, [&HashBuffer](uint64_t Size) -> UniqueBuffer { + HashBuffer.resize(Size); + return UniqueBuffer::MakeMutableView(HashBuffer.data(), Size); + }); + Hash = HashField.AsAttachment(); + if (HashField.HasError() || ObjectView.GetHash() != Hash) + { + // Error + return false; + } + } + else + { + Hash = IoHash::HashMemory(MemoryView()); + } + Value.emplace(CbObject(ObjectView, Field.GetOuterBuffer()), Hash); } + return true; } void CbAttachment::Save(CbWriter& Writer) const { - if (Object.IsObject()) + if (const CbObjectValue* ObjectValue = std::get_if(&Value)) { - Writer.AddBinary(Buffer); - if (CbFieldView(Object).AsObjectView()) + Writer.AddObject(ObjectValue->Object); + if (ObjectValue->Object) { - Writer.AddObjectAttachment(Hash); + Writer.AddObjectAttachment(ObjectValue->Hash); } } - else if (Buffer && Buffer.GetSize()) - { - Writer.AddBinary(Buffer); - Writer.AddBinaryAttachment(Hash); - } - else // Null + else { - Writer.AddBinary(MemoryView()); + const CompressedBuffer& BufferValue = std::get(Value); + if (BufferValue.GetRawSize()) + { + Writer.AddBinary(BufferValue.GetCompressed()); + } + else // Null + { + Writer.AddBinary(MemoryView()); + } } } @@ -168,6 +189,80 @@ CbAttachment::Save(BinaryWriter& Writer) const TempWriter.Save(Writer); } +bool +CbAttachment::IsNull() const +{ + if (const CompressedBuffer* Buffer = std::get_if(&Value)) + { + return Buffer->IsNull(); + } + else + { + return false; + } +} + +bool +CbAttachment::IsObject() const +{ + return std::holds_alternative(Value); +} + +IoHash +CbAttachment::GetHash() const +{ + if (const CompressedBuffer* Buffer = std::get_if(&Value)) + { + return Buffer->IsNull() ? IoHash::HashMemory(MemoryView()) : IoHash::FromBLAKE3(Buffer->GetRawHash()); + } + else + { + return std::get(Value).Hash; + } +} + +SharedBuffer +CbAttachment::AsBinary() const +{ + if (const CompressedBuffer* Buffer = std::get_if(&Value)) + { + return Buffer->Decompress(); + } + else + { + return std::get(Value).Object.GetBuffer(); + } +} + +CompressedBuffer +CbAttachment::AsCompressedBinary() const +{ + if (const CompressedBuffer* Buffer = std::get_if(&Value)) + { + return *Buffer; + } + else + { + return CompressedBuffer::Compress(std::get(Value).Object.GetBuffer(), + OodleCompressor::NotSet, + OodleCompressionLevel::None); + } +} + +/** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */ +CbObject +CbAttachment::AsObject() const +{ + if (const CbObjectValue* ObjectValue = std::get_if(&Value)) + { + return ObjectValue->Object; + } + else + { + return {}; + } +} + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void -- cgit v1.2.3