Introduced support for compressed buffer attachments

1 files changed, 173 insertions, 78 deletions

diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp
index 92bb5d411..c4df54616 100644
--- a/zencore/compactbinarypackage.cpp
+++ b/zencore/compactbinarypackage.cpp
@@ -12,47 +12,49 @@ namespace zen {
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-CbAttachment::CbAttachment(CbObject InValue, const IoHash* const InHash)
+CbAttachment::CbAttachment(const CompressedBuffer& InValue) : CbAttachment(InValue.MakeOwned())
 {
-	MemoryView View;
-	if (!InValue.IsOwned() || !InValue.TryGetSerializedView(View))
-	{
-		InValue = CbObject::Clone(InValue);
-	}
+}
 
-	Object = InValue.AsFieldView();
-	Buffer = std::move(InValue).GetBuffer();
+CbAttachment::CbAttachment(CompressedBuffer&& InValue)
+{
+	Value.emplace<CompressedBuffer>(std::move(InValue).MakeOwned());
+}
 
-	if (InHash)
-	{
-		Hash = *InHash;
-	}
-	else
-	{
-		Hash = IoHash::HashMemory(Buffer);
-	}
+CbAttachment::CbAttachment(const SharedBuffer& InValue)
+: CbAttachment(InValue.IsNull() ? CompressedBuffer()
+								: CompressedBuffer::Compress(InValue, OodleCompressor::NotSet, OodleCompressionLevel::None))
+{
 }
 
-CbAttachment::CbAttachment(SharedBuffer InBuffer, const IoHash* const InHash) : Buffer(std::move(InBuffer).MakeOwned())
+CbAttachment::CbAttachment(const SharedBuffer& InValue, [[maybe_unused]] const IoHash& InHash)
+: CbAttachment(InValue.IsNull() ? CompressedBuffer()
+								: CompressedBuffer::Compress(InValue, OodleCompressor::NotSet, OodleCompressionLevel::None))
 {
-	if (InHash)
-	{
-		Hash = *InHash;
-		if (Buffer.GetSize())
-		{
-			// This is disabled for now as it forces disk-based attachments to get mapped which
-			// then prevents us from making them delete themselves on close
+	// This could be more efficient, and should at the very least try to validate the hash
+}
 
-			// ZEN_ASSERT_SLOW(Hash == IoHash::HashMemory(Buffer.GetData(), Buffer.GetSize()));
+CbAttachment::CbAttachment(const CbObject& InValue, const IoHash* const InHash)
+{
+	auto SetValue = [&](const CbObject& ValueToSet) {
+		if (InHash)
+		{
+			Value.emplace<CbObjectValue>(ValueToSet, *InHash);
 		}
 		else
 		{
-			ZEN_ASSERT_SLOW(Hash == IoHash::Zero, TEXT("A null or empty buffer must use a hash of zero."));
+			Value.emplace<CbObjectValue>(ValueToSet, ValueToSet.GetHash());
 		}
+	};
+
+	MemoryView View;
+	if (!InValue.IsOwned() || !InValue.TryGetSerializedView(View))
+	{
+		SetValue(CbObject::Clone(InValue));
 	}
 	else
 	{
-		Hash = IoHash::HashMemory(Buffer.GetData(), Buffer.GetSize());
+		SetValue(InValue);
 	}
 }
 
@@ -68,33 +70,39 @@ CbAttachment::TryLoad(IoBuffer& InBuffer, BufferAllocator Allocator)
 bool
 CbAttachment::TryLoad(CbFieldIterator& Fields)
 {
-	const MemoryView View = Fields.AsBinaryView();
+	const CbObjectView ObjectView = Fields.AsObjectView();
 	if (Fields.HasError())
 	{
-		return false;
-	}
-
-	if (View.GetSize() > 0)
-	{
-		Buffer = SharedBuffer::MakeView(View, Fields.GetOuterBuffer()).MakeOwned();
-		++Fields;
-		Hash = Fields.AsAttachment();
-		if (Fields.HasError())
+		// Is a buffer
+		const MemoryView BinaryView = Fields.AsBinaryView();
+		if (BinaryView.GetSize() > 0)
 		{
-			return false;
+			Value.emplace<CompressedBuffer>(
+				CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())).MakeOwned());
+
+			++Fields;
 		}
-		if (Fields.IsObjectAttachment())
+		else
 		{
-			Object = CbFieldView(Buffer.GetData());
+			++Fields;
+			Value.emplace<CompressedBuffer>();
 		}
-		++Fields;
 	}
 	else
 	{
+		// It's an object
 		++Fields;
-		Buffer.Reset();
-		Object = CbFieldView();
-		Hash   = IoHash::Zero;
+		IoHash Hash;
+		if (ObjectView)
+		{
+			Hash = Fields.AsObjectAttachment();
+			++Fields;
+		}
+		else
+		{
+			Hash = IoHash::HashMemory(MemoryView{});
+		}
+		Value.emplace<CbObjectValue>(CbObject(ObjectView, Fields->GetOuterBuffer()), Hash);
 	}
 
 	return true;
@@ -103,60 +111,73 @@ CbAttachment::TryLoad(CbFieldIterator& Fields)
 bool
 CbAttachment::TryLoad(BinaryReader& Reader, BufferAllocator Allocator)
 {
-	CbField			 BufferField = LoadCompactBinary(Reader, Allocator);
-	const MemoryView View		 = BufferField.AsBinaryView();
-	if (BufferField.HasError())
-	{
-		return false;
-	}
-	if (View.GetSize() > 0)
-	{
-		Buffer = SharedBuffer::MakeView(View, BufferField.GetOuterBuffer()).MakeOwned();
-		Object = CbFieldView();
+	CbField			   Field	  = LoadCompactBinary(Reader, Allocator);
+	const CbObjectView ObjectView = Field.AsObjectView();
 
-		std::vector<uint8_t> HashBuffer;
-		CbField				 HashField = LoadCompactBinary(Reader, [&HashBuffer](uint64_t Size) -> UniqueBuffer {
-			 HashBuffer.resize(Size);
-			 return UniqueBuffer::MakeMutableView(HashBuffer.data(), Size);
-		 });
-		Hash						   = HashField.AsAttachment();
-		if (HashField.HasError() || IoHash::HashMemory(Buffer) != Hash)
+	if (Field.HasError())
+	{
+		// It's a buffer
+		const MemoryView BinaryView = Field.AsBinaryView();
+		if (BinaryView.GetSize() > 0)
 		{
-			return false;
+			Value.emplace<CompressedBuffer>(
+				CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Field.GetOuterBuffer())).MakeOwned());
 		}
-		if (HashField.IsObjectAttachment())
+		else
 		{
-			Object = CbFieldView(Buffer.GetData());
+			Value.emplace<CompressedBuffer>();
 		}
 	}
 	else
 	{
-		Buffer.Reset();
-		Object = CbFieldView();
-		Hash   = IoHash::Zero;
+		// It's an object
+		IoHash Hash;
+		if (ObjectView)
+		{
+			std::vector<uint8_t> HashBuffer;
+			CbField				 HashField = LoadCompactBinary(Reader, [&HashBuffer](uint64_t Size) -> UniqueBuffer {
+				 HashBuffer.resize(Size);
+				 return UniqueBuffer::MakeMutableView(HashBuffer.data(), Size);
+			 });
+			Hash						   = HashField.AsAttachment();
+			if (HashField.HasError() || ObjectView.GetHash() != Hash)
+			{
+				// Error
+				return false;
+			}
+		}
+		else
+		{
+			Hash = IoHash::HashMemory(MemoryView());
+		}
+		Value.emplace<CbObjectValue>(CbObject(ObjectView, Field.GetOuterBuffer()), Hash);
 	}
+
 	return true;
 }
 
 void
 CbAttachment::Save(CbWriter& Writer) const
 {
-	if (Object.IsObject())
+	if (const CbObjectValue* ObjectValue = std::get_if<CbObjectValue>(&Value))
 	{
-		Writer.AddBinary(Buffer);
-		if (CbFieldView(Object).AsObjectView())
+		Writer.AddObject(ObjectValue->Object);
+		if (ObjectValue->Object)
 		{
-			Writer.AddObjectAttachment(Hash);
+			Writer.AddObjectAttachment(ObjectValue->Hash);
 		}
 	}
-	else if (Buffer && Buffer.GetSize())
-	{
-		Writer.AddBinary(Buffer);
-		Writer.AddBinaryAttachment(Hash);
-	}
-	else  // Null
+	else
 	{
-		Writer.AddBinary(MemoryView());
+		const CompressedBuffer& BufferValue = std::get<CompressedBuffer>(Value);
+		if (BufferValue.GetRawSize())
+		{
+			Writer.AddBinary(BufferValue.GetCompressed());
+		}
+		else  // Null
+		{
+			Writer.AddBinary(MemoryView());
+		}
 	}
 }
 
@@ -168,6 +189,80 @@ CbAttachment::Save(BinaryWriter& Writer) const
 	TempWriter.Save(Writer);
 }
 
+bool
+CbAttachment::IsNull() const
+{
+	if (const CompressedBuffer* Buffer = std::get_if<CompressedBuffer>(&Value))
+	{
+		return Buffer->IsNull();
+	}
+	else
+	{
+		return false;
+	}
+}
+
+bool
+CbAttachment::IsObject() const
+{
+	return std::holds_alternative<CbObjectValue>(Value);
+}
+
+IoHash
+CbAttachment::GetHash() const
+{
+	if (const CompressedBuffer* Buffer = std::get_if<CompressedBuffer>(&Value))
+	{
+		return Buffer->IsNull() ? IoHash::HashMemory(MemoryView()) : IoHash::FromBLAKE3(Buffer->GetRawHash());
+	}
+	else
+	{
+		return std::get<CbObjectValue>(Value).Hash;
+	}
+}
+
+SharedBuffer
+CbAttachment::AsBinary() const
+{
+	if (const CompressedBuffer* Buffer = std::get_if<CompressedBuffer>(&Value))
+	{
+		return Buffer->Decompress();
+	}
+	else
+	{
+		return std::get<CbObjectValue>(Value).Object.GetBuffer();
+	}
+}
+
+CompressedBuffer
+CbAttachment::AsCompressedBinary() const
+{
+	if (const CompressedBuffer* Buffer = std::get_if<CompressedBuffer>(&Value))
+	{
+		return *Buffer;
+	}
+	else
+	{
+		return CompressedBuffer::Compress(std::get<CbObjectValue>(Value).Object.GetBuffer(),
+										  OodleCompressor::NotSet,
+										  OodleCompressionLevel::None);
+	}
+}
+
+/** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */
+CbObject
+CbAttachment::AsObject() const
+{
+	if (const CbObjectValue* ObjectValue = std::get_if<CbObjectValue>(&Value))
+	{
+		return ObjectValue->Object;
+	}
+	else
+	{
+		return {};
+	}
+}
+
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 void