Introduced support for compressed buffer attachments

8 files changed, 265 insertions, 118 deletions

diff --git a/zencore/compactbinarybuilder.cpp b/zencore/compactbinarybuilder.cpp
index a30c8f04a..08f37a23d 100644
--- a/zencore/compactbinarybuilder.cpp
+++ b/zencore/compactbinarybuilder.cpp
@@ -459,6 +459,12 @@ CbWriter::AddBinary(SharedBuffer Buffer)
 }
 
 void
+CbWriter::AddBinary(const CompositeBuffer& Buffer)
+{
+	AddBinary(Buffer.Flatten());
+}
+
+void
 CbWriter::AddString(const std::string_view Value)
 {
 	BeginField();
diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp
index 92bb5d411..c4df54616 100644
--- a/zencore/compactbinarypackage.cpp
+++ b/zencore/compactbinarypackage.cpp
@@ -12,47 +12,49 @@ namespace zen {
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-CbAttachment::CbAttachment(CbObject InValue, const IoHash* const InHash)
+CbAttachment::CbAttachment(const CompressedBuffer& InValue) : CbAttachment(InValue.MakeOwned())
 {
-	MemoryView View;
-	if (!InValue.IsOwned() || !InValue.TryGetSerializedView(View))
-	{
-		InValue = CbObject::Clone(InValue);
-	}
+}
 
-	Object = InValue.AsFieldView();
-	Buffer = std::move(InValue).GetBuffer();
+CbAttachment::CbAttachment(CompressedBuffer&& InValue)
+{
+	Value.emplace<CompressedBuffer>(std::move(InValue).MakeOwned());
+}
 
-	if (InHash)
-	{
-		Hash = *InHash;
-	}
-	else
-	{
-		Hash = IoHash::HashMemory(Buffer);
-	}
+CbAttachment::CbAttachment(const SharedBuffer& InValue)
+: CbAttachment(InValue.IsNull() ? CompressedBuffer()
+								: CompressedBuffer::Compress(InValue, OodleCompressor::NotSet, OodleCompressionLevel::None))
+{
 }
 
-CbAttachment::CbAttachment(SharedBuffer InBuffer, const IoHash* const InHash) : Buffer(std::move(InBuffer).MakeOwned())
+CbAttachment::CbAttachment(const SharedBuffer& InValue, [[maybe_unused]] const IoHash& InHash)
+: CbAttachment(InValue.IsNull() ? CompressedBuffer()
+								: CompressedBuffer::Compress(InValue, OodleCompressor::NotSet, OodleCompressionLevel::None))
 {
-	if (InHash)
-	{
-		Hash = *InHash;
-		if (Buffer.GetSize())
-		{
-			// This is disabled for now as it forces disk-based attachments to get mapped which
-			// then prevents us from making them delete themselves on close
+	// This could be more efficient, and should at the very least try to validate the hash
+}
 
-			// ZEN_ASSERT_SLOW(Hash == IoHash::HashMemory(Buffer.GetData(), Buffer.GetSize()));
+CbAttachment::CbAttachment(const CbObject& InValue, const IoHash* const InHash)
+{
+	auto SetValue = [&](const CbObject& ValueToSet) {
+		if (InHash)
+		{
+			Value.emplace<CbObjectValue>(ValueToSet, *InHash);
 		}
 		else
 		{
-			ZEN_ASSERT_SLOW(Hash == IoHash::Zero, TEXT("A null or empty buffer must use a hash of zero."));
+			Value.emplace<CbObjectValue>(ValueToSet, ValueToSet.GetHash());
 		}
+	};
+
+	MemoryView View;
+	if (!InValue.IsOwned() || !InValue.TryGetSerializedView(View))
+	{
+		SetValue(CbObject::Clone(InValue));
 	}
 	else
 	{
-		Hash = IoHash::HashMemory(Buffer.GetData(), Buffer.GetSize());
+		SetValue(InValue);
 	}
 }
 
@@ -68,33 +70,39 @@ CbAttachment::TryLoad(IoBuffer& InBuffer, BufferAllocator Allocator)
 bool
 CbAttachment::TryLoad(CbFieldIterator& Fields)
 {
-	const MemoryView View = Fields.AsBinaryView();
+	const CbObjectView ObjectView = Fields.AsObjectView();
 	if (Fields.HasError())
 	{
-		return false;
-	}
-
-	if (View.GetSize() > 0)
-	{
-		Buffer = SharedBuffer::MakeView(View, Fields.GetOuterBuffer()).MakeOwned();
-		++Fields;
-		Hash = Fields.AsAttachment();
-		if (Fields.HasError())
+		// Is a buffer
+		const MemoryView BinaryView = Fields.AsBinaryView();
+		if (BinaryView.GetSize() > 0)
 		{
-			return false;
+			Value.emplace<CompressedBuffer>(
+				CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())).MakeOwned());
+
+			++Fields;
 		}
-		if (Fields.IsObjectAttachment())
+		else
 		{
-			Object = CbFieldView(Buffer.GetData());
+			++Fields;
+			Value.emplace<CompressedBuffer>();
 		}
-		++Fields;
 	}
 	else
 	{
+		// It's an object
 		++Fields;
-		Buffer.Reset();
-		Object = CbFieldView();
-		Hash   = IoHash::Zero;
+		IoHash Hash;
+		if (ObjectView)
+		{
+			Hash = Fields.AsObjectAttachment();
+			++Fields;
+		}
+		else
+		{
+			Hash = IoHash::HashMemory(MemoryView{});
+		}
+		Value.emplace<CbObjectValue>(CbObject(ObjectView, Fields->GetOuterBuffer()), Hash);
 	}
 
 	return true;
@@ -103,60 +111,73 @@ CbAttachment::TryLoad(CbFieldIterator& Fields)
 bool
 CbAttachment::TryLoad(BinaryReader& Reader, BufferAllocator Allocator)
 {
-	CbField			 BufferField = LoadCompactBinary(Reader, Allocator);
-	const MemoryView View		 = BufferField.AsBinaryView();
-	if (BufferField.HasError())
-	{
-		return false;
-	}
-	if (View.GetSize() > 0)
-	{
-		Buffer = SharedBuffer::MakeView(View, BufferField.GetOuterBuffer()).MakeOwned();
-		Object = CbFieldView();
+	CbField			   Field	  = LoadCompactBinary(Reader, Allocator);
+	const CbObjectView ObjectView = Field.AsObjectView();
 
-		std::vector<uint8_t> HashBuffer;
-		CbField				 HashField = LoadCompactBinary(Reader, [&HashBuffer](uint64_t Size) -> UniqueBuffer {
-			 HashBuffer.resize(Size);
-			 return UniqueBuffer::MakeMutableView(HashBuffer.data(), Size);
-		 });
-		Hash						   = HashField.AsAttachment();
-		if (HashField.HasError() || IoHash::HashMemory(Buffer) != Hash)
+	if (Field.HasError())
+	{
+		// It's a buffer
+		const MemoryView BinaryView = Field.AsBinaryView();
+		if (BinaryView.GetSize() > 0)
 		{
-			return false;
+			Value.emplace<CompressedBuffer>(
+				CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Field.GetOuterBuffer())).MakeOwned());
 		}
-		if (HashField.IsObjectAttachment())
+		else
 		{
-			Object = CbFieldView(Buffer.GetData());
+			Value.emplace<CompressedBuffer>();
 		}
 	}
 	else
 	{
-		Buffer.Reset();
-		Object = CbFieldView();
-		Hash   = IoHash::Zero;
+		// It's an object
+		IoHash Hash;
+		if (ObjectView)
+		{
+			std::vector<uint8_t> HashBuffer;
+			CbField				 HashField = LoadCompactBinary(Reader, [&HashBuffer](uint64_t Size) -> UniqueBuffer {
+				 HashBuffer.resize(Size);
+				 return UniqueBuffer::MakeMutableView(HashBuffer.data(), Size);
+			 });
+			Hash						   = HashField.AsAttachment();
+			if (HashField.HasError() || ObjectView.GetHash() != Hash)
+			{
+				// Error
+				return false;
+			}
+		}
+		else
+		{
+			Hash = IoHash::HashMemory(MemoryView());
+		}
+		Value.emplace<CbObjectValue>(CbObject(ObjectView, Field.GetOuterBuffer()), Hash);
 	}
+
 	return true;
 }
 
 void
 CbAttachment::Save(CbWriter& Writer) const
 {
-	if (Object.IsObject())
+	if (const CbObjectValue* ObjectValue = std::get_if<CbObjectValue>(&Value))
 	{
-		Writer.AddBinary(Buffer);
-		if (CbFieldView(Object).AsObjectView())
+		Writer.AddObject(ObjectValue->Object);
+		if (ObjectValue->Object)
 		{
-			Writer.AddObjectAttachment(Hash);
+			Writer.AddObjectAttachment(ObjectValue->Hash);
 		}
 	}
-	else if (Buffer && Buffer.GetSize())
-	{
-		Writer.AddBinary(Buffer);
-		Writer.AddBinaryAttachment(Hash);
-	}
-	else  // Null
+	else
 	{
-		Writer.AddBinary(MemoryView());
+		const CompressedBuffer& BufferValue = std::get<CompressedBuffer>(Value);
+		if (BufferValue.GetRawSize())
+		{
+			Writer.AddBinary(BufferValue.GetCompressed());
+		}
+		else  // Null
+		{
+			Writer.AddBinary(MemoryView());
+		}
 	}
 }
 
@@ -168,6 +189,80 @@ CbAttachment::Save(BinaryWriter& Writer) const
 	TempWriter.Save(Writer);
 }
 
+bool
+CbAttachment::IsNull() const
+{
+	if (const CompressedBuffer* Buffer = std::get_if<CompressedBuffer>(&Value))
+	{
+		return Buffer->IsNull();
+	}
+	else
+	{
+		return false;
+	}
+}
+
+bool
+CbAttachment::IsObject() const
+{
+	return std::holds_alternative<CbObjectValue>(Value);
+}
+
+IoHash
+CbAttachment::GetHash() const
+{
+	if (const CompressedBuffer* Buffer = std::get_if<CompressedBuffer>(&Value))
+	{
+		return Buffer->IsNull() ? IoHash::HashMemory(MemoryView()) : IoHash::FromBLAKE3(Buffer->GetRawHash());
+	}
+	else
+	{
+		return std::get<CbObjectValue>(Value).Hash;
+	}
+}
+
+SharedBuffer
+CbAttachment::AsBinary() const
+{
+	if (const CompressedBuffer* Buffer = std::get_if<CompressedBuffer>(&Value))
+	{
+		return Buffer->Decompress();
+	}
+	else
+	{
+		return std::get<CbObjectValue>(Value).Object.GetBuffer();
+	}
+}
+
+CompressedBuffer
+CbAttachment::AsCompressedBinary() const
+{
+	if (const CompressedBuffer* Buffer = std::get_if<CompressedBuffer>(&Value))
+	{
+		return *Buffer;
+	}
+	else
+	{
+		return CompressedBuffer::Compress(std::get<CbObjectValue>(Value).Object.GetBuffer(),
+										  OodleCompressor::NotSet,
+										  OodleCompressionLevel::None);
+	}
+}
+
+/** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */
+CbObject
+CbAttachment::AsObject() const
+{
+	if (const CbObjectValue* ObjectValue = std::get_if<CbObjectValue>(&Value))
+	{
+		return ObjectValue->Object;
+	}
+	else
+	{
+		return {};
+	}
+}
+
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 void
diff --git a/zencore/compactbinaryvalidation.cpp b/zencore/compactbinaryvalidation.cpp
index 80b8bf32a..87455860c 100644
--- a/zencore/compactbinaryvalidation.cpp
+++ b/zencore/compactbinaryvalidation.cpp
@@ -415,34 +415,59 @@ ValidateCbPackageField(MemoryView& View, CbValidateMode Mode, CbValidateError& E
 static IoHash
 ValidateCbPackageAttachment(CbFieldView& Value, MemoryView& View, CbValidateMode Mode, CbValidateError& Error)
 {
-	const MemoryView ValueView = Value.AsBinaryView();
-	if (Value.HasError() && EnumHasAnyFlags(Mode, CbValidateMode::Package))
+	const CbObjectView ObjectView = Value.AsObjectView();
+	if (Value.HasError())
 	{
-		if (EnumHasAnyFlags(Mode, CbValidateMode::Package))
+		const MemoryView BinaryView = Value.AsBinaryView();
+		if (Value.HasError() && EnumHasAnyFlags(Mode, CbValidateMode::Package))
 		{
-			AddError(Error, CbValidateError::InvalidPackageFormat);
+			if (EnumHasAnyFlags(Mode, CbValidateMode::Package))
+			{
+				AddError(Error, CbValidateError::InvalidPackageFormat);
+			}
+		}
+		else if (BinaryView.GetSize())
+		{
+			if (EnumHasAnyFlags(Mode, CbValidateMode::Package | CbValidateMode::PackageHash))
+			{
+				CompressedBuffer Buffer = CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView));
+				if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && Buffer.IsNull())
+				{
+					AddError(Error, CbValidateError::InvalidPackageFormat);
+				}
+				if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) &&
+					(IoHash::FromBLAKE3(Buffer.GetRawHash()) != IoHash::HashMemory(Buffer.DecompressToComposite())))
+				{
+					AddError(Error, CbValidateError::InvalidPackageHash);
+				}
+				return IoHash::FromBLAKE3(Buffer.GetRawHash());
+			}
 		}
 	}
-	else if (ValueView.GetSize())
+	else
 	{
-		if (CbFieldView HashField = ValidateCbPackageField(View, Mode, Error))
+		if (ObjectView)
 		{
-			const IoHash Hash = HashField.AsAttachment();
-			if (EnumHasAnyFlags(Mode, CbValidateMode::Package))
+			if (CbFieldView HashField = ValidateCbPackageField(View, Mode, Error))
 			{
-				if (HashField.HasError())
+				const IoHash Hash = HashField.AsAttachment();
+				if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && HashField.HasError())
 				{
 					AddError(Error, CbValidateError::InvalidPackageFormat);
 				}
-				else if (Hash != IoHash::HashMemory(ValueView.GetData(), ValueView.GetSize()))
+				if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) && (Hash != ObjectView.GetHash()))
 				{
 					AddError(Error, CbValidateError::InvalidPackageHash);
 				}
+				return Hash;
 			}
-			return Hash;
+		}
+		else
+		{
+			return CbObject().GetHash();
 		}
 	}
-	return IoHash();
+	return {};
 }
 
 static IoHash
diff --git a/zencore/include/zencore/compactbinarybuilder.h b/zencore/include/zencore/compactbinarybuilder.h
index f2bf773bf..f7f2bbfd3 100644
--- a/zencore/include/zencore/compactbinarybuilder.h
+++ b/zencore/include/zencore/compactbinarybuilder.h
@@ -210,6 +210,13 @@ public:
 	ZENCORE_API void AddBinary(IoBuffer Value);
 	ZENCORE_API void AddBinary(SharedBuffer Value);
 
+	inline void AddBinary(std::string_view Name, const CompositeBuffer& Buffer)
+	{
+		SetName(Name);
+		AddBinary(Buffer);
+	}
+	ZENCORE_API void AddBinary(const CompositeBuffer& Buffer);
+
 	/** Write a string field by copying the UTF-8 value. */
 	inline void AddString(std::string_view Name, std::string_view Value)
 	{
diff --git a/zencore/include/zencore/compactbinarypackage.h b/zencore/include/zencore/compactbinarypackage.h
index 66bace294..181bf1818 100644
--- a/zencore/include/zencore/compactbinarypackage.h
+++ b/zencore/include/zencore/compactbinarypackage.h
@@ -5,10 +5,12 @@
 #include <zencore/zencore.h>
 
 #include <zencore/compactbinary.h>
+#include <zencore/compress.h>
 #include <zencore/iohash.h>
 
 #include <functional>
 #include <span>
+#include <variant>
 
 namespace zen {
 
@@ -36,16 +38,20 @@ public:
 	CbAttachment() = default;
 
 	/** Construct a compact binary attachment. Value is cloned if not owned. */
-	inline explicit CbAttachment(CbObject Value) : CbAttachment(std::move(Value), nullptr) {}
+	inline explicit CbAttachment(const CbObject& Value) : CbAttachment(Value, nullptr) {}
 
 	/** Construct a compact binary attachment. Value is cloned if not owned. Hash must match Value. */
-	inline explicit CbAttachment(CbObject Value, const IoHash& Hash) : CbAttachment(std::move(Value), &Hash) {}
+	inline explicit CbAttachment(const CbObject& Value, const IoHash& Hash) : CbAttachment(Value, &Hash) {}
 
 	/** Construct a binary attachment. Value is cloned if not owned. */
-	inline explicit CbAttachment(SharedBuffer Value) : CbAttachment(std::move(Value), nullptr) {}
+	ZENCORE_API explicit CbAttachment(const SharedBuffer& Value);
 
 	/** Construct a binary attachment. Value is cloned if not owned. Hash must match Value. */
-	inline explicit CbAttachment(SharedBuffer Value, const IoHash& Hash) : CbAttachment(std::move(Value), &Hash) {}
+	ZENCORE_API explicit CbAttachment(const SharedBuffer& Value, const IoHash& Hash);
+
+	/** Construct a binary attachment. Value is cloned if not owned. */
+	ZENCORE_API explicit CbAttachment(const CompressedBuffer& InValue);
+	ZENCORE_API explicit CbAttachment(CompressedBuffer&& InValue);
 
 	/** Reset this to a null attachment. */
 	inline void Reset() { *this = CbAttachment(); }
@@ -54,27 +60,30 @@ public:
 	inline explicit operator bool() const { return !IsNull(); }
 
 	/** Whether the attachment has a value. */
-	inline bool IsNull() const { return !Buffer; }
+	ZENCORE_API [[nodiscard]] bool IsNull() const;
 
 	/** Access the attachment as binary. Defaults to a null buffer on error. */
-	inline SharedBuffer AsBinary() const { return Buffer; }
+	ZENCORE_API [[nodiscard]] SharedBuffer AsBinary() const;
+
+	/** Access the attachment as compressed binary. Defaults to a null buffer if the attachment is null. */
+	ZENCORE_API [[nodiscard]] CompressedBuffer AsCompressedBinary() const;
 
 	/** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */
-	inline CbObject AsObject() const { return Object ? CbObject(CbFieldView(Object).AsObjectView(), Buffer) : CbObject{}; }
+	ZENCORE_API [[nodiscard]] CbObject AsObject() const;
 
-	/** Returns whether the attachment is binary or compact binary. */
-	inline bool IsBinary() const { return !Buffer.IsNull(); }
+	/** Returns true if the attachment is either binary or an object */
+	inline [[nodiscard]] bool IsBinary() const { return !IsNull(); }
 
-	/** Returns whether the attachment is compact binary. */
-	inline bool IsObject() const { return Object.IsObject(); }
+	/** Returns whether the attachment is an object. */
+	ZENCORE_API [[nodiscard]] bool IsObject() const;
 
 	/** Returns the hash of the attachment value. */
-	inline const IoHash& GetHash() const { return Hash; }
+	ZENCORE_API [[nodiscard]] IoHash GetHash() const;
 
 	/** Compares attachments by their hash. Any discrepancy in type must be handled externally. */
-	inline bool operator==(const CbAttachment& Attachment) const { return Hash == Attachment.Hash; }
-	inline bool operator!=(const CbAttachment& Attachment) const { return Hash != Attachment.Hash; }
-	inline bool operator<(const CbAttachment& Attachment) const { return Hash < Attachment.Hash; }
+	inline bool operator==(const CbAttachment& Attachment) const { return GetHash() == Attachment.GetHash(); }
+	inline bool operator!=(const CbAttachment& Attachment) const { return GetHash() != Attachment.GetHash(); }
+	inline bool operator<(const CbAttachment& Attachment) const { return GetHash() < Attachment.GetHash(); }
 
 	/**
 	 * Load the attachment from compact binary as written by Save.
@@ -102,15 +111,18 @@ public:
 	ZENCORE_API void Save(BinaryWriter& Writer) const;
 
 private:
-	ZENCORE_API CbAttachment(CbObject Value, const IoHash* Hash);
-	ZENCORE_API CbAttachment(SharedBuffer Value, const IoHash* Hash);
-
-	/** An owned buffer containing the binary or compact binary data. */
-	SharedBuffer Buffer;
-	/** A field iterator that is valid only for compact binary attachments. */
-	CbFieldView Object;
-	/** A hash of the attachment value. */
-	IoHash Hash;
+	ZENCORE_API CbAttachment(const CbObject& Value, const IoHash* Hash);
+
+	struct CbObjectValue
+	{
+		CbObject Object;
+		IoHash	 Hash;
+
+		CbObjectValue(const CbObject& InObject, const IoHash& InHash) : Object(InObject), Hash(InHash) {}
+		CbObjectValue(CbObject&& InObject, const IoHash& InHash) : Object(std::move(InObject)), Hash(InHash) {}
+	};
+
+	std::variant<CompressedBuffer, CbObjectValue> Value;
 };
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/zencore/include/zencore/compactbinaryvalidation.h b/zencore/include/zencore/compactbinaryvalidation.h
index 656eb3d96..9799c594a 100644
--- a/zencore/include/zencore/compactbinaryvalidation.h
+++ b/zencore/include/zencore/compactbinaryvalidation.h
@@ -62,8 +62,10 @@ enum class CbValidateMode : uint32_t
 	 */
 	Package = 1 << 4,
 
+	PackageHash = 1 << 5,
+
 	/** Perform all validation described above. */
-	All = Default | Names | Format | Padding | Package,
+	All = Default | Names | Format | Padding | Package | PackageHash,
 };
 
 ENUM_CLASS_FLAGS(CbValidateMode);
diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp
index b0d21d53a..c3d8c01ab 100644
--- a/zenserver/compute/apply.cpp
+++ b/zenserver/compute/apply.cpp
@@ -429,7 +429,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
 										ZEN_ASSERT(Attachment.IsBinary());
 
 										const IoHash DataHash = Attachment.GetHash();
-										SharedBuffer DataView = Attachment.AsBinaryView();
+										SharedBuffer DataView = Attachment.AsBinary();
 
 										TotalAttachmentBytes += DataView.GetSize();
 										++AttachmentCount;
@@ -571,7 +571,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
 									ZEN_ASSERT(Attachment.IsBinary());
 
 									const IoHash DataHash = Attachment.GetHash();
-									SharedBuffer DataView = Attachment.AsBinaryView();
+									SharedBuffer DataView = Attachment.AsBinary();
 
 									TotalAttachmentBytes += DataView.GetSize();
 									++AttachmentCount;
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 90ff79b30..1785aad9e 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -542,7 +542,7 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbPackage OpPackage)
 
 	for (const auto& Attach : Attachments)
 	{
-		IoBuffer			   AttachmentData = Attach.AsBinaryView().AsIoBuffer();
+		IoBuffer			   AttachmentData = Attach.AsBinary().AsIoBuffer();
 		CasStore::InsertResult Result		  = m_CasStore.InsertChunk(AttachmentData, Attach.GetHash());
 
 		const uint64_t AttachmentSize = AttachmentData.Size();