Add `import-project` and `export-project` (#183)

* Add `import-project` and `export-project` command line parsing

9 files changed, 275 insertions, 113 deletions

diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp
index 0e4a46fa1..0db9f02ea 100644
--- a/zencore/compactbinary.cpp
+++ b/zencore/compactbinary.cpp
@@ -839,10 +839,10 @@ CbFieldView::CopyTo(MutableMemoryView Buffer) const
 void
 CbFieldView::CopyTo(BinaryWriter& Ar) const
 {
-	const MemoryView Source			= GetViewNoType();
+	const MemoryView SourceView		= GetViewNoType();
 	CbFieldType		 SerializedType = CbFieldTypeOps::GetSerializedType(Type);
-	Ar.Write(&SerializedType, sizeof(SerializedType));
-	Ar.Write(Source.GetData(), Source.GetSize());
+	const MemoryView TypeView(reinterpret_cast<const uint8_t*>(&SerializedType), sizeof(SerializedType));
+	Ar.Write({TypeView, SourceView});
 }
 
 MemoryView
@@ -948,10 +948,10 @@ CbArrayView::CopyTo(MutableMemoryView Buffer) const
 void
 CbArrayView::CopyTo(BinaryWriter& Ar) const
 {
-	const MemoryView Source			= GetPayloadView();
-	CbFieldType		 SerializedType = CbFieldTypeOps::GetType(GetType());
-	Ar.Write(&SerializedType, sizeof(SerializedType));
-	Ar.Write(Source.GetData(), Source.GetSize());
+	const MemoryView SourceView		= GetPayloadView();
+	CbFieldType		 SerializedType = CbFieldTypeOps::GetSerializedType(GetType());
+	const MemoryView TypeView(reinterpret_cast<const uint8_t*>(&SerializedType), sizeof(SerializedType));
+	Ar.Write({TypeView, SourceView});
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -988,7 +988,7 @@ CbObjectView::VisitFields(ICbVisitor&)
 CbFieldView
 CbObjectView::FindView(const std::string_view Name) const
 {
-	for (const CbFieldView Field : *this)
+	for (const CbFieldView& Field : *this)
 	{
 		if (Name == Field.GetName())
 		{
@@ -1001,7 +1001,7 @@ CbObjectView::FindView(const std::string_view Name) const
 CbFieldView
 CbObjectView::FindViewIgnoreCase(const std::string_view Name) const
 {
-	for (const CbFieldView Field : *this)
+	for (const CbFieldView& Field : *this)
 	{
 		if (Name == Field.GetName())
 		{
@@ -1061,10 +1061,10 @@ CbObjectView::CopyTo(MutableMemoryView Buffer) const
 void
 CbObjectView::CopyTo(BinaryWriter& Ar) const
 {
-	const MemoryView Source			= GetPayloadView();
-	CbFieldType		 SerializedType = CbFieldTypeOps::GetType(GetType());
-	Ar.Write(&SerializedType, sizeof(SerializedType));
-	Ar.Write(Source.GetData(), Source.GetSize());
+	const MemoryView SourceView		= GetPayloadView();
+	CbFieldType		 SerializedType = CbFieldTypeOps::GetSerializedType(GetType());
+	const MemoryView TypeView(reinterpret_cast<const uint8_t*>(&SerializedType), sizeof(SerializedType));
+	Ar.Write({TypeView, SourceView});
 }
 
 //////////////////////////////////////////////////////////////////////////
diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp
index a25466bed..19675b9cf 100644
--- a/zencore/compactbinarypackage.cpp
+++ b/zencore/compactbinarypackage.cpp
@@ -11,7 +11,7 @@ namespace zen {
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-CbAttachment::CbAttachment(const CompressedBuffer& InValue) : CbAttachment(InValue.MakeOwned())
+CbAttachment::CbAttachment(const CompressedBuffer& InValue, const IoHash& Hash) : CbAttachment(InValue.MakeOwned(), Hash)
 {
 }
 
@@ -19,36 +19,40 @@ CbAttachment::CbAttachment(const SharedBuffer& InValue) : CbAttachment(Composite
 {
 }
 
-CbAttachment::CbAttachment(const SharedBuffer& InValue, [[maybe_unused]] const IoHash& InHash)
-: CbAttachment(CompositeBuffer(InValue), InHash)
+CbAttachment::CbAttachment(const SharedBuffer& InValue, const IoHash& InHash) : CbAttachment(CompositeBuffer(InValue), InHash)
 {
 }
 
-CbAttachment::CbAttachment(const CompositeBuffer& InValue) : Value{std::in_place_type<BinaryValue>, InValue}
+CbAttachment::CbAttachment(const CompositeBuffer& InValue)
+: Hash(InValue.IsNull() ? IoHash::Zero : IoHash::HashBuffer(InValue))
+, Value(InValue)
 {
-	if (std::get<BinaryValue>(Value).Buffer.IsNull())
+	if (std::get<CompositeBuffer>(Value).IsNull())
 	{
 		Value.emplace<std::nullptr_t>();
 	}
 }
 
-CbAttachment::CbAttachment(CompositeBuffer&& InValue) : Value{std::in_place_type<BinaryValue>, InValue}
+CbAttachment::CbAttachment(CompositeBuffer&& InValue)
+: Hash(InValue.IsNull() ? IoHash::Zero : IoHash::HashBuffer(InValue))
+, Value(std::move(InValue))
+
 {
-	if (std::get<BinaryValue>(Value).Buffer.IsNull())
+	if (std::get<CompositeBuffer>(Value).IsNull())
 	{
 		Value.emplace<std::nullptr_t>();
 	}
 }
 
-CbAttachment::CbAttachment(CompositeBuffer&& InValue, const IoHash& Hash) : Value{std::in_place_type<BinaryValue>, InValue, Hash}
+CbAttachment::CbAttachment(CompositeBuffer&& InValue, const IoHash& InHash) : Hash(InHash), Value(InValue)
 {
-	if (std::get<BinaryValue>(Value).Buffer.IsNull())
+	if (std::get<CompositeBuffer>(Value).IsNull())
 	{
 		Value.emplace<std::nullptr_t>();
 	}
 }
 
-CbAttachment::CbAttachment(CompressedBuffer&& InValue) : Value(std::in_place_type<CompressedBuffer>, InValue)
+CbAttachment::CbAttachment(CompressedBuffer&& InValue, const IoHash& InHash) : Hash(InHash), Value(InValue)
 {
 	if (std::get<CompressedBuffer>(Value).IsNull())
 	{
@@ -61,11 +65,13 @@ CbAttachment::CbAttachment(const CbObject& InValue, const IoHash* const InHash)
 	auto SetValue = [&](const CbObject& ValueToSet) {
 		if (InHash)
 		{
-			Value.emplace<CbObjectValue>(ValueToSet, *InHash);
+			Value.emplace<CbObject>(ValueToSet);
+			Hash = *InHash;
 		}
 		else
 		{
-			Value.emplace<CbObjectValue>(ValueToSet, ValueToSet.GetHash());
+			Value.emplace<CbObject>(ValueToSet);
+			Hash = ValueToSet.GetHash();
 		}
 	};
 
@@ -94,7 +100,8 @@ CbAttachment::TryLoad(CbFieldIterator& Fields)
 	if (const CbObjectView ObjectView = Fields.AsObjectView(); !Fields.HasError())
 	{
 		// Is a null object or object not prefixed with a precomputed hash value
-		Value.emplace<CbObjectValue>(CbObject(ObjectView, Fields.GetOuterBuffer()), ObjectView.GetHash());
+		Value.emplace<CbObject>(CbObject(ObjectView, Fields.GetOuterBuffer()));
+		Hash = ObjectView.GetHash();
 		++Fields;
 	}
 	else if (const IoHash ObjectAttachmentHash = Fields.AsObjectAttachment(); !Fields.HasError())
@@ -106,7 +113,8 @@ CbAttachment::TryLoad(CbFieldIterator& Fields)
 		{
 			return false;
 		}
-		Value.emplace<CbObjectValue>(CbObject(InnerObjectView, Fields.GetOuterBuffer()), ObjectAttachmentHash);
+		Value.emplace<CbObject>(CbObject(InnerObjectView, Fields.GetOuterBuffer()));
+		Hash = ObjectAttachmentHash;
 		++Fields;
 	}
 	else if (const IoHash BinaryAttachmentHash = Fields.AsBinaryAttachment(); !Fields.HasError())
@@ -118,7 +126,8 @@ CbAttachment::TryLoad(CbFieldIterator& Fields)
 		{
 			return false;
 		}
-		Value.emplace<BinaryValue>(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer()), BinaryAttachmentHash);
+		Value.emplace<CompositeBuffer>(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer()));
+		Hash = BinaryAttachmentHash;
 		++Fields;
 	}
 	else if (MemoryView BinaryView = Fields.AsBinaryView(); !Fields.HasError())
@@ -126,14 +135,17 @@ CbAttachment::TryLoad(CbFieldIterator& Fields)
 		if (BinaryView.GetSize() > 0)
 		{
 			// Is a compressed binary blob
-			Value.emplace<CompressedBuffer>(
-				CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())).MakeOwned());
+			CompressedBuffer Compressed =
+				CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())).MakeOwned();
+			Value.emplace<CompressedBuffer>(Compressed);
+			Hash = IoHash::FromBLAKE3(Compressed.GetRawHash());
 			++Fields;
 		}
 		else
 		{
 			// Is an uncompressed empty binary blob
-			Value.emplace<BinaryValue>(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer()), IoHash::HashBuffer(nullptr, 0));
+			Value.emplace<CompositeBuffer>(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer()));
+			Hash = IoHash::HashBuffer(nullptr, 0);
 			++Fields;
 		}
 	}
@@ -179,7 +191,8 @@ TryLoad_ArchiveFieldIntoAttachment(CbAttachment& TargetAttachment, CbField&& Fie
 		if (Buffer.GetSize() > 0)
 		{
 			// Is a compressed binary blob
-			TargetAttachment = CbAttachment(CompressedBuffer::FromCompressed(std::move(Buffer)));
+			CompressedBuffer Compressed = CompressedBuffer::FromCompressed(std::move(Buffer));
+			TargetAttachment			= CbAttachment(Compressed, IoHash::FromBLAKE3(Compressed.GetRawHash()));
 		}
 		else
 		{
@@ -205,25 +218,25 @@ CbAttachment::TryLoad(BinaryReader& Reader, BufferAllocator Allocator)
 void
 CbAttachment::Save(CbWriter& Writer) const
 {
-	if (const CbObjectValue* ObjValue = std::get_if<CbObjectValue>(&Value))
+	if (const CbObject* Object = std::get_if<CbObject>(&Value))
 	{
-		if (ObjValue->Object)
+		if (*Object)
 		{
-			Writer.AddObjectAttachment(ObjValue->Hash);
+			Writer.AddObjectAttachment(Hash);
 		}
-		Writer.AddObject(ObjValue->Object);
+		Writer.AddObject(*Object);
 	}
-	else if (const BinaryValue* BinValue = std::get_if<BinaryValue>(&Value))
+	else if (const CompositeBuffer* Binary = std::get_if<CompositeBuffer>(&Value))
 	{
-		if (BinValue->Buffer.GetSize() > 0)
+		if (Binary->GetSize() > 0)
 		{
-			Writer.AddBinaryAttachment(BinValue->Hash);
+			Writer.AddBinaryAttachment(Hash);
 		}
-		Writer.AddBinary(BinValue->Buffer);
+		Writer.AddBinary(*Binary);
 	}
-	else if (const CompressedBuffer* BufferValue = std::get_if<CompressedBuffer>(&Value))
+	else if (const CompressedBuffer* Compressed = std::get_if<CompressedBuffer>(&Value))
 	{
-		Writer.AddBinary(BufferValue->GetCompressed());
+		Writer.AddBinary(Compressed->GetCompressed());
 	}
 }
 
@@ -244,7 +257,7 @@ CbAttachment::IsNull() const
 bool
 CbAttachment::IsBinary() const
 {
-	return std::holds_alternative<BinaryValue>(Value);
+	return std::holds_alternative<CompositeBuffer>(Value);
 }
 
 bool
@@ -256,36 +269,21 @@ CbAttachment::IsCompressedBinary() const
 bool
 CbAttachment::IsObject() const
 {
-	return std::holds_alternative<CbObjectValue>(Value);
+	return std::holds_alternative<CbObject>(Value);
 }
 
 IoHash
 CbAttachment::GetHash() const
 {
-	if (const CompressedBuffer* Buffer = std::get_if<CompressedBuffer>(&Value))
-	{
-		return IoHash::FromBLAKE3(Buffer->GetRawHash());
-	}
-
-	if (const BinaryValue* BinValue = std::get_if<BinaryValue>(&Value))
-	{
-		return BinValue->Hash;
-	}
-
-	if (const CbObjectValue* ObjectValue = std::get_if<CbObjectValue>(&Value))
-	{
-		return ObjectValue->Hash;
-	}
-
-	return IoHash::Zero;
+	return Hash;
 }
 
 CompositeBuffer
 CbAttachment::AsCompositeBinary() const
 {
-	if (const BinaryValue* BinValue = std::get_if<BinaryValue>(&Value))
+	if (const CompositeBuffer* BinValue = std::get_if<CompositeBuffer>(&Value))
 	{
-		return BinValue->Buffer;
+		return *BinValue;
 	}
 
 	return CompositeBuffer::Null;
@@ -294,9 +292,9 @@ CbAttachment::AsCompositeBinary() const
 SharedBuffer
 CbAttachment::AsBinary() const
 {
-	if (const BinaryValue* BinValue = std::get_if<BinaryValue>(&Value))
+	if (const CompositeBuffer* BinValue = std::get_if<CompositeBuffer>(&Value))
 	{
-		return BinValue->Buffer.Flatten();
+		return BinValue->Flatten();
 	}
 
 	return {};
@@ -305,9 +303,9 @@ CbAttachment::AsBinary() const
 CompressedBuffer
 CbAttachment::AsCompressedBinary() const
 {
-	if (const CompressedBuffer* Buffer = std::get_if<CompressedBuffer>(&Value))
+	if (const CompressedBuffer* CompValue = std::get_if<CompressedBuffer>(&Value))
 	{
-		return *Buffer;
+		return *CompValue;
 	}
 
 	return CompressedBuffer::Null;
@@ -317,9 +315,9 @@ CbAttachment::AsCompressedBinary() const
 CbObject
 CbAttachment::AsObject() const
 {
-	if (const CbObjectValue* ObjectValue = std::get_if<CbObjectValue>(&Value))
+	if (const CbObject* ObjectValue = std::get_if<CbObject>(&Value))
 	{
-		return ObjectValue->Object;
+		return *ObjectValue;
 	}
 
 	return {};
@@ -377,6 +375,19 @@ CbPackage::AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Res
 	}
 }
 
+void
+CbPackage::AddAttachments(std::span<const CbAttachment> InAttachments)
+{
+	if (InAttachments.empty())
+	{
+		return;
+	}
+	// Assume we have no duplicates!
+	Attachments.insert(Attachments.end(), InAttachments.begin(), InAttachments.end());
+	std::sort(Attachments.begin(), Attachments.end());
+	ZEN_ASSERT_SLOW(std::unique(Attachments.begin(), Attachments.end()) == Attachments.end());
+}
+
 int32_t
 CbPackage::RemoveAttachment(const IoHash& Hash)
 {
@@ -710,7 +721,7 @@ namespace legacy {
 						{
 							return false;
 						}
-						Package.AddAttachment(CbAttachment(Compressed));
+						Package.AddAttachment(CbAttachment(Compressed, Hash));
 					}
 					else
 					{
@@ -738,7 +749,7 @@ namespace legacy {
 				{
 					if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(AttachmentData))
 					{
-						Package.AddAttachment(CbAttachment(Compressed));
+						Package.AddAttachment(CbAttachment(Compressed, Hash));
 					}
 					else
 					{
@@ -978,15 +989,15 @@ TEST_CASE("usonpackage.serialization")
 {
 	using namespace std::literals;
 
-	const auto TestSaveLoadValidate = [&](const char* Test, const CbPackage& Package) {
+	const auto TestSaveLoadValidate = [&](const char* Test, CbPackage& InOutPackage) {
 		ZEN_UNUSED(Test);
 
 		CbWriter Writer;
-		Package.Save(Writer);
+		InOutPackage.Save(Writer);
 		CbFieldIterator Fields = Writer.Save();
 
 		BinaryWriter MemStream;
-		Package.Save(MemStream);
+		InOutPackage.Save(MemStream);
 
 		CHECK(MakeMemoryView(MemStream).EqualBytes(Fields.GetRangeBuffer().GetView()));
 		CHECK(ValidateCompactBinaryRange(MakeMemoryView(MemStream), CbValidateMode::All) == CbValidateError::None);
@@ -995,13 +1006,14 @@ TEST_CASE("usonpackage.serialization")
 		CbPackage FromFields;
 		FromFields.TryLoad(Fields);
 		CHECK_FALSE(bool(Fields));
-		CHECK(FromFields == Package);
+		CHECK(FromFields == InOutPackage);
 
 		CbPackage	 FromArchive;
 		BinaryReader ReadAr(MakeMemoryView(MemStream));
 		FromArchive.TryLoad(ReadAr);
 		CHECK(ReadAr.CurrentOffset() == ReadAr.Size());
-		CHECK(FromArchive == Package);
+		CHECK(FromArchive == InOutPackage);
+		InOutPackage = FromArchive;
 	};
 
 	SUBCASE("Empty")
@@ -1083,7 +1095,7 @@ TEST_CASE("usonpackage.serialization")
 		const CbAttachment* const Object2Attachment = Package.FindAttachment(Object2.GetHash());
 
 		CHECK((Object1Attachment && Object1Attachment->AsObject().Equals(Object1)));
-		CHECK((Object2Attachment && Object2Attachment->AsBinary() == Object2.GetBuffer()));
+		CHECK((Object2Attachment && Object2Attachment->AsBinary().GetView().EqualBytes(Object2.GetBuffer().GetView())));
 
 		SharedBuffer Object1ClonedBuffer = SharedBuffer::Clone(Object1.GetOuterBuffer());
 		Package.AddAttachment(CbAttachment(Object1ClonedBuffer));
diff --git a/zencore/compositebuffer.cpp b/zencore/compositebuffer.cpp
index 3190ca5ea..e4ca93cc2 100644
--- a/zencore/compositebuffer.cpp
+++ b/zencore/compositebuffer.cpp
@@ -124,6 +124,100 @@ CompositeBuffer::ViewOrCopyRange(uint64_t Offset, uint64_t Size, UniqueBuffer& C
 	return View;
 }
 
+CompositeBuffer::Iterator
+CompositeBuffer::GetIterator(uint64_t Offset) const
+{
+	size_t SegmentCount = m_Segments.size();
+	size_t SegmentIndex = 0;
+	while (SegmentIndex < SegmentCount)
+	{
+		size_t SegmentSize = m_Segments[SegmentIndex].GetSize();
+		if (Offset < SegmentSize)
+		{
+			return {.SegmentIndex = SegmentIndex, .OffsetInSegment = Offset};
+		}
+		Offset -= SegmentSize;
+		SegmentIndex++;
+	}
+	return {.SegmentIndex = ~0ull, .OffsetInSegment = ~0ull};
+}
+
+MemoryView
+CompositeBuffer::ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& CopyBuffer) const
+{
+	MutableMemoryView WriteView;
+	size_t			  SegmentCount = m_Segments.size();
+	ZEN_ASSERT(It.SegmentIndex < SegmentCount);
+	uint64_t SizeLeft = Size;
+	while (SizeLeft > 0 && It.SegmentIndex < SegmentCount)
+	{
+		const SharedBuffer& Segment		= m_Segments[It.SegmentIndex];
+		size_t				SegmentSize = Segment.GetSize();
+		if (Size == SizeLeft && Size <= (SegmentSize - It.OffsetInSegment))
+		{
+			MemoryView View = Segment.GetView();
+			View.RightChopInline(It.OffsetInSegment);
+			View.LeftInline(SizeLeft);
+			It.OffsetInSegment += SizeLeft;
+			ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize);
+			if (It.OffsetInSegment == SegmentSize)
+			{
+				It.SegmentIndex++;
+				It.OffsetInSegment = 0;
+			}
+			return View;
+		}
+		if (WriteView.GetSize() == 0)
+		{
+			if (CopyBuffer.GetSize() < Size)
+			{
+				CopyBuffer = UniqueBuffer::Alloc(Size);
+			}
+			WriteView = CopyBuffer.GetMutableView();
+		}
+		size_t	   CopySize = zen::Min(SegmentSize - It.OffsetInSegment, SizeLeft);
+		MemoryView ReadView = Segment.GetView();
+		ReadView.RightChopInline(It.OffsetInSegment);
+		ReadView.LeftInline(CopySize);
+		WriteView = WriteView.CopyFrom(ReadView);
+		It.OffsetInSegment += CopySize;
+		ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize);
+		if (It.OffsetInSegment == SegmentSize)
+		{
+			It.SegmentIndex++;
+			It.OffsetInSegment = 0;
+		}
+		SizeLeft -= CopySize;
+	}
+	return CopyBuffer.GetView().Left(Size - SizeLeft);
+}
+
+void
+CompositeBuffer::CopyTo(MutableMemoryView WriteView, Iterator& It) const
+{
+	size_t SizeLeft		= WriteView.GetSize();
+	size_t SegmentCount = m_Segments.size();
+	ZEN_ASSERT(It.SegmentIndex < SegmentCount);
+	while (WriteView.GetSize() > 0 && It.SegmentIndex < SegmentCount)
+	{
+		const SharedBuffer& Segment		= m_Segments[It.SegmentIndex];
+		size_t				SegmentSize = Segment.GetSize();
+		size_t				CopySize	= zen::Min(SegmentSize - It.OffsetInSegment, SizeLeft);
+		MemoryView			ReadView	= Segment.GetView();
+		ReadView.RightChopInline(It.OffsetInSegment);
+		ReadView.LeftInline(CopySize);
+		WriteView = WriteView.CopyFrom(ReadView);
+		It.OffsetInSegment += CopySize;
+		ZEN_ASSERT_SLOW(It.OffsetInSegment <= SegmentSize);
+		if (It.OffsetInSegment == SegmentSize)
+		{
+			It.SegmentIndex++;
+			It.OffsetInSegment = 0;
+		}
+		SizeLeft -= CopySize;
+	}
+}
+
 void
 CompositeBuffer::CopyTo(MutableMemoryView Target, uint64_t Offset) const
 {
diff --git a/zencore/compress.cpp b/zencore/compress.cpp
index 35a5acb3a..15cc5f6a7 100644
--- a/zencore/compress.cpp
+++ b/zencore/compress.cpp
@@ -65,7 +65,8 @@ struct BufferHeader
 		BufferHeader Header;
 		if (sizeof(BufferHeader) <= CompressedData.GetSize())
 		{
-			CompressedData.CopyTo(MakeMutableMemoryView(&Header, &Header + 1));
+			CompositeBuffer::Iterator It;
+			CompressedData.CopyTo(MakeMutableMemoryView(&Header, &Header + 1), It);
 			Header.ByteSwap();
 		}
 		return Header;
@@ -235,10 +236,13 @@ BlockEncoder::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize)
 	{
 		UniqueBuffer	  RawBlockCopy;
 		MutableMemoryView CompressedBlocksView = CompressedData.GetMutableView() + sizeof(BufferHeader) + MetaSize;
+
+		CompositeBuffer::Iterator It = RawData.GetIterator(0);
+
 		for (uint64_t RawOffset = 0; RawOffset < RawSize;)
 		{
 			const uint64_t	 RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize);
-			const MemoryView RawBlock	  = RawData.ViewOrCopyRange(RawOffset, RawBlockSize, RawBlockCopy);
+			const MemoryView RawBlock	  = RawData.ViewOrCopyRange(It, RawBlockSize, RawBlockCopy);
 			RawHash.Append(RawBlock);
 
 			MutableMemoryView CompressedBlock = CompressedBlocksView;
@@ -669,8 +673,10 @@ BufferHeader::IsValid(const CompositeBuffer& CompressedData)
 		{
 			if (const BaseDecoder* const Decoder = GetDecoder(Header.Method))
 			{
-				UniqueBuffer	 HeaderCopy;
-				const MemoryView HeaderView = CompressedData.ViewOrCopyRange(0, Decoder->GetHeaderSize(Header), HeaderCopy);
+				UniqueBuffer			  HeaderCopy = UniqueBuffer::Alloc(Decoder->GetHeaderSize(Header));
+				CompositeBuffer::Iterator It;
+				CompressedData.CopyTo(HeaderCopy.GetMutableView(), It);
+				const MemoryView HeaderView = HeaderCopy.GetView();
 				if (Header.Crc32 == BufferHeader::CalculateCrc32(HeaderView))
 				{
 					return true;
@@ -851,6 +857,30 @@ CompressedBuffer::FromCompressed(SharedBuffer&& InCompressedData)
 	return Local;
 }
 
+CompressedBuffer
+CompressedBuffer::FromCompressedNoValidate(IoBuffer&& InCompressedData)
+{
+	if (InCompressedData.GetSize() <= sizeof(detail::BufferHeader))
+	{
+		return CompressedBuffer();
+	}
+	CompressedBuffer Local;
+	Local.CompressedData = CompositeBuffer(SharedBuffer(std::move(InCompressedData)));
+	return Local;
+}
+
+CompressedBuffer
+CompressedBuffer::FromCompressedNoValidate(CompositeBuffer&& InCompressedData)
+{
+	if (InCompressedData.GetSize() <= sizeof(detail::BufferHeader))
+	{
+		return CompressedBuffer();
+	}
+	CompressedBuffer Local;
+	Local.CompressedData = std::move(InCompressedData);
+	return Local;
+}
+
 uint64_t
 CompressedBuffer::GetRawSize() const
 {
@@ -927,7 +957,9 @@ CompressedBuffer::DecompressToComposite() const
 }
 
 bool
-CompressedBuffer::TryGetCompressParameters(OodleCompressor& OutCompressor, OodleCompressionLevel& OutCompressionLevel) const
+CompressedBuffer::TryGetCompressParameters(OodleCompressor&		  OutCompressor,
+										   OodleCompressionLevel& OutCompressionLevel,
+										   uint64_t&			  OutBlockSize) const
 {
 	using namespace detail;
 	if (CompressedData)
@@ -937,10 +969,12 @@ CompressedBuffer::TryGetCompressParameters(OodleCompressor& OutCompressor, Oodle
 			case CompressionMethod::None:
 				OutCompressor		= OodleCompressor::NotSet;
 				OutCompressionLevel = OodleCompressionLevel::None;
+				OutBlockSize		= 0;
 				return true;
 			case CompressionMethod::Oodle:
 				OutCompressor		= OodleCompressor(Header.Compressor);
 				OutCompressionLevel = OodleCompressionLevel(Header.CompressionLevel);
+				OutBlockSize		= uint64_t(1) << Header.BlockSizeExponent;
 				return true;
 			default:
 				break;
diff --git a/zencore/include/zencore/compactbinarypackage.h b/zencore/include/zencore/compactbinarypackage.h
index c3e587f40..16f723edc 100644
--- a/zencore/include/zencore/compactbinarypackage.h
+++ b/zencore/include/zencore/compactbinarypackage.h
@@ -64,8 +64,8 @@ public:
 	ZENCORE_API explicit CbAttachment(CompositeBuffer&& InValue, const IoHash& Hash);
 
 	/** Construct a compressed binary attachment. Value is cloned if not owned. */
-	ZENCORE_API explicit CbAttachment(const CompressedBuffer& InValue);
-	ZENCORE_API explicit CbAttachment(CompressedBuffer&& InValue);
+	ZENCORE_API explicit CbAttachment(const CompressedBuffer& InValue, const IoHash& Hash);
+	ZENCORE_API explicit CbAttachment(CompressedBuffer&& InValue, const IoHash& Hash);
 
 	/** Reset this to a null attachment. */
 	inline void Reset() { *this = CbAttachment(); }
@@ -133,28 +133,8 @@ public:
 private:
 	ZENCORE_API CbAttachment(const CbObject& Value, const IoHash* Hash);
 
-	struct CbObjectValue
-	{
-		CbObject Object;
-		IoHash	 Hash;
-
-		CbObjectValue(const CbObject& InObject, const IoHash& InHash) : Object(InObject), Hash(InHash) {}
-		CbObjectValue(CbObject&& InObject, const IoHash& InHash) : Object(std::move(InObject)), Hash(InHash) {}
-	};
-
-	struct BinaryValue
-	{
-		CompositeBuffer Buffer;
-		IoHash			Hash;
-
-		BinaryValue(const CompositeBuffer& InBuffer) : Buffer(InBuffer.MakeOwned()), Hash(IoHash::HashBuffer(InBuffer)) {}
-		BinaryValue(const CompositeBuffer& InBuffer, const IoHash& InHash) : Buffer(InBuffer.MakeOwned()), Hash(InHash) {}
-		BinaryValue(CompositeBuffer&& InBuffer) : Buffer(std::move(InBuffer)), Hash(IoHash::HashBuffer(Buffer)) {}
-		BinaryValue(CompositeBuffer&& InBuffer, const IoHash& InHash) : Buffer(std::move(InBuffer)), Hash(InHash) {}
-		BinaryValue(SharedBuffer&& InBuffer, const IoHash& InHash) : Buffer(std::move(InBuffer)), Hash(InHash) {}
-	};
-
-	std::variant<std::nullptr_t, CbObjectValue, BinaryValue, CompressedBuffer> Value;
+	IoHash																	  Hash;
+	std::variant<std::nullptr_t, CbObject, CompositeBuffer, CompressedBuffer> Value;
 };
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -300,6 +280,8 @@ public:
 	/** Add the attachment to this package, along with any references that can be resolved. */
 	inline void AddAttachment(const CbAttachment& Attachment, AttachmentResolver Resolver) { AddAttachment(Attachment, &Resolver); }
 
+	void AddAttachments(std::span<const CbAttachment> Attachments);
+
 	/**
 	 * Remove an attachment by hash.
 	 *
diff --git a/zencore/include/zencore/compositebuffer.h b/zencore/include/zencore/compositebuffer.h
index 4a3b60428..4e4b4d002 100644
--- a/zencore/include/zencore/compositebuffer.h
+++ b/zencore/include/zencore/compositebuffer.h
@@ -99,6 +99,15 @@ public:
 								  uint64_t															  Size,
 								  std::function<void(MemoryView View, const SharedBuffer& ViewOuter)> Visitor) const;
 
+	struct Iterator
+	{
+		size_t	 SegmentIndex	 = 0;
+		uint64_t OffsetInSegment = 0;
+	};
+	ZENCORE_API Iterator   GetIterator(uint64_t Offset) const;
+	ZENCORE_API MemoryView ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& CopyBuffer) const;
+	ZENCORE_API void	   CopyTo(MutableMemoryView Target, Iterator& It) const;
+
 	/** A null composite buffer. */
 	static const CompositeBuffer Null;
 
diff --git a/zencore/include/zencore/compress.h b/zencore/include/zencore/compress.h
index d37ecfa79..92dc1fb76 100644
--- a/zencore/include/zencore/compress.h
+++ b/zencore/include/zencore/compress.h
@@ -75,6 +75,8 @@ public:
 	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData);
 	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData);
 	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(IoBuffer&& CompressedData);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(CompositeBuffer&& CompressedData);
 
 	/** Reset this to null. */
 	inline void Reset() { CompressedData.Reset(); }
@@ -89,8 +91,8 @@ public:
 	[[nodiscard]] inline bool IsOwned() const { return CompressedData.IsOwned(); }
 
 	/** Returns a copy of the compressed buffer that owns its underlying memory. */
-	[[nodiscard]] inline CompressedBuffer MakeOwned() const& { return FromCompressed(CompressedData.MakeOwned()); }
-	[[nodiscard]] inline CompressedBuffer MakeOwned() && { return FromCompressed(std::move(CompressedData).MakeOwned()); }
+	[[nodiscard]] inline CompressedBuffer MakeOwned() const& { return FromCompressedNoValidate(CompressedData.MakeOwned()); }
+	[[nodiscard]] inline CompressedBuffer MakeOwned() && { return FromCompressedNoValidate(std::move(CompressedData).MakeOwned()); }
 
 	/** Returns a composite buffer containing the compressed data. May be null. May not be owned. */
 	[[nodiscard]] inline const CompositeBuffer& GetCompressed() const& { return CompressedData; }
@@ -117,7 +119,8 @@ public:
 	 * @return True if parameters were written, otherwise false.
 	 */
 	[[nodiscard]] ZENCORE_API bool TryGetCompressParameters(OodleCompressor&	   OutCompressor,
-															OodleCompressionLevel& OutCompressionLevel) const;
+															OodleCompressionLevel& OutCompressionLevel,
+															uint64_t&			   OutBlockSize) const;
 
 	/**
 	 * Decompress into a memory view that is less or equal GetRawSize() bytes.
diff --git a/zencore/include/zencore/stream.h b/zencore/include/zencore/stream.h
index ec303e1f8..9e4996249 100644
--- a/zencore/include/zencore/stream.h
+++ b/zencore/include/zencore/stream.h
@@ -28,6 +28,7 @@ public:
 	}
 
 	inline void Write(MemoryView Memory) { Write(Memory.GetData(), Memory.GetSize()); }
+	void		Write(std::initializer_list<const MemoryView> Buffers);
 
 	inline uint64_t CurrentOffset() const { return m_Offset; }
 
@@ -41,7 +42,6 @@ public:
 	inline MutableMemoryView GetMutableView() { return MutableMemoryView(m_Buffer.data(), m_Offset); }
 
 private:
-	RwLock				 m_Lock;
 	std::vector<uint8_t> m_Buffer;
 	uint64_t			 m_Offset = 0;
 
diff --git a/zencore/stream.cpp b/zencore/stream.cpp
index 8faf90af2..3402e51be 100644
--- a/zencore/stream.cpp
+++ b/zencore/stream.cpp
@@ -11,10 +11,28 @@
 namespace zen {
 
 void
-BinaryWriter::Write(const void* data, size_t ByteCount, uint64_t Offset)
+BinaryWriter::Write(std::initializer_list<const MemoryView> Buffers)
 {
-	RwLock::ExclusiveLockScope _(m_Lock);
+	size_t TotalByteCount = 0;
+	for (const MemoryView& View : Buffers)
+	{
+		TotalByteCount += View.GetSize();
+	}
+	const size_t NeedEnd = m_Offset + TotalByteCount;
+	if (NeedEnd > m_Buffer.size())
+	{
+		m_Buffer.resize(NeedEnd);
+	}
+	for (const MemoryView& View : Buffers)
+	{
+		memcpy(m_Buffer.data() + m_Offset, View.GetData(), View.GetSize());
+		m_Offset += View.GetSize();
+	}
+}
 
+void
+BinaryWriter::Write(const void* data, size_t ByteCount, uint64_t Offset)
+{
 	const size_t NeedEnd = Offset + ByteCount;
 
 	if (NeedEnd > m_Buffer.size())
@@ -28,8 +46,6 @@ BinaryWriter::Write(const void* data, size_t ByteCount, uint64_t Offset)
 void
 BinaryWriter::Reset()
 {
-	RwLock::ExclusiveLockScope _(m_Lock);
-
 	m_Buffer.clear();
 	m_Offset = 0;
 }
@@ -41,6 +57,18 @@ BinaryWriter::Reset()
 
 #if ZEN_WITH_TESTS
 
+TEST_CASE("binary.writer.span")
+{
+	BinaryWriter	 Writer;
+	const MemoryView View1("apa", 3);
+	const MemoryView View2(" ", 1);
+	const MemoryView View3("banan", 5);
+	Writer.Write({View1, View2, View3});
+	MemoryView Result = Writer.GetView();
+	CHECK(Result.GetSize() == 9);
+	CHECK(memcmp(Result.GetData(), "apa banan", 9) == 0);
+}
+
 void
 stream_forcelink()
 {