moved source directories into `/src` (#264)

* moved source directories into `/src`
* updated bundle.lua for new `src` path
* moved some docs, icon
* removed old test trees

1 files changed, 809 insertions, 0 deletions

diff --git a/src/zenhttp/httpshared.cpp b/src/zenhttp/httpshared.cpp
new file mode 100644
index 000000000..7aade56d2
--- /dev/null
+++ b/src/zenhttp/httpshared.cpp
@@ -0,0 +1,809 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenhttp/httpshared.h>
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinarypackage.h>
+#include <zencore/compositebuffer.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/logging.h>
+#include <zencore/scopeguard.h>
+#include <zencore/stream.h>
+#include <zencore/testing.h>
+#include <zencore/testutils.h>
+
+#include <span>
+#include <vector>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+const std::string_view HandlePrefix(":?#:");
+
+std::vector<IoBuffer>
+FormatPackageMessage(const CbPackage& Data, int TargetProcessPid)
+{
+	return FormatPackageMessage(Data, FormatFlags::kDefault, TargetProcessPid);
+}
+CompositeBuffer
+FormatPackageMessageBuffer(const CbPackage& Data, int TargetProcessPid)
+{
+	return FormatPackageMessageBuffer(Data, FormatFlags::kDefault, TargetProcessPid);
+}
+
+CompositeBuffer
+FormatPackageMessageBuffer(const CbPackage& Data, FormatFlags Flags, int TargetProcessPid)
+{
+	std::vector<IoBuffer> Message = FormatPackageMessage(Data, Flags, TargetProcessPid);
+
+	std::vector<SharedBuffer> Buffers;
+
+	for (IoBuffer& Buf : Message)
+	{
+		Buffers.push_back(SharedBuffer(Buf));
+	}
+
+	return CompositeBuffer(std::move(Buffers));
+}
+
+std::vector<IoBuffer>
+FormatPackageMessage(const CbPackage& Data, FormatFlags Flags, int TargetProcessPid)
+{
+	void* TargetProcessHandle = nullptr;
+#if ZEN_PLATFORM_WINDOWS
+	std::vector<HANDLE> DuplicatedHandles;
+	auto				_ = MakeGuard([&DuplicatedHandles, &TargetProcessHandle]() {
+		   if (TargetProcessHandle == nullptr)
+		   {
+			   return;
+		   }
+
+		   for (HANDLE DuplicatedHandle : DuplicatedHandles)
+		   {
+			   HANDLE ClosingHandle;
+			   if (::DuplicateHandle((HANDLE)TargetProcessHandle,
+									 DuplicatedHandle,
+									 GetCurrentProcess(),
+									 &ClosingHandle,
+									 0,
+									 FALSE,
+									 DUPLICATE_CLOSE_SOURCE | DUPLICATE_SAME_ACCESS) == TRUE)
+			   {
+				   ::CloseHandle(ClosingHandle);
+			   }
+		   }
+		   ::CloseHandle((HANDLE)TargetProcessHandle);
+		   TargetProcessHandle = nullptr;
+	   });
+
+	if (EnumHasAllFlags(Flags, FormatFlags::kAllowLocalReferences) && TargetProcessPid != 0)
+	{
+		TargetProcessHandle = OpenProcess(PROCESS_DUP_HANDLE, FALSE, TargetProcessPid);
+	}
+#else
+	ZEN_UNUSED(TargetProcessPid);
+	void* DuplicatedHandles = nullptr;
+#endif	// ZEN_PLATFORM_WINDOWS
+
+	const std::span<const CbAttachment>& Attachments = Data.GetAttachments();
+	std::vector<IoBuffer>				 ResponseBuffers;
+
+	ResponseBuffers.reserve(3 + Attachments.size());  // TODO: may want to use an additional fudge factor here to avoid growing since each
+													  // attachment is likely to consist of several buffers
+
+	// Fixed size header
+
+	CbPackageHeader Hdr{.HeaderMagic = kCbPkgMagic, .AttachmentCount = gsl::narrow<uint32_t>(Attachments.size())};
+
+	ResponseBuffers.push_back(IoBufferBuilder::MakeCloneFromMemory(&Hdr, sizeof Hdr));
+
+	// Attachment metadata array
+
+	IoBuffer		   AttachmentMetadataBuffer = IoBuffer{sizeof(CbAttachmentEntry) * (Attachments.size() + /* root */ 1)};
+	CbAttachmentEntry* AttachmentInfo			= reinterpret_cast<CbAttachmentEntry*>(AttachmentMetadataBuffer.MutableData());
+
+	ResponseBuffers.push_back(AttachmentMetadataBuffer);  // Attachment metadata
+
+	// Root object
+
+	IoBuffer RootIoBuffer = Data.GetObject().GetBuffer().AsIoBuffer();
+	ResponseBuffers.push_back(RootIoBuffer);  // Root object
+
+	*AttachmentInfo++ = {.PayloadSize = RootIoBuffer.Size(), .Flags = CbAttachmentEntry::kIsObject, .AttachmentHash = Data.GetObjectHash()};
+
+	// Attachment payloads
+
+	auto MarshalLocal = [&AttachmentInfo, &ResponseBuffers](const std::string&			 Path8,
+															CbAttachmentReferenceHeader& LocalRef,
+															const IoHash&				 AttachmentHash,
+															bool						 IsCompressed) {
+		IoBuffer RefBuffer(sizeof(CbAttachmentReferenceHeader) + Path8.size());
+
+		CbAttachmentReferenceHeader* RefHdr = RefBuffer.MutableData<CbAttachmentReferenceHeader>();
+		*RefHdr++							= LocalRef;
+		memcpy(RefHdr, Path8.data(), Path8.size());
+
+		*AttachmentInfo++ = {.PayloadSize = RefBuffer.GetSize(),
+							 .Flags = (IsCompressed ? uint32_t(CbAttachmentEntry::kIsCompressed) : 0u) | CbAttachmentEntry::kIsLocalRef,
+							 .AttachmentHash = AttachmentHash};
+
+		ResponseBuffers.push_back(std::move(RefBuffer));
+	};
+
+	tsl::robin_map<void*, std::string> FileNameMap;
+
+	auto IsLocalRef = [&FileNameMap, &DuplicatedHandles](const CompositeBuffer&		  AttachmentBinary,
+														 bool						  DenyPartialLocalReferences,
+														 void*						  TargetProcessHandle,
+														 CbAttachmentReferenceHeader& LocalRef,
+														 std::string&				  Path8) -> bool {
+		const SharedBuffer&	  Segment = AttachmentBinary.GetSegments().front();
+		IoBufferFileReference Ref;
+		const IoBuffer&		  SegmentBuffer = Segment.AsIoBuffer();
+
+		if (!SegmentBuffer.GetFileReference(Ref))
+		{
+			return false;
+		}
+
+		if (DenyPartialLocalReferences && !SegmentBuffer.IsWholeFile())
+		{
+			return false;
+		}
+
+		if (auto It = FileNameMap.find(Ref.FileHandle); It != FileNameMap.end())
+		{
+			Path8 = It->second;
+		}
+		else
+		{
+			bool UseFilePath = true;
+#if ZEN_PLATFORM_WINDOWS
+			if (TargetProcessHandle != nullptr)
+			{
+				HANDLE TargetHandle = INVALID_HANDLE_VALUE;
+				BOOL   OK			= ::DuplicateHandle(GetCurrentProcess(),
+											Ref.FileHandle,
+											(HANDLE)TargetProcessHandle,
+											&TargetHandle,
+											FILE_GENERIC_READ,
+											FALSE,
+											0);
+				if (OK)
+				{
+					DuplicatedHandles.push_back(TargetHandle);
+					Path8		= fmt::format("{}{}", HandlePrefix, reinterpret_cast<uint64_t>(TargetHandle));
+					UseFilePath = false;
+				}
+			}
+#else	// ZEN_PLATFORM_WINDOWS
+			ZEN_UNUSED(TargetProcessHandle);
+			// Not supported on Linux/Mac. Could potentially use pidfd_getfd() but that requires a fairly new Linux kernel/includes and to
+			// deal with acceess rights etc.
+#endif	// ZEN_PLATFORM_WINDOWS
+			if (UseFilePath)
+			{
+				ExtendablePathBuilder<256> LocalRefFile;
+				LocalRefFile.Append(std::filesystem::absolute(PathFromHandle(Ref.FileHandle)));
+				Path8 = LocalRefFile.ToUtf8();
+			}
+			FileNameMap.insert_or_assign(Ref.FileHandle, Path8);
+		}
+
+		LocalRef.AbsolutePathLength = gsl::narrow<uint16_t>(Path8.size());
+		LocalRef.PayloadByteOffset	= Ref.FileChunkOffset;
+		LocalRef.PayloadByteSize	= Ref.FileChunkSize;
+
+		return true;
+	};
+
+	for (const CbAttachment& Attachment : Attachments)
+	{
+		if (Attachment.IsNull())
+		{
+			ZEN_NOT_IMPLEMENTED("Null attachments are not supported");
+		}
+		else if (CompressedBuffer AttachmentBuffer = Attachment.AsCompressedBinary())
+		{
+			CompositeBuffer Compressed	   = AttachmentBuffer.GetCompressed();
+			IoHash			AttachmentHash = Attachment.GetHash();
+
+			// If the data is either not backed by a file, or there are multiple
+			// fragments then we cannot marshal it by local reference. We might
+			// want/need to extend this in the future to allow multiple chunk
+			// segments to be marshaled at once
+
+			bool MarshalByLocalRef = EnumHasAllFlags(Flags, FormatFlags::kAllowLocalReferences) && (Compressed.GetSegments().size() == 1);
+			bool DenyPartialLocalReferences = EnumHasAllFlags(Flags, FormatFlags::kDenyPartialLocalReferences);
+			CbAttachmentReferenceHeader LocalRef;
+			std::string					Path8;
+
+			if (MarshalByLocalRef)
+			{
+				MarshalByLocalRef = IsLocalRef(Compressed, DenyPartialLocalReferences, TargetProcessHandle, LocalRef, Path8);
+			}
+
+			if (MarshalByLocalRef)
+			{
+				const bool IsCompressed = true;
+				bool	   IsHandle		= false;
+#if ZEN_PLATFORM_WINDOWS
+				IsHandle = Path8.starts_with(HandlePrefix);
+#endif
+				MarshalLocal(Path8, LocalRef, AttachmentHash, IsCompressed);
+				ZEN_DEBUG("Marshalled '{}' as file {} of {} bytes", Path8, IsHandle ? "handle" : "path", Compressed.GetSize());
+			}
+			else
+			{
+				*AttachmentInfo++ = {.PayloadSize	 = AttachmentBuffer.GetCompressedSize(),
+									 .Flags			 = CbAttachmentEntry::kIsCompressed,
+									 .AttachmentHash = AttachmentHash};
+
+				for (const SharedBuffer& Segment : Compressed.GetSegments())
+				{
+					ResponseBuffers.push_back(Segment.AsIoBuffer());
+				}
+			}
+		}
+		else if (CbObject AttachmentObject = Attachment.AsObject())
+		{
+			IoBuffer ObjIoBuffer = AttachmentObject.GetBuffer().AsIoBuffer();
+			ResponseBuffers.push_back(ObjIoBuffer);
+
+			*AttachmentInfo++ = {.PayloadSize	 = ObjIoBuffer.Size(),
+								 .Flags			 = CbAttachmentEntry::kIsObject,
+								 .AttachmentHash = Attachment.GetHash()};
+		}
+		else if (CompositeBuffer AttachmentBinary = Attachment.AsCompositeBinary())
+		{
+			IoHash AttachmentHash = Attachment.GetHash();
+			bool   MarshalByLocalRef =
+				EnumHasAllFlags(Flags, FormatFlags::kAllowLocalReferences) && (AttachmentBinary.GetSegments().size() == 1);
+			bool DenyPartialLocalReferences = EnumHasAllFlags(Flags, FormatFlags::kDenyPartialLocalReferences);
+
+			CbAttachmentReferenceHeader LocalRef;
+			std::string					Path8;
+
+			if (MarshalByLocalRef)
+			{
+				MarshalByLocalRef = IsLocalRef(AttachmentBinary, DenyPartialLocalReferences, TargetProcessHandle, LocalRef, Path8);
+			}
+
+			if (MarshalByLocalRef)
+			{
+				const bool IsCompressed = false;
+				bool	   IsHandle		= false;
+#if ZEN_PLATFORM_WINDOWS
+				IsHandle = Path8.starts_with(HandlePrefix);
+#endif
+				MarshalLocal(Path8, LocalRef, AttachmentHash, IsCompressed);
+				ZEN_DEBUG("Marshalled '{}' as file {} of {} bytes", Path8, IsHandle ? "handle" : "path", AttachmentBinary.GetSize());
+			}
+			else
+			{
+				*AttachmentInfo++ = {.PayloadSize = AttachmentBinary.GetSize(), .Flags = 0, .AttachmentHash = Attachment.GetHash()};
+
+				for (const SharedBuffer& Segment : AttachmentBinary.GetSegments())
+				{
+					ResponseBuffers.push_back(Segment.AsIoBuffer());
+				}
+			}
+		}
+		else
+		{
+			ZEN_NOT_IMPLEMENTED("Unknown attachment kind");
+		}
+	}
+	FileNameMap.clear();
+#if ZEN_PLATFORM_WINDOWS
+	DuplicatedHandles.clear();
+#endif	// ZEN_PLATFORM_WINDOWS
+
+	return ResponseBuffers;
+}
+
+bool
+IsPackageMessage(IoBuffer Payload)
+{
+	if (!Payload)
+	{
+		return false;
+	}
+
+	BinaryReader Reader(Payload);
+
+	CbPackageHeader Hdr;
+	Reader.Read(&Hdr, sizeof Hdr);
+
+	if (Hdr.HeaderMagic != kCbPkgMagic)
+	{
+		return false;
+	}
+
+	return true;
+}
+
+CbPackage
+ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint64_t)> CreateBuffer)
+{
+	if (!Payload)
+	{
+		return {};
+	}
+
+	BinaryReader Reader(Payload);
+
+	CbPackageHeader Hdr;
+	Reader.Read(&Hdr, sizeof Hdr);
+
+	if (Hdr.HeaderMagic != kCbPkgMagic)
+	{
+		throw std::runtime_error("invalid CbPackage header magic");
+	}
+
+	const uint32_t ChunkCount = Hdr.AttachmentCount + 1;
+
+	std::unique_ptr<CbAttachmentEntry[]> AttachmentEntries{new CbAttachmentEntry[ChunkCount]};
+
+	Reader.Read(AttachmentEntries.get(), sizeof(CbAttachmentEntry) * ChunkCount);
+
+	CbPackage Package;
+
+	std::vector<CbAttachment> Attachments;
+	Attachments.reserve(ChunkCount);  // Guessing here...
+
+	tsl::robin_map<std::string, IoBuffer> PartialFileBuffers;
+
+	// TODO: Throwing before this loop completes could result in leaking handles as we might not have picked up all the handles in the
+	// message
+	for (uint32_t i = 0; i < ChunkCount; ++i)
+	{
+		const CbAttachmentEntry& Entry			= AttachmentEntries[i];
+		const uint64_t			 AttachmentSize = Entry.PayloadSize;
+
+		const IoBuffer AttachmentBuffer(Payload, Reader.CurrentOffset(), AttachmentSize);
+		Reader.Skip(AttachmentSize);
+
+		if (Entry.Flags & CbAttachmentEntry::kIsLocalRef)
+		{
+			// Marshal local reference - a "pointer" to the chunk backing file
+
+			ZEN_ASSERT(AttachmentBuffer.Size() >= sizeof(CbAttachmentReferenceHeader));
+
+			const CbAttachmentReferenceHeader* AttachRefHdr = AttachmentBuffer.Data<CbAttachmentReferenceHeader>();
+			const char*						   PathPointer	= reinterpret_cast<const char*>(AttachRefHdr + 1);
+
+			ZEN_ASSERT(AttachmentBuffer.Size() >= (sizeof(CbAttachmentReferenceHeader) + AttachRefHdr->AbsolutePathLength));
+			std::string_view PathView(PathPointer, AttachRefHdr->AbsolutePathLength);
+
+			IoBuffer FullFileBuffer;
+
+			std::filesystem::path Path(Utf8ToWide(PathView));
+			if (auto It = PartialFileBuffers.find(Path.string()); It != PartialFileBuffers.end())
+			{
+				FullFileBuffer = It->second;
+			}
+			else
+			{
+				if (PathView.starts_with(HandlePrefix))
+				{
+#if ZEN_PLATFORM_WINDOWS
+					std::string_view		HandleString(PathView.substr(HandlePrefix.length()));
+					std::optional<uint64_t> HandleNumber(ParseInt<uint64_t>(HandleString));
+					if (HandleNumber.has_value())
+					{
+						HANDLE		   FileHandle = HANDLE(HandleNumber.value());
+						ULARGE_INTEGER liFileSize;
+						liFileSize.LowPart = ::GetFileSize(FileHandle, &liFileSize.HighPart);
+						if (liFileSize.LowPart != INVALID_FILE_SIZE)
+						{
+							FullFileBuffer = IoBuffer(IoBuffer::File, (void*)FileHandle, 0, uint64_t(liFileSize.QuadPart));
+							PartialFileBuffers.insert_or_assign(Path.string(), FullFileBuffer);
+						}
+					}
+#else	// ZEN_PLATFORM_WINDOWS
+		// Not supported on Linux/Mac. Could potentially use pidfd_getfd() but that requires a fairly new Linux kernel/includes
+		// and to deal with acceess rights etc.
+					ZEN_ASSERT(false);
+#endif	// ZEN_PLATFORM_WINDOWS
+				}
+				else
+				{
+					FullFileBuffer = PartialFileBuffers.insert_or_assign(Path.string(), IoBufferBuilder::MakeFromFile(Path)).first->second;
+				}
+			}
+
+			if (!FullFileBuffer)
+			{
+				// Unable to open chunk reference
+				throw std::runtime_error(fmt::format("unable to resolve chunk #{} at '{}' (offset {}, size {})",
+													 i,
+													 Path,
+													 AttachRefHdr->PayloadByteOffset,
+													 AttachRefHdr->PayloadByteSize));
+			}
+
+			IoBuffer ChunkReference = AttachRefHdr->PayloadByteOffset == 0 && AttachRefHdr->PayloadByteSize == FullFileBuffer.GetSize()
+										  ? FullFileBuffer
+										  : IoBuffer(FullFileBuffer, AttachRefHdr->PayloadByteOffset, AttachRefHdr->PayloadByteSize);
+
+			CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkReference)));
+			if (!CompBuf)
+			{
+				throw std::runtime_error(fmt::format("invalid format for chunk #{} at '{}' (offset {}, size {})",
+													 i,
+													 Path,
+													 AttachRefHdr->PayloadByteOffset,
+													 AttachRefHdr->PayloadByteSize));
+			}
+			Attachments.emplace_back(CbAttachment(std::move(CompBuf), Entry.AttachmentHash));
+		}
+		else if (Entry.Flags & CbAttachmentEntry::kIsCompressed)
+		{
+			if (Entry.Flags & CbAttachmentEntry::kIsObject)
+			{
+				if (i == 0)
+				{
+					CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer)));
+					if (!CompBuf)
+					{
+						throw std::runtime_error(fmt::format("invalid format for chunk #{} expected compressed buffer for CbObject", i));
+					}
+					// First payload is always a compact binary object
+					Package.SetObject(LoadCompactBinaryObject(std::move(CompBuf)));
+				}
+				else
+				{
+					ZEN_NOT_IMPLEMENTED("Object attachments are not currently supported");
+				}
+			}
+			else
+			{
+				CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer)));
+				if (!CompBuf)
+				{
+					throw std::runtime_error(fmt::format("invalid format for chunk #{} expected compressed buffer for attachment", i));
+				}
+				Attachments.emplace_back(CbAttachment(std::move(CompBuf), Entry.AttachmentHash));
+			}
+		}
+		else /* not compressed */
+		{
+			if (Entry.Flags & CbAttachmentEntry::kIsObject)
+			{
+				if (i == 0)
+				{
+					Package.SetObject(LoadCompactBinaryObject(AttachmentBuffer));
+				}
+				else
+				{
+					ZEN_NOT_IMPLEMENTED("Object attachments are not currently supported");
+				}
+			}
+			else
+			{
+				// Make a copy of the buffer so we attachements don't reference the entire payload
+				IoBuffer AttachmentBufferCopy = CreateBuffer(Entry.AttachmentHash, AttachmentSize);
+				ZEN_ASSERT(AttachmentBufferCopy);
+				ZEN_ASSERT(AttachmentBufferCopy.Size() == AttachmentSize);
+				AttachmentBufferCopy.GetMutableView().CopyFrom(AttachmentBuffer.GetView());
+
+				CbAttachment Attachment(SharedBuffer{AttachmentBufferCopy});
+				Attachments.emplace_back(SharedBuffer{AttachmentBufferCopy});
+			}
+		}
+	}
+	PartialFileBuffers.clear();
+
+	Package.AddAttachments(Attachments);
+
+	return Package;
+}
+
+bool
+ParsePackageMessageWithLegacyFallback(const IoBuffer& Response, CbPackage& OutPackage)
+{
+	if (IsPackageMessage(Response))
+	{
+		OutPackage = ParsePackageMessage(Response);
+		return true;
+	}
+	return OutPackage.TryLoad(Response);
+}
+
+CbPackageReader::CbPackageReader() : m_CreateBuffer([](const IoHash&, uint64_t Size) -> IoBuffer { return IoBuffer{Size}; })
+{
+}
+
+CbPackageReader::~CbPackageReader()
+{
+}
+
+void
+CbPackageReader::SetPayloadBufferCreator(std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> CreateBuffer)
+{
+	m_CreateBuffer = CreateBuffer;
+}
+
+uint64_t
+CbPackageReader::ProcessPackageHeaderData(const void* Data, uint64_t DataBytes)
+{
+	ZEN_ASSERT(m_CurrentState != State::kReadingBuffers);
+
+	switch (m_CurrentState)
+	{
+		case State::kInitialState:
+			ZEN_ASSERT(Data == nullptr);
+			m_CurrentState = State::kReadingHeader;
+			return sizeof m_PackageHeader;
+
+		case State::kReadingHeader:
+			ZEN_ASSERT(DataBytes == sizeof m_PackageHeader);
+			memcpy(&m_PackageHeader, Data, sizeof m_PackageHeader);
+			ZEN_ASSERT(m_PackageHeader.HeaderMagic == kCbPkgMagic);
+			m_CurrentState = State::kReadingAttachmentEntries;
+			m_AttachmentEntries.resize(m_PackageHeader.AttachmentCount + 1);
+			return (m_PackageHeader.AttachmentCount + 1) * sizeof(CbAttachmentEntry);
+
+		case State::kReadingAttachmentEntries:
+			ZEN_ASSERT(DataBytes == ((m_PackageHeader.AttachmentCount + 1) * sizeof(CbAttachmentEntry)));
+			memcpy(m_AttachmentEntries.data(), Data, DataBytes);
+
+			for (CbAttachmentEntry& Entry : m_AttachmentEntries)
+			{
+				// This preallocates memory for payloads but note that for the local references
+				// the caller will need to handle the payload differently (i.e it's a
+				// CbAttachmentReferenceHeader not the actual payload)
+
+				m_PayloadBuffers.push_back(IoBuffer{Entry.PayloadSize});
+			}
+
+			m_CurrentState = State::kReadingBuffers;
+			return 0;
+
+		default:
+			ZEN_ASSERT(false);
+			return 0;
+	}
+}
+
+IoBuffer
+CbPackageReader::MarshalLocalChunkReference(IoBuffer AttachmentBuffer)
+{
+	// Marshal local reference - a "pointer" to the chunk backing file
+
+	ZEN_ASSERT(AttachmentBuffer.Size() >= sizeof(CbAttachmentReferenceHeader));
+
+	const CbAttachmentReferenceHeader* AttachRefHdr = AttachmentBuffer.Data<CbAttachmentReferenceHeader>();
+	const char8_t*					   PathPointer	= reinterpret_cast<const char8_t*>(AttachRefHdr + 1);
+
+	ZEN_ASSERT(AttachmentBuffer.Size() >= (sizeof(CbAttachmentReferenceHeader) + AttachRefHdr->AbsolutePathLength));
+
+	std::u8string_view PathView{PathPointer, AttachRefHdr->AbsolutePathLength};
+
+	std::filesystem::path Path{PathView};
+
+	IoBuffer ChunkReference = IoBufferBuilder::MakeFromFile(Path, AttachRefHdr->PayloadByteOffset, AttachRefHdr->PayloadByteSize);
+
+	if (!ChunkReference)
+	{
+		// Unable to open chunk reference
+
+		throw std::runtime_error(fmt::format("unable to resolve local reference to '{}' (offset {}, size {})",
+											 PathToUtf8(Path),
+											 AttachRefHdr->PayloadByteOffset,
+											 AttachRefHdr->PayloadByteSize));
+	}
+
+	return ChunkReference;
+};
+
+void
+CbPackageReader::Finalize()
+{
+	if (m_AttachmentEntries.empty())
+	{
+		return;
+	}
+
+	m_Attachments.reserve(m_AttachmentEntries.size() - 1);
+
+	int CurrentAttachmentIndex = 0;
+	for (CbAttachmentEntry& Entry : m_AttachmentEntries)
+	{
+		IoBuffer AttachmentBuffer = m_PayloadBuffers[CurrentAttachmentIndex];
+
+		if (CurrentAttachmentIndex == 0)
+		{
+			// Root object
+			if (Entry.Flags & CbAttachmentEntry::kIsObject)
+			{
+				if (Entry.Flags & CbAttachmentEntry::kIsLocalRef)
+				{
+					m_RootObject = LoadCompactBinaryObject(MarshalLocalChunkReference(AttachmentBuffer));
+				}
+				else if (Entry.Flags & CbAttachmentEntry::kIsCompressed)
+				{
+					IoHash			 RawHash;
+					uint64_t		 RawSize;
+					CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer), RawHash, RawSize);
+					if (RawHash == Entry.AttachmentHash)
+					{
+						m_RootObject = LoadCompactBinaryObject(Compressed);
+					}
+				}
+				else
+				{
+					m_RootObject = LoadCompactBinaryObject(std::move(AttachmentBuffer));
+				}
+			}
+			else
+			{
+				throw std::runtime_error("missing or invalid root object");
+			}
+		}
+		else if (Entry.Flags & CbAttachmentEntry::kIsLocalRef)
+		{
+			IoBuffer ChunkReference = MarshalLocalChunkReference(AttachmentBuffer);
+
+			if (Entry.Flags & CbAttachmentEntry::kIsCompressed)
+			{
+				IoHash			 RawHash;
+				uint64_t		 RawSize;
+				CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(ChunkReference), RawHash, RawSize);
+				if (RawHash == Entry.AttachmentHash)
+				{
+					m_Attachments.push_back(CbAttachment(Compressed, Entry.AttachmentHash));
+				}
+			}
+			else
+			{
+				CompressedBuffer Compressed =
+					CompressedBuffer::Compress(SharedBuffer(ChunkReference), OodleCompressor::NotSet, OodleCompressionLevel::None);
+				m_Attachments.push_back(CbAttachment(std::move(Compressed), Compressed.DecodeRawHash()));
+			}
+		}
+
+		++CurrentAttachmentIndex;
+	}
+}
+
+/**
+	______________________ _____________________________
+	\__    ___/\_   _____//   _____/\__    ___/   _____/
+	  |    |    |    __)_ \_____  \   |    |  \_____  \
+	  |    |    |        \/        \  |    |  /        \
+	  |____|   /_______  /_______  /  |____| /_______  /
+					   \/        \/                  \/
+ */
+
+#if ZEN_WITH_TESTS
+
+TEST_CASE("CbPackage.Serialization")
+{
+	// Make a test package
+
+	CbAttachment Attach1{SharedBuffer::MakeView(MakeMemoryView("abcd"))};
+	CbAttachment Attach2{SharedBuffer::MakeView(MakeMemoryView("efgh"))};
+
+	CbObjectWriter Cbo;
+	Cbo.AddAttachment("abcd", Attach1);
+	Cbo.AddAttachment("efgh", Attach2);
+
+	CbPackage Pkg;
+	Pkg.AddAttachment(Attach1);
+	Pkg.AddAttachment(Attach2);
+	Pkg.SetObject(Cbo.Save());
+
+	SharedBuffer   Buffer		  = FormatPackageMessageBuffer(Pkg).Flatten();
+	const uint8_t* CursorPtr	  = reinterpret_cast<const uint8_t*>(Buffer.GetData());
+	uint64_t	   RemainingBytes = Buffer.GetSize();
+
+	auto ConsumeBytes = [&](uint64_t ByteCount) {
+		ZEN_ASSERT(ByteCount <= RemainingBytes);
+		void* ReturnPtr = (void*)CursorPtr;
+		CursorPtr += ByteCount;
+		RemainingBytes -= ByteCount;
+		return ReturnPtr;
+	};
+
+	auto CopyBytes = [&](void* TargetBuffer, uint64_t ByteCount) {
+		ZEN_ASSERT(ByteCount <= RemainingBytes);
+		memcpy(TargetBuffer, CursorPtr, ByteCount);
+		CursorPtr += ByteCount;
+		RemainingBytes -= ByteCount;
+	};
+
+	CbPackageReader Reader;
+	uint64_t		InitialRead = Reader.ProcessPackageHeaderData(nullptr, 0);
+	uint64_t		NextBytes	= Reader.ProcessPackageHeaderData(ConsumeBytes(InitialRead), InitialRead);
+	NextBytes					= Reader.ProcessPackageHeaderData(ConsumeBytes(NextBytes), NextBytes);
+	auto Buffers				= Reader.GetPayloadBuffers();
+
+	for (auto& PayloadBuffer : Buffers)
+	{
+		CopyBytes(PayloadBuffer.MutableData(), PayloadBuffer.GetSize());
+	}
+
+	Reader.Finalize();
+}
+
+TEST_CASE("CbPackage.LocalRef")
+{
+	ScopedTemporaryDirectory TempDir;
+
+	auto Path1 = TempDir.Path() / "abcd";
+	auto Path2 = TempDir.Path() / "efgh";
+
+	{
+		IoBuffer Buffer1 = IoBufferBuilder::MakeCloneFromMemory(MakeMemoryView("abcd"));
+		IoBuffer Buffer2 = IoBufferBuilder::MakeCloneFromMemory(MakeMemoryView("efgh"));
+
+		WriteFile(Path1, Buffer1);
+		WriteFile(Path2, Buffer2);
+	}
+
+	// Make a test package
+
+	IoBuffer FileBuffer1 = IoBufferBuilder::MakeFromFile(Path1);
+	IoBuffer FileBuffer2 = IoBufferBuilder::MakeFromFile(Path2);
+
+	CbAttachment Attach1{SharedBuffer(FileBuffer1)};
+	CbAttachment Attach2{SharedBuffer(FileBuffer2)};
+
+	CbObjectWriter Cbo;
+	Cbo.AddAttachment("abcd", Attach1);
+	Cbo.AddAttachment("efgh", Attach2);
+
+	CbPackage Pkg;
+	Pkg.AddAttachment(Attach1);
+	Pkg.AddAttachment(Attach2);
+	Pkg.SetObject(Cbo.Save());
+
+	SharedBuffer   Buffer		  = FormatPackageMessageBuffer(Pkg, FormatFlags::kAllowLocalReferences).Flatten();
+	const uint8_t* CursorPtr	  = reinterpret_cast<const uint8_t*>(Buffer.GetData());
+	uint64_t	   RemainingBytes = Buffer.GetSize();
+
+	auto ConsumeBytes = [&](uint64_t ByteCount) {
+		ZEN_ASSERT(ByteCount <= RemainingBytes);
+		void* ReturnPtr = (void*)CursorPtr;
+		CursorPtr += ByteCount;
+		RemainingBytes -= ByteCount;
+		return ReturnPtr;
+	};
+
+	auto CopyBytes = [&](void* TargetBuffer, uint64_t ByteCount) {
+		ZEN_ASSERT(ByteCount <= RemainingBytes);
+		memcpy(TargetBuffer, CursorPtr, ByteCount);
+		CursorPtr += ByteCount;
+		RemainingBytes -= ByteCount;
+	};
+
+	CbPackageReader Reader;
+	uint64_t		InitialRead = Reader.ProcessPackageHeaderData(nullptr, 0);
+	uint64_t		NextBytes	= Reader.ProcessPackageHeaderData(ConsumeBytes(InitialRead), InitialRead);
+	NextBytes					= Reader.ProcessPackageHeaderData(ConsumeBytes(NextBytes), NextBytes);
+	auto Buffers				= Reader.GetPayloadBuffers();
+
+	for (auto& PayloadBuffer : Buffers)
+	{
+		CopyBytes(PayloadBuffer.MutableData(), PayloadBuffer.GetSize());
+	}
+
+	Reader.Finalize();
+}
+
+void
+forcelink_httpshared()
+{
+}
+
+#endif
+
+}  // namespace zen