Improved package serialization to allow round tripping

7 files changed, 144 insertions, 23 deletions

diff --git a/zenhttp/httpclient.cpp b/zenhttp/httpclient.cpp
index 7e3e9d374..fb1df30b2 100644
--- a/zenhttp/httpclient.cpp
+++ b/zenhttp/httpclient.cpp
@@ -10,8 +10,7 @@
 #include <zencore/session.h>
 #include <zencore/sharedbuffer.h>
 #include <zencore/stream.h>
-
-#include "httpshared.h"
+#include <zenhttp/httpshared.h>
 
 #include <doctest/doctest.h>
 
diff --git a/zenhttp/httpserver.cpp b/zenhttp/httpserver.cpp
index f4a5f4345..62ee66a08 100644
--- a/zenhttp/httpserver.cpp
+++ b/zenhttp/httpserver.cpp
@@ -3,7 +3,6 @@
 #include <zenhttp/httpserver.h>
 
 #include "httpnull.h"
-#include "httpshared.h"
 #include "httpsys.h"
 #include "httpuws.h"
 
@@ -15,6 +14,7 @@
 #include <zencore/stream.h>
 #include <zencore/string.h>
 #include <zencore/thread.h>
+#include <zenhttp/httpshared.h>
 
 #include <conio.h>
 #include <new.h>
diff --git a/zenhttp/httpshared.cpp b/zenhttp/httpshared.cpp
index 68252a763..2dbf95959 100644
--- a/zenhttp/httpshared.cpp
+++ b/zenhttp/httpshared.cpp
@@ -1,6 +1,6 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
-#include "httpshared.h"
+#include <zenhttp/httpshared.h>
 
 #include <zencore/compactbinarypackage.h>
 #include <zencore/compositebuffer.h>
@@ -58,22 +58,54 @@ FormatPackageMessage(const CbPackage& Data)
 	IoBuffer RootIoBuffer = Data.GetObject().GetBuffer().AsIoBuffer();
 	ResponseBuffers.push_back(RootIoBuffer);  // Root object
 
-	*AttachmentInfo++ = {.AttachmentSize = RootIoBuffer.Size(), .AttachmentHash = Data.GetObjectHash()};
+	*AttachmentInfo++ = {.AttachmentSize = RootIoBuffer.Size(),
+						 .Flags			 = CbAttachmentEntry::kIsObject,
+						 .AttachmentHash = Data.GetObjectHash()};
 
 	// Attachment payloads
 
 	for (const CbAttachment& Attachment : Attachments)
 	{
-		CompressedBuffer AttachmentBuffer = Attachment.AsCompressedBinary();
-		CompositeBuffer	 Compressed		  = AttachmentBuffer.GetCompressed();
+		if (Attachment.IsNull())
+		{
+			ZEN_NOT_IMPLEMENTED("Null attachments are not supported");
+		}
+		else if (CompressedBuffer AttachmentBuffer = Attachment.AsCompressedBinary())
+		{
+			CompositeBuffer Compressed = AttachmentBuffer.GetCompressed();
 
-		*AttachmentInfo++ = {.AttachmentSize = AttachmentBuffer.GetCompressedSize(),
-							 .AttachmentHash = IoHash::FromBLAKE3(AttachmentBuffer.GetRawHash())};
+			*AttachmentInfo++ = {.AttachmentSize = AttachmentBuffer.GetCompressedSize(),
+								 .Flags			 = CbAttachmentEntry::kIsCompressed,
+								 .AttachmentHash = IoHash::FromBLAKE3(AttachmentBuffer.GetRawHash())};
 
-		for (const SharedBuffer& Segment : Compressed.GetSegments())
+			for (const SharedBuffer& Segment : Compressed.GetSegments())
+			{
+				ResponseBuffers.push_back(Segment.AsIoBuffer());
+				TotalAttachmentsSize += Segment.GetSize();
+			}
+		}
+		else if (CbObject AttachmentObject = Attachment.AsObject())
 		{
-			ResponseBuffers.push_back(Segment.AsIoBuffer());
-			TotalAttachmentsSize += Segment.GetSize();
+			IoBuffer ObjIoBuffer = AttachmentObject.GetBuffer().AsIoBuffer();
+			ResponseBuffers.push_back(ObjIoBuffer);
+
+			*AttachmentInfo++ = {.AttachmentSize = ObjIoBuffer.Size(),
+								 .Flags			 = CbAttachmentEntry::kIsObject,
+								 .AttachmentHash = Attachment.GetHash()};
+		}
+		else if (CompositeBuffer AttachmentBinary = Attachment.AsCompositeBinary())
+		{
+			*AttachmentInfo++ = {.AttachmentSize = AttachmentBinary.GetSize(), .Flags = 0, .AttachmentHash = Attachment.GetHash()};
+
+			for (const SharedBuffer& Segment : AttachmentBinary.GetSegments())
+			{
+				ResponseBuffers.push_back(Segment.AsIoBuffer());
+				TotalAttachmentsSize += Segment.GetSize();
+			}
+		}
+		else
+		{
+			ZEN_NOT_IMPLEMENTED("Unknown attachment kind");
 		}
 	}
 
@@ -119,16 +151,45 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
 
 		Reader.Read(AttachmentBuffer.MutableData(), AttachmentSize);
 
-		CompressedBuffer CompBuf(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer)));
-
-		if (i == 0)
+		if (Entry.Flags & CbAttachmentEntry::kIsCompressed)
 		{
-			Package.SetObject(LoadCompactBinaryObject(std::move(CompBuf)));
+			CompressedBuffer CompBuf(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer)));
+
+			if (Entry.Flags & CbAttachmentEntry::kIsObject)
+			{
+				if (i == 0)
+				{
+					Package.SetObject(LoadCompactBinaryObject(std::move(CompBuf)));
+				}
+				else
+				{
+					ZEN_NOT_IMPLEMENTED("Object attachments are not currently supported");
+				}
+			}
+			else
+			{
+				CbAttachment Attachment(std::move(CompBuf));
+				Package.AddAttachment(Attachment);
+			}
 		}
-		else
+		else /* not compressed */
 		{
-			CbAttachment Attachment(std::move(CompBuf));
-			Package.AddAttachment(Attachment);
+			if (Entry.Flags & CbAttachmentEntry::kIsObject)
+			{
+				if (i == 0)
+				{
+					Package.SetObject(LoadCompactBinaryObject(AttachmentBuffer));
+				}
+				else
+				{
+					ZEN_NOT_IMPLEMENTED("Object attachments are not currently supported");
+				}
+			}
+			else
+			{
+				CbAttachment Attachment(SharedBuffer{AttachmentBuffer});
+				Package.AddAttachment(Attachment);
+			}
 		}
 	}
 
diff --git a/zenhttp/httpshared.h b/zenhttp/httpshared.h
index 06fdb104f..06dc4a872 100644
--- a/zenhttp/httpshared.h
+++ b/zenhttp/httpshared.h
@@ -28,8 +28,14 @@ static constinit uint32_t kCbPkgMagic = 0xaa77aacc;
 struct CbAttachmentEntry
 {
 	uint64_t AttachmentSize;
-	uint32_t Reserved1;
+	uint32_t Flags;
 	IoHash	 AttachmentHash;
+
+	enum
+	{
+		kIsCompressed = (1u << 0),		// Is marshaled using compressed buffer storage format
+		kIsObject	  = (1u << 1),		// Is compact binary object
+	};
 };
 
 static_assert(sizeof(CbAttachmentEntry) == 32);
diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp
index 4536d0ed9..c2d4ef14c 100644
--- a/zenhttp/httpsys.cpp
+++ b/zenhttp/httpsys.cpp
@@ -2,8 +2,6 @@
 
 #include "httpsys.h"
 
-#include "httpshared.h"
-
 #include <zencore/compactbinary.h>
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/compactbinarypackage.h>
@@ -11,6 +9,7 @@
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
 #include <zencore/string.h>
+#include <zenhttp/httpshared.h>
 
 #if ZEN_WITH_HTTPSYS
 
diff --git a/zenhttp/include/zenhttp/httpshared.h b/zenhttp/include/zenhttp/httpshared.h
new file mode 100644
index 000000000..06dc4a872
--- /dev/null
+++ b/zenhttp/include/zenhttp/httpshared.h
@@ -0,0 +1,51 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+
+#include <functional>
+
+namespace zen {
+
+class IoBuffer;
+class CbPackage;
+class CompositeBuffer;
+
+struct CbPackageHeader
+{
+	uint32_t HeaderMagic;
+	uint32_t AttachmentCount;
+	uint32_t Reserved1;
+	uint32_t Reserved2;
+};
+
+static_assert(sizeof(CbPackageHeader) == 16);
+
+static constinit uint32_t kCbPkgMagic = 0xaa77aacc;
+
+struct CbAttachmentEntry
+{
+	uint64_t AttachmentSize;
+	uint32_t Flags;
+	IoHash	 AttachmentHash;
+
+	enum
+	{
+		kIsCompressed = (1u << 0),		// Is marshaled using compressed buffer storage format
+		kIsObject	  = (1u << 1),		// Is compact binary object
+	};
+};
+
+static_assert(sizeof(CbAttachmentEntry) == 32);
+
+std::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data);
+CompositeBuffer		  FormatPackageMessageBuffer(const CbPackage& Data);
+CbPackage			  ParsePackageMessage(
+				IoBuffer												  Payload,
+				std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> CreateBuffer = [](const IoHash&, uint64_t Size) -> IoBuffer {
+		return IoBuffer{Size};
+				});
+
+}  // namespace zen
diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp
index e68161ccf..8baf57d7e 100644
--- a/zenserver-test/zenserver-test.cpp
+++ b/zenserver-test/zenserver-test.cpp
@@ -14,8 +14,9 @@
 #include <zencore/string.h>
 #include <zencore/thread.h>
 #include <zencore/timer.h>
-#include <zenhttp/zenhttp.h>
 #include <zenhttp/httpclient.h>
+#include <zenhttp/httpshared.h>
+#include <zenhttp/zenhttp.h>
 #include <zenserverprocess.h>
 
 #include <mimalloc.h>
@@ -1484,6 +1485,10 @@ TEST_CASE("http.package")
 
 	zen::HttpClient			  TestClient(BaseUri);
 	zen::HttpClient::Response Response = TestClient.TransactPackage("/testing/package"sv, TestPackage);
+
+	zen::CbPackage ResponsePackage = ParsePackageMessage(Response.ResponsePayload);
+
+	CHECK_EQ(ResponsePackage, TestPackage);
 }
 
 #endif