cbpackage: added initial support for marshaling of attachment by local reference

this mode allows local clients to avoid unnecessary copying of data from zen and instead reference data directly

1 files changed, 54 insertions, 8 deletions

diff --git a/zenhttp/include/zenhttp/httpshared.h b/zenhttp/include/zenhttp/httpshared.h
index a6a61485f..24ce0c85a 100644
--- a/zenhttp/include/zenhttp/httpshared.h
+++ b/zenhttp/include/zenhttp/httpshared.h
@@ -2,10 +2,12 @@
 
 #pragma once
 
+#include <zencore/compactbinarypackage.h>
 #include <zencore/iobuffer.h>
 #include <zencore/iohash.h>
 
 #include <functional>
+#include <gsl/gsl-lite.hpp>
 
 namespace zen {
 
@@ -24,6 +26,13 @@ class CompositeBuffer;
 
 	Structures and code related to handling CbPackage transactions
 
+	CbPackage instances are marshaled across the wire using a distinct message
+	format. We don't use the CbPackage serialization format provided by the
+	CbPackage implementation itself since that does not provide much flexibility
+	in how the attachment payloads are transmitted. The scheme below separates
+	metadata cleanly from payloads and this enables us to more efficiently
+	transmit them either via sendfile/TransmitFile like mechanisms, or by
+	reference/memory mapping in the local case.
  */
 
 struct CbPackageHeader
@@ -43,33 +52,59 @@ enum : uint32_t
 
 struct CbAttachmentEntry
 {
-	uint64_t PayloadSize;
-	uint32_t Flags;
-	IoHash	 AttachmentHash;
+	uint64_t PayloadSize;	  // Size of the associated payload data in the message
+	uint32_t Flags;			  // See flags below
+	IoHash	 AttachmentHash;  // Content Id for the attachment
 
 	enum
 	{
 		kIsCompressed = (1u << 0),	// Is marshaled using compressed buffer storage format
 		kIsObject	  = (1u << 1),	// Is compact binary object
 		kIsError	  = (1u << 2),	// Is error (compact binary formatted) object
+		kIsLocalRef	  = (1u << 3),	// Is "local reference"
 	};
 };
 
+struct CbAttachmentReferenceHeader
+{
+	uint64_t PayloadByteOffset	= 0;
+	uint64_t PayloadByteSize	= ~0u;
+	uint16_t AbsolutePathLength = 0;
+
+	// This header will be followed by UTF8 encoded absolute path to backing file
+};
+
 static_assert(sizeof(CbAttachmentEntry) == 32);
 
-std::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data);
-CompositeBuffer		  FormatPackageMessageBuffer(const CbPackage& Data);
+enum class FormatFlags
+{
+	kDefault			  = 0,
+	kAllowLocalReferences = (1u << 0)
+};
+
+gsl_DEFINE_ENUM_BITMASK_OPERATORS(FormatFlags);
+
+std::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data, FormatFlags Flags);
+CompositeBuffer		  FormatPackageMessageBuffer(const CbPackage& Data, FormatFlags Flags);
 CbPackage			  ParsePackageMessage(
 				IoBuffer												  Payload,
 				std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> CreateBuffer = [](const IoHash&, uint64_t Size) -> IoBuffer {
 		return IoBuffer{Size};
 				});
+bool IsPackageMessage(IoBuffer Payload);
+
+std::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data);
+CompositeBuffer		  FormatPackageMessageBuffer(const CbPackage& Data);
 
 /** Streaming reader for compact binary packages
 
 	The goal is to ultimately support zero-copy I/O, but for now there'll be some
 	copying involved on some platforms at least.
 
+	This approach to deserializing CbPackage data is more efficient than
+	`ParsePackageMessage` since it does not require the entire message to
+	be resident in a memory buffer
+
   */
 class CbPackageReader
 {
@@ -79,11 +114,18 @@ public:
 
 	void SetPayloadBufferCreator(std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> CreateBuffer);
 
-	/** Process header data
+	/** Process compact binary package data stream
+
+		The data stream must be in the serialization format produced by FormatPackageMessage
+
+		\return How many bytes must be fed to this function in the next call
 	 */
-	uint64_t ProcessHeaderData(const void* Data, uint64_t DataBytes);
+	uint64_t ProcessPackageHeaderData(const void* Data, uint64_t DataBytes);
 
-	std::span<IoBuffer> GetPayloadBuffers() { return m_PayloadBuffers; }
+	void							 Finalize();
+	const std::vector<CbAttachment>& GetAttachments() { return m_Attachments; }
+	CbObject						 GetRootObject() { return m_RootObject; }
+	std::span<IoBuffer>				 GetPayloadBuffers() { return m_PayloadBuffers; }
 
 private:
 	enum class State
@@ -97,7 +139,11 @@ private:
 	std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> m_CreateBuffer;
 	std::vector<IoBuffer>									  m_PayloadBuffers;
 	std::vector<CbAttachmentEntry>							  m_AttachmentEntries;
+	std::vector<CbAttachment>								  m_Attachments;
+	CbObject												  m_RootObject;
 	CbPackageHeader											  m_PackageHeader;
+
+	IoBuffer MarshalLocalChunkReference(IoBuffer AttachmentBuffer);
 };
 
 void forcelink_httpshared();