Merge branch 'main' of https://github.com/EpicGames/zen

24 files changed, 704 insertions, 187 deletions

diff --git a/API.md b/API.md
new file mode 100644
index 000000000..33ca713c8
--- /dev/null
+++ b/API.md
@@ -0,0 +1,6 @@
+# API status
+
+Zen exposes a REST API which is currently intended to be used from Unreal Engine 
+only. It is an internal implementation detail and is subject to change especially 
+during the beta phase. Thus it is not recommended that you interface with this API 
+directly.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ea472ddde..2526ac4d1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
 ##
+-- Enable adding namespace to URI based upstream requests
 -- Add logging of namespace name and bucket name if we get invalid names in requests
 -- Updated README.md with Linux dev prerequisites
 -- asio: added some logging to indicate concurrency
diff --git a/CODING.md b/CODING.md
index 54774590f..ddb46139a 100644
--- a/CODING.md
+++ b/CODING.md
@@ -5,6 +5,9 @@ The naming conventions for Zen are intended to resemble the Unreal Engine coding
 * Classes/Structs - `PascalCase`
 * Functions - `PascalCase()`
 * Class member variables - `m_PascalCase`
+* Global variables - `g_PascalCase`
+* Static variables - `s_PascalCase`
+* Thread local variables - `t_PascalCase`
 
 Those who are familiar with the UE coding standards will note that we do not require or encourage `F` prefixes on struct or classes, and we expect class members to have a `m_` member prefix.
 
diff --git a/xmake.lua b/xmake.lua
index c492ca181..de63f205b 100644
--- a/xmake.lua
+++ b/xmake.lua
@@ -108,7 +108,9 @@ end
 
 if is_os("windows") then
     option("vfs")
-        set_showmenu(true)
+		-- note: this is an old prototype and is not functional at all at the moment
+		set_default(false)
+        set_showmenu(false)
         set_description("Enable VFS functionality")
     option_end()
     add_define_by_config("ZEN_WITH_VFS", "vfs")
@@ -121,6 +123,13 @@ if is_os("windows") then
     add_define_by_config("ZEN_WITH_HTTPSYS", "httpsys")
 end
 
+option("catch2")
+    set_default(false)
+    set_showmenu(true)
+    set_description("Use catch2 to run tests")
+option_end()
+add_define_by_config("ZEN_USE_CATCH2", "catch2")
+
 option("compute")
     set_default(true)
     set_showmenu(true)
@@ -136,6 +145,7 @@ option_end()
 add_define_by_config("ZEN_WITH_EXEC_SERVICES", "exec")
 
 option("zenmesh")
+	-- note: this is an old prototype and is not functional at all at the moment
     set_default(false)
     set_showmenu(true)
     set_description("Enables Zen's mesh feature")
diff --git a/zen/cmds/print.cpp b/zen/cmds/print.cpp
index f66f433f1..a8b2215a2 100644
--- a/zen/cmds/print.cpp
+++ b/zen/cmds/print.cpp
@@ -5,13 +5,31 @@
 #include <zencore/compactbinarypackage.h>
 #include <zencore/compactbinaryvalidation.h>
 #include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/string.h>
+#include <zenhttp/httpshared.h>
 
 using namespace std::literals;
 
 namespace zen {
 
+static void
+PrintCbObject(CbObject Object)
+{
+	zen::StringBuilder<1024> ObjStr;
+	zen::CompactBinaryToJson(Object, ObjStr);
+	zen::ConsoleLog().info("{}", ObjStr);
+}
+
+static void
+PrintCbObject(IoBuffer Data)
+{
+	zen::CbObject Object{SharedBuffer(Data)};
+
+	PrintCbObject(Object);
+}
+
 PrintCommand::PrintCommand()
 {
 	m_Options.add_options()("h,help", "Print help");
@@ -41,18 +59,90 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	if (m_Filename.empty())
 		throw std::runtime_error("No file specified");
 
-	zen::FileContents Fc = zen::ReadFile(m_Filename);
+	zen::FileContents Fc;
+
+	if (m_Filename == "-")
+	{
+		Fc = zen::ReadStdIn();
+	}
+	else
+	{
+		Fc = zen::ReadFile(m_Filename);
+	}
 
 	if (Fc.ErrorCode)
 	{
-		zen::ConsoleLog().error("Failed to open file '{}': {}", m_Filename, Fc.ErrorCode.message());
+		zen::ConsoleLog().error("Failed to read file '{}': {}", m_Filename, Fc.ErrorCode.message());
 
 		return 1;
 	}
 
 	IoBuffer Data = Fc.Flatten();
 
-	if (CbValidateError Result = ValidateCompactBinary(Data, CbValidateMode::All); Result != CbValidateError::None)
+	if (CompressedBuffer Compressed{CompressedBuffer::FromCompressed(SharedBuffer(Data))})
+	{
+		zen::ConsoleLog().info("Compressed binary: size {}, raw size {}, hash: {}",
+							   Compressed.GetCompressedSize(),
+							   Compressed.GetRawSize(),
+							   IoHash::FromBLAKE3(Compressed.GetRawHash()));
+	}
+	else if (IsPackageMessage(Data))
+	{
+		CbPackage Package = ParsePackageMessage(Data);
+
+		CbObject					  Object	  = Package.GetObject();
+		std::span<const CbAttachment> Attachments = Package.GetAttachments();
+
+		zen::ConsoleLog().info("Package - {} attachments, object hash {}", Package.GetAttachments().size(), Package.GetObjectHash());
+		zen::ConsoleLog().info("");
+
+		int AttachmentIndex = 1;
+
+		for (const CbAttachment& Attachment : Attachments)
+		{
+			std::string AttachmentSize = "n/a";
+			const char* AttachmentType = "unknown";
+
+			if (Attachment.IsCompressedBinary())
+			{
+				AttachmentType = "Compressed";
+				AttachmentSize = fmt::format("{} ({} uncompressed)",
+											 Attachment.AsCompressedBinary().GetCompressedSize(),
+											 Attachment.AsCompressedBinary().GetRawSize());
+			}
+			else if (Attachment.IsBinary())
+			{
+				AttachmentType = "Binary";
+				AttachmentSize = fmt::format("{}", Attachment.AsBinary().GetSize());
+			}
+			else if (Attachment.IsObject())
+			{
+				AttachmentType = "Object";
+				AttachmentSize = fmt::format("{}", Attachment.AsObject().GetSize());
+			}
+			else if (Attachment.IsNull())
+			{
+				AttachmentType = "null";
+			}
+
+			zen::ConsoleLog().info("Attachment #{} : {}, {}, size {}",
+								   AttachmentIndex,
+								   Attachment.GetHash(),
+								   AttachmentType,
+								   AttachmentSize);
+
+			++AttachmentIndex;
+		}
+
+		zen::ConsoleLog().info("---8<---");
+
+		PrintCbObject(Object);
+	}
+	else if (CbValidateError Result = ValidateCompactBinary(Data, CbValidateMode::All); Result == CbValidateError::None)
+	{
+		PrintCbObject(Data);
+	}
+	else
 	{
 		zen::ConsoleLog().error("Data in file '{}' does not appear to be compact binary (validation error {:#x})",
 								m_Filename,
@@ -61,12 +151,6 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		return 1;
 	}
 
-	zen::CbObject Object{SharedBuffer(Data)};
-
-	zen::StringBuilder<1024> ObjStr;
-	zen::CompactBinaryToJson(Object, ObjStr);
-	zen::ConsoleLog().info("{}", ObjStr);
-
 	return 0;
 }
 
diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
index bd85f5a11..01997daae 100644
--- a/zencore/filesystem.cpp
+++ b/zencore/filesystem.cpp
@@ -6,6 +6,7 @@
 #include <zencore/fmtutils.h>
 #include <zencore/iobuffer.h>
 #include <zencore/logging.h>
+#include <zencore/stream.h>
 #include <zencore/string.h>
 #include <zencore/testing.h>
 
@@ -637,6 +638,25 @@ FileContents::Flatten()
 }
 
 FileContents
+ReadStdIn()
+{
+	BinaryWriter Writer;
+
+	do
+	{
+		uint8_t ReadBuffer[1024];
+
+		size_t BytesRead = fread(ReadBuffer, 1, sizeof ReadBuffer, stdin);
+		Writer.Write(ReadBuffer, BytesRead);
+	} while (!feof(stdin));
+
+	FileContents Contents;
+	Contents.Data.emplace_back(IoBuffer(IoBuffer::Clone, Writer.GetData(), Writer.GetSize()));
+
+	return Contents;
+}
+
+FileContents
 ReadFile(std::filesystem::path Path)
 {
 	uint64_t FileSizeBytes;
diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h
index 6d07a79b4..f49135687 100644
--- a/zencore/include/zencore/filesystem.h
+++ b/zencore/include/zencore/filesystem.h
@@ -46,6 +46,7 @@ struct FileContents
 	IoBuffer Flatten();
 };
 
+ZENCORE_API FileContents ReadStdIn();
 ZENCORE_API FileContents ReadFile(std::filesystem::path Path);
 ZENCORE_API bool ScanFile(std::filesystem::path Path, uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc);
 ZENCORE_API void WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount);
diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
index bf658922d..b38201ba3 100644
--- a/zencore/include/zencore/iobuffer.h
+++ b/zencore/include/zencore/iobuffer.h
@@ -390,6 +390,12 @@ public:
 		return reinterpret_cast<const T*>(m_Core->DataPointer());
 	}
 
+	template<typename T>
+	[[nodiscard]] T* MutableData() const
+	{
+		return reinterpret_cast<T*>(m_Core->MutableDataPointer());
+	}
+
 private:
 	RefPtr<IoBufferCore> m_Core = new IoBufferCore;
 
diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h
index 92f567dae..7ea8c029f 100644
--- a/zencore/include/zencore/string.h
+++ b/zencore/include/zencore/string.h
@@ -933,7 +933,7 @@ public:
 		return *Skip(Str, Set) == '\0';
 	}
 
-	////////// Algorithms for string types like FStringView and FString //////////
+	////////// Algorithms for string types like std::string_view and std::string //////////
 
 	/** Get initial substring with all characters in set */
 	template<class StringType>
diff --git a/zenhttp/httpasio.cpp b/zenhttp/httpasio.cpp
index 81ceef888..1ca5fb1c1 100644
--- a/zenhttp/httpasio.cpp
+++ b/zenhttp/httpasio.cpp
@@ -1059,7 +1059,7 @@ HttpAsioServerRequest::HttpAsioServerRequest(asio_http::HttpRequest& Request, Ht
 
 			if (AcceptContentType != HttpContentType::kUnknownContentType)
 			{
-				m_Uri.remove_suffix(uint32_t(m_Uri.size() - LastComponentIndex - LastDotIndex - 1));
+				m_Uri.remove_suffix(uint32_t(UriSuffix8.size() + 1));
 			}
 		}
 	}
diff --git a/zenhttp/httpserver.cpp b/zenhttp/httpserver.cpp
index df5a0596a..840b90931 100644
--- a/zenhttp/httpserver.cpp
+++ b/zenhttp/httpserver.cpp
@@ -76,22 +76,32 @@ MapContentTypeToString(HttpContentType ContentType)
 }
 
 //////////////////////////////////////////////////////////////////////////
+//
+// Note that in addition to MIME types we accept abbreviated versions, for
+// use in suffix parsing as well as for convenience when using curl
 
 static constinit uint32_t HashBinary					= HashStringDjb2("application/octet-stream"sv);
+static constinit uint32_t HashJson						= HashStringDjb2("json"sv);
 static constinit uint32_t HashApplicationJson			= HashStringDjb2("application/json"sv);
-static constinit uint32_t HashApplicationYaml			= HashStringDjb2("text/yaml"sv);
+static constinit uint32_t HashYaml						= HashStringDjb2("yaml"sv);
+static constinit uint32_t HashTextYaml					= HashStringDjb2("text/yaml"sv);
 static constinit uint32_t HashText						= HashStringDjb2("text/plain"sv);
-static constinit uint32_t HashCompactBinary				= HashStringDjb2("application/x-ue-cb"sv);
+static constinit uint32_t HashApplicationCompactBinary	= HashStringDjb2("application/x-ue-cb"sv);
+static constinit uint32_t HashCompactBinary				= HashStringDjb2("ucb"sv);
 static constinit uint32_t HashCompactBinaryPackage		= HashStringDjb2("application/x-ue-cbpkg"sv);
+static constinit uint32_t HashCompactBinaryPackageShort = HashStringDjb2("cbpkg"sv);
 static constinit uint32_t HashCompactBinaryPackageOffer = HashStringDjb2("application/x-ue-offer"sv);
 static constinit uint32_t HashCompressedBinary			= HashStringDjb2("application/x-ue-comp"sv);
-static constinit uint32_t HashJson						= HashStringDjb2("json"sv);
-static constinit uint32_t HashYaml						= HashStringDjb2("yaml"sv);
-static constinit uint32_t HashHtml						= HashStringDjb2("text/html"sv);
-static constinit uint32_t HashJavaScript				= HashStringDjb2("application/javascript"sv);
-static constinit uint32_t HashCss						= HashStringDjb2("text/css"sv);
-static constinit uint32_t HashPng						= HashStringDjb2("image/png"sv);
-static constinit uint32_t HashIcon						= HashStringDjb2("image/x-icon"sv);
+static constinit uint32_t HashHtml						= HashStringDjb2("html"sv);
+static constinit uint32_t HashTextHtml					= HashStringDjb2("text/html"sv);
+static constinit uint32_t HashJavaScript				= HashStringDjb2("js"sv);
+static constinit uint32_t HashApplicationJavaScript		= HashStringDjb2("application/javascript"sv);
+static constinit uint32_t HashCss						= HashStringDjb2("css"sv);
+static constinit uint32_t HashTextCss					= HashStringDjb2("text/css"sv);
+static constinit uint32_t HashPng						= HashStringDjb2("png"sv);
+static constinit uint32_t HashImagePng					= HashStringDjb2("image/png"sv);
+static constinit uint32_t HashIcon						= HashStringDjb2("ico"sv);
+static constinit uint32_t HashImageIcon					= HashStringDjb2("image/x-icon"sv);
 
 std::once_flag InitContentTypeLookup;
 
@@ -102,20 +112,27 @@ struct HashedTypeEntry
 } TypeHashTable[] = {
 	// clang-format off
 	{HashBinary,					HttpContentType::kBinary},
+	{HashApplicationCompactBinary,	HttpContentType::kCbObject},
 	{HashCompactBinary,				HttpContentType::kCbObject},
 	{HashCompactBinaryPackage,		HttpContentType::kCbPackage},
+	{HashCompactBinaryPackageShort,	HttpContentType::kCbPackage},
 	{HashCompactBinaryPackageOffer, HttpContentType::kCbPackageOffer},
 	{HashJson,						HttpContentType::kJSON},
 	{HashApplicationJson,			HttpContentType::kJSON},
 	{HashYaml,						HttpContentType::kYAML},
-	{HashApplicationYaml,			HttpContentType::kYAML},
+	{HashTextYaml,					HttpContentType::kYAML},
 	{HashText,						HttpContentType::kText},
 	{HashCompressedBinary,			HttpContentType::kCompressedBinary},
 	{HashHtml,						HttpContentType::kHTML},
+	{HashTextHtml,					HttpContentType::kHTML},
 	{HashJavaScript,				HttpContentType::kJavaScript},
+	{HashApplicationJavaScript,		HttpContentType::kJavaScript},
 	{HashCss,						HttpContentType::kCSS},
+	{HashTextCss,					HttpContentType::kCSS},
 	{HashPng,						HttpContentType::kPNG},
+	{HashImagePng,					HttpContentType::kPNG},
 	{HashIcon,						HttpContentType::kIcon},
+	{HashImageIcon,					HttpContentType::kIcon},
 	// clang-format on
 };
 
diff --git a/zenhttp/httpshared.cpp b/zenhttp/httpshared.cpp
index f2ce17e16..769b5e3f3 100644
--- a/zenhttp/httpshared.cpp
+++ b/zenhttp/httpshared.cpp
@@ -5,20 +5,35 @@
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/compactbinarypackage.h>
 #include <zencore/compositebuffer.h>
+#include <zencore/filesystem.h>
 #include <zencore/iobuffer.h>
 #include <zencore/iohash.h>
 #include <zencore/stream.h>
 #include <zencore/testing.h>
+#include <zencore/testutils.h>
+
+#include <zencore/fmtutils.h>
 
 #include <span>
 #include <vector>
 
 namespace zen {
 
+std::vector<IoBuffer>
+FormatPackageMessage(const CbPackage& Data)
+{
+	return FormatPackageMessage(Data, FormatFlags::kDefault);
+}
 CompositeBuffer
 FormatPackageMessageBuffer(const CbPackage& Data)
 {
-	std::vector<IoBuffer> Message = FormatPackageMessage(Data);
+	return FormatPackageMessageBuffer(Data, FormatFlags::kDefault);
+}
+
+CompositeBuffer
+FormatPackageMessageBuffer(const CbPackage& Data, FormatFlags Flags)
+{
+	std::vector<IoBuffer> Message = FormatPackageMessage(Data, Flags);
 
 	std::vector<SharedBuffer> Buffers;
 
@@ -31,11 +46,11 @@ FormatPackageMessageBuffer(const CbPackage& Data)
 }
 
 std::vector<IoBuffer>
-FormatPackageMessage(const CbPackage& Data)
+FormatPackageMessage(const CbPackage& Data, FormatFlags Flags)
 {
 	const std::span<const CbAttachment>& Attachments = Data.GetAttachments();
+	std::vector<IoBuffer>				 ResponseBuffers;
 
-	std::vector<IoBuffer> ResponseBuffers;
 	ResponseBuffers.reserve(3 + Attachments.size());  // TODO: may want to use an additional fudge factor here to avoid growing since each
 													  // attachment is likely to consist of several buffers
 
@@ -47,9 +62,8 @@ FormatPackageMessage(const CbPackage& Data)
 
 	// Attachment metadata array
 
-	IoBuffer AttachmentMetadataBuffer = IoBuffer{sizeof(CbAttachmentEntry) * (Attachments.size() + /* root */ 1)};
-
-	CbAttachmentEntry* AttachmentInfo = reinterpret_cast<CbAttachmentEntry*>(AttachmentMetadataBuffer.MutableData());
+	IoBuffer		   AttachmentMetadataBuffer = IoBuffer{sizeof(CbAttachmentEntry) * (Attachments.size() + /* root */ 1)};
+	CbAttachmentEntry* AttachmentInfo			= reinterpret_cast<CbAttachmentEntry*>(AttachmentMetadataBuffer.MutableData());
 
 	ResponseBuffers.push_back(AttachmentMetadataBuffer);  // Attachment metadata
 
@@ -62,6 +76,44 @@ FormatPackageMessage(const CbPackage& Data)
 
 	// Attachment payloads
 
+	auto MarshalLocal = [&AttachmentInfo, &ResponseBuffers](const std::string&			 Path8,
+															CbAttachmentReferenceHeader& LocalRef,
+															const IoHash&				 AttachmentHash,
+															bool						 IsCompressed) {
+		IoBuffer RefBuffer(sizeof(CbAttachmentReferenceHeader) + Path8.size());
+
+		CbAttachmentReferenceHeader* RefHdr = RefBuffer.MutableData<CbAttachmentReferenceHeader>();
+		*RefHdr++							= LocalRef;
+		memcpy(RefHdr, Path8.data(), Path8.size());
+
+		*AttachmentInfo++ = {.PayloadSize = RefBuffer.GetSize(),
+							 .Flags = (IsCompressed ? uint32_t(CbAttachmentEntry::kIsCompressed) : 0u) | CbAttachmentEntry::kIsLocalRef,
+							 .AttachmentHash = AttachmentHash};
+
+		ResponseBuffers.push_back(std::move(RefBuffer));
+	};
+
+	auto IsLocalRef = [](const CompositeBuffer& AttachmentBinary, CbAttachmentReferenceHeader& LocalRef, std::string& Path8) -> bool {
+		const SharedBuffer&	  Segment = AttachmentBinary.GetSegments().front();
+		IoBufferFileReference Ref;
+		const IoBuffer&		  SegmentBuffer = Segment.AsIoBuffer();
+
+		if (!SegmentBuffer.GetFileReference(Ref))
+		{
+			return false;
+		}
+
+		ExtendablePathBuilder<256> LocalRefFile;
+		LocalRefFile.Append(std::filesystem::absolute(PathFromHandle(Ref.FileHandle)));
+		Path8 = LocalRefFile.ToUtf8();
+
+		LocalRef.AbsolutePathLength = gsl::narrow<uint16_t>(Path8.size());
+		LocalRef.PayloadByteOffset	= Ref.FileChunkOffset;
+		LocalRef.PayloadByteSize	= Ref.FileChunkSize;
+
+		return true;
+	};
+
 	for (const CbAttachment& Attachment : Attachments)
 	{
 		if (Attachment.IsNull())
@@ -70,15 +122,39 @@ FormatPackageMessage(const CbPackage& Data)
 		}
 		else if (CompressedBuffer AttachmentBuffer = Attachment.AsCompressedBinary())
 		{
-			CompositeBuffer Compressed = AttachmentBuffer.GetCompressed();
+			CompositeBuffer Compressed	   = AttachmentBuffer.GetCompressed();
+			IoHash			AttachmentHash = IoHash::FromBLAKE3(AttachmentBuffer.GetRawHash());
+
+			// If the data is either not backed by a file, or there are multiple
+			// fragments then we cannot marshal it by local reference. We might
+			// want/need to extend this in the future to allow multiple chunk
+			// segments to be marshaled at once
 
-			*AttachmentInfo++ = {.PayloadSize	 = AttachmentBuffer.GetCompressedSize(),
-								 .Flags			 = CbAttachmentEntry::kIsCompressed,
-								 .AttachmentHash = IoHash::FromBLAKE3(AttachmentBuffer.GetRawHash())};
+			bool MarshalByLocalRef = EnumHasAllFlags(Flags, FormatFlags::kAllowLocalReferences) && (Compressed.GetSegments().size() == 1);
 
-			for (const SharedBuffer& Segment : Compressed.GetSegments())
+			CbAttachmentReferenceHeader LocalRef;
+			std::string					Path8;
+
+			if (MarshalByLocalRef)
 			{
-				ResponseBuffers.push_back(Segment.AsIoBuffer());
+				MarshalByLocalRef = IsLocalRef(Compressed, LocalRef, Path8);
+			}
+
+			if (MarshalByLocalRef)
+			{
+				const bool IsCompressed = true;
+				MarshalLocal(Path8, LocalRef, AttachmentHash, IsCompressed);
+			}
+			else
+			{
+				*AttachmentInfo++ = {.PayloadSize	 = AttachmentBuffer.GetCompressedSize(),
+									 .Flags			 = CbAttachmentEntry::kIsCompressed,
+									 .AttachmentHash = IoHash::FromBLAKE3(AttachmentBuffer.GetRawHash())};
+
+				for (const SharedBuffer& Segment : Compressed.GetSegments())
+				{
+					ResponseBuffers.push_back(Segment.AsIoBuffer());
+				}
 			}
 		}
 		else if (CbObject AttachmentObject = Attachment.AsObject())
@@ -92,11 +168,31 @@ FormatPackageMessage(const CbPackage& Data)
 		}
 		else if (CompositeBuffer AttachmentBinary = Attachment.AsCompositeBinary())
 		{
-			*AttachmentInfo++ = {.PayloadSize = AttachmentBinary.GetSize(), .Flags = 0, .AttachmentHash = Attachment.GetHash()};
+			IoHash AttachmentHash = IoHash::FromBLAKE3(AttachmentBuffer.GetRawHash());
+			bool   MarshalByLocalRef =
+				EnumHasAllFlags(Flags, FormatFlags::kAllowLocalReferences) && (AttachmentBinary.GetSegments().size() == 1);
+
+			CbAttachmentReferenceHeader LocalRef;
+			std::string					Path8;
+
+			if (MarshalByLocalRef)
+			{
+				MarshalByLocalRef = IsLocalRef(AttachmentBinary, LocalRef, Path8);
+			}
 
-			for (const SharedBuffer& Segment : AttachmentBinary.GetSegments())
+			if (MarshalByLocalRef)
 			{
-				ResponseBuffers.push_back(Segment.AsIoBuffer());
+				const bool IsCompressed = false;
+				MarshalLocal(Path8, LocalRef, AttachmentHash, IsCompressed);
+			}
+			else
+			{
+				*AttachmentInfo++ = {.PayloadSize = AttachmentBinary.GetSize(), .Flags = 0, .AttachmentHash = Attachment.GetHash()};
+
+				for (const SharedBuffer& Segment : AttachmentBinary.GetSegments())
+				{
+					ResponseBuffers.push_back(Segment.AsIoBuffer());
+				}
 			}
 		}
 		else
@@ -108,6 +204,27 @@ FormatPackageMessage(const CbPackage& Data)
 	return ResponseBuffers;
 }
 
+bool
+IsPackageMessage(IoBuffer Payload)
+{
+	if (!Payload)
+	{
+		return false;
+	}
+
+	BinaryReader Reader(Payload);
+
+	CbPackageHeader Hdr;
+	Reader.Read(&Hdr, sizeof Hdr);
+
+	if (Hdr.HeaderMagic != kCbPkgMagic)
+	{
+		return false;
+	}
+
+	return true;
+}
+
 CbPackage
 ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint64_t)> CreateBuffer)
 {
@@ -145,7 +262,38 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
 
 		Reader.Read(AttachmentBuffer.MutableData(), AttachmentSize);
 
-		if (Entry.Flags & CbAttachmentEntry::kIsCompressed)
+		if (Entry.Flags & CbAttachmentEntry::kIsLocalRef)
+		{
+			// Marshal local reference - a "pointer" to the chunk backing file
+
+			ZEN_ASSERT(AttachmentBuffer.Size() >= sizeof(CbAttachmentReferenceHeader));
+
+			const CbAttachmentReferenceHeader* AttachRefHdr = AttachmentBuffer.Data<CbAttachmentReferenceHeader>();
+			const char8_t*					   PathPointer	= reinterpret_cast<const char8_t*>(AttachRefHdr + 1);
+
+			ZEN_ASSERT(AttachmentBuffer.Size() >= (sizeof(CbAttachmentReferenceHeader) + AttachRefHdr->AbsolutePathLength));
+
+			std::filesystem::path Path{PathPointer};
+
+			if (IoBuffer ChunkReference =
+					IoBufferBuilder::MakeFromFile(Path, AttachRefHdr->PayloadByteOffset, AttachRefHdr->PayloadByteSize))
+			{
+				CompressedBuffer CompBuf(CompressedBuffer::FromCompressed(SharedBuffer(ChunkReference)));
+				CbAttachment	 Attachment(std::move(CompBuf));
+				Package.AddAttachment(Attachment);
+			}
+			else
+			{
+				// Unable to open chunk reference
+
+				throw std::runtime_error(fmt::format("unable to resolve chunk #{} at '{}' (offset {}, size {})",
+													 i,
+													 PathToUtf8(Path),
+													 AttachRefHdr->PayloadByteOffset,
+													 AttachRefHdr->PayloadByteSize));
+			}
+		}
+		else if (Entry.Flags & CbAttachmentEntry::kIsCompressed)
 		{
 			CompressedBuffer CompBuf(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer)));
 
@@ -153,6 +301,7 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
 			{
 				if (i == 0)
 				{
+					// First payload is always a compact binary object
 					Package.SetObject(LoadCompactBinaryObject(std::move(CompBuf)));
 				}
 				else
@@ -204,10 +353,8 @@ CbPackageReader::SetPayloadBufferCreator(std::function<IoBuffer(const IoHash& Ci
 	m_CreateBuffer = CreateBuffer;
 }
 
-/** Process data
- */
 uint64_t
-CbPackageReader::ProcessHeaderData(const void* Data, uint64_t DataBytes)
+CbPackageReader::ProcessPackageHeaderData(const void* Data, uint64_t DataBytes)
 {
 	ZEN_ASSERT(m_CurrentState != State::kReadingBuffers);
 
@@ -232,6 +379,10 @@ CbPackageReader::ProcessHeaderData(const void* Data, uint64_t DataBytes)
 
 			for (CbAttachmentEntry& Entry : m_AttachmentEntries)
 			{
+				// This preallocates memory for payloads but note that for the local references
+				// the caller will need to handle the payload differently (i.e it's a
+				// CbAttachmentReferenceHeader not the actual payload)
+
 				m_PayloadBuffers.push_back(IoBuffer{Entry.PayloadSize});
 			}
 
@@ -244,6 +395,94 @@ CbPackageReader::ProcessHeaderData(const void* Data, uint64_t DataBytes)
 	}
 }
 
+IoBuffer
+CbPackageReader::MarshalLocalChunkReference(IoBuffer AttachmentBuffer)
+{
+	// Marshal local reference - a "pointer" to the chunk backing file
+
+	ZEN_ASSERT(AttachmentBuffer.Size() >= sizeof(CbAttachmentReferenceHeader));
+
+	const CbAttachmentReferenceHeader* AttachRefHdr = AttachmentBuffer.Data<CbAttachmentReferenceHeader>();
+	const char8_t*					   PathPointer	= reinterpret_cast<const char8_t*>(AttachRefHdr + 1);
+
+	ZEN_ASSERT(AttachmentBuffer.Size() >= (sizeof(CbAttachmentReferenceHeader) + AttachRefHdr->AbsolutePathLength));
+
+	std::u8string_view PathView{PathPointer, AttachRefHdr->AbsolutePathLength};
+
+	std::filesystem::path Path{PathView};
+
+	IoBuffer ChunkReference = IoBufferBuilder::MakeFromFile(Path, AttachRefHdr->PayloadByteOffset, AttachRefHdr->PayloadByteSize);
+
+	if (!ChunkReference)
+	{
+		// Unable to open chunk reference
+
+		throw std::runtime_error(fmt::format("unable to resolve local reference to '{}' (offset {}, size {})",
+											 PathToUtf8(Path),
+											 AttachRefHdr->PayloadByteOffset,
+											 AttachRefHdr->PayloadByteSize));
+	}
+
+	return ChunkReference;
+};
+
+void
+CbPackageReader::Finalize()
+{
+	if (m_AttachmentEntries.empty())
+	{
+		return;
+	}
+
+	m_Attachments.reserve(m_AttachmentEntries.size() - 1);
+
+	int CurrentAttachmentIndex = 0;
+	for (CbAttachmentEntry& Entry : m_AttachmentEntries)
+	{
+		IoBuffer AttachmentBuffer = m_PayloadBuffers[CurrentAttachmentIndex];
+
+		if (CurrentAttachmentIndex == 0)
+		{
+			// Root object
+			if (Entry.Flags & CbAttachmentEntry::kIsObject)
+			{
+				if (Entry.Flags & CbAttachmentEntry::kIsLocalRef)
+				{
+					m_RootObject = LoadCompactBinaryObject(MarshalLocalChunkReference(AttachmentBuffer));
+				}
+				else if (Entry.Flags & CbAttachmentEntry::kIsCompressed)
+				{
+					m_RootObject = LoadCompactBinaryObject(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer)));
+				}
+				else
+				{
+					m_RootObject = LoadCompactBinaryObject(std::move(AttachmentBuffer));
+				}
+			}
+			else
+			{
+				throw std::runtime_error("missing or invalid root object");
+			}
+		}
+		else if (Entry.Flags & CbAttachmentEntry::kIsLocalRef)
+		{
+			IoBuffer ChunkReference = MarshalLocalChunkReference(AttachmentBuffer);
+
+			if (Entry.Flags & CbAttachmentEntry::kIsCompressed)
+			{
+				m_Attachments.push_back(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(ChunkReference))));
+			}
+			else
+			{
+				m_Attachments.push_back(CbAttachment(
+					CompressedBuffer::Compress(SharedBuffer(ChunkReference), OodleCompressor::NotSet, OodleCompressionLevel::None)));
+			}
+		}
+
+		++CurrentAttachmentIndex;
+	}
+}
+
 /**
 	______________________ _____________________________
 	\__    ___/\_   _____//   _____/\__    ___/   _____/
@@ -291,15 +530,82 @@ TEST_CASE("CbPackage.Serialization")
 	};
 
 	CbPackageReader Reader;
-	uint64_t		InitialRead = Reader.ProcessHeaderData(nullptr, 0);
-	uint64_t		NextBytes	= Reader.ProcessHeaderData(ConsumeBytes(InitialRead), InitialRead);
-	NextBytes					= Reader.ProcessHeaderData(ConsumeBytes(NextBytes), NextBytes);
+	uint64_t		InitialRead = Reader.ProcessPackageHeaderData(nullptr, 0);
+	uint64_t		NextBytes	= Reader.ProcessPackageHeaderData(ConsumeBytes(InitialRead), InitialRead);
+	NextBytes					= Reader.ProcessPackageHeaderData(ConsumeBytes(NextBytes), NextBytes);
 	auto Buffers				= Reader.GetPayloadBuffers();
 
 	for (auto& PayloadBuffer : Buffers)
 	{
 		CopyBytes(PayloadBuffer.MutableData(), PayloadBuffer.GetSize());
 	}
+
+	Reader.Finalize();
+}
+
+TEST_CASE("CbPackage.LocalRef")
+{
+	ScopedTemporaryDirectory TempDir;
+
+	auto Path1 = TempDir.Path() / "abcd";
+	auto Path2 = TempDir.Path() / "efgh";
+
+	{
+		IoBuffer Buffer1 = IoBufferBuilder::MakeCloneFromMemory(MakeMemoryView("abcd"));
+		IoBuffer Buffer2 = IoBufferBuilder::MakeCloneFromMemory(MakeMemoryView("efgh"));
+
+		WriteFile(Path1, Buffer1);
+		WriteFile(Path2, Buffer2);
+	}
+
+	// Make a test package
+
+	IoBuffer FileBuffer1 = IoBufferBuilder::MakeFromFile(Path1);
+	IoBuffer FileBuffer2 = IoBufferBuilder::MakeFromFile(Path2);
+
+	CbAttachment Attach1{SharedBuffer(FileBuffer1)};
+	CbAttachment Attach2{SharedBuffer(FileBuffer2)};
+
+	CbObjectWriter Cbo;
+	Cbo.AddAttachment("abcd", Attach1);
+	Cbo.AddAttachment("efgh", Attach2);
+
+	CbPackage Pkg;
+	Pkg.AddAttachment(Attach1);
+	Pkg.AddAttachment(Attach2);
+	Pkg.SetObject(Cbo.Save());
+
+	SharedBuffer   Buffer		  = FormatPackageMessageBuffer(Pkg, FormatFlags::kAllowLocalReferences).Flatten();
+	const uint8_t* CursorPtr	  = reinterpret_cast<const uint8_t*>(Buffer.GetData());
+	uint64_t	   RemainingBytes = Buffer.GetSize();
+
+	auto ConsumeBytes = [&](uint64_t ByteCount) {
+		ZEN_ASSERT(ByteCount <= RemainingBytes);
+		void* ReturnPtr = (void*)CursorPtr;
+		CursorPtr += ByteCount;
+		RemainingBytes -= ByteCount;
+		return ReturnPtr;
+	};
+
+	auto CopyBytes = [&](void* TargetBuffer, uint64_t ByteCount) {
+		ZEN_ASSERT(ByteCount <= RemainingBytes);
+		memcpy(TargetBuffer, CursorPtr, ByteCount);
+		CursorPtr += ByteCount;
+		RemainingBytes -= ByteCount;
+	};
+
+	CbPackageReader Reader;
+	uint64_t		InitialRead = Reader.ProcessPackageHeaderData(nullptr, 0);
+	uint64_t		NextBytes	= Reader.ProcessPackageHeaderData(ConsumeBytes(InitialRead), InitialRead);
+	NextBytes					= Reader.ProcessPackageHeaderData(ConsumeBytes(NextBytes), NextBytes);
+	auto Buffers				= Reader.GetPayloadBuffers();
+
+	for (auto& PayloadBuffer : Buffers)
+	{
+		CopyBytes(PayloadBuffer.MutableData(), PayloadBuffer.GetSize());
+	}
+
+	Reader.Finalize();
 }
 
 void
diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp
index 8e898eb18..19dba126a 100644
--- a/zenhttp/httpsys.cpp
+++ b/zenhttp/httpsys.cpp
@@ -1248,7 +1248,7 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService&
 
 			if (AcceptContentType != HttpContentType::kUnknownContentType)
 			{
-				m_UriUtf8.RemoveSuffix(uint32_t(m_UriUtf8.Size() - LastComponentIndex - LastDotIndex - 1));
+				m_UriUtf8.RemoveSuffix((uint32_t)(UriSuffix8.size() + 1));
 			}
 		}
 	}
diff --git a/zenhttp/include/zenhttp/httpshared.h b/zenhttp/include/zenhttp/httpshared.h
index a6a61485f..24ce0c85a 100644
--- a/zenhttp/include/zenhttp/httpshared.h
+++ b/zenhttp/include/zenhttp/httpshared.h
@@ -2,10 +2,12 @@
 
 #pragma once
 
+#include <zencore/compactbinarypackage.h>
 #include <zencore/iobuffer.h>
 #include <zencore/iohash.h>
 
 #include <functional>
+#include <gsl/gsl-lite.hpp>
 
 namespace zen {
 
@@ -24,6 +26,13 @@ class CompositeBuffer;
 
 	Structures and code related to handling CbPackage transactions
 
+	CbPackage instances are marshaled across the wire using a distinct message
+	format. We don't use the CbPackage serialization format provided by the
+	CbPackage implementation itself since that does not provide much flexibility
+	in how the attachment payloads are transmitted. The scheme below separates
+	metadata cleanly from payloads and this enables us to more efficiently
+	transmit them either via sendfile/TransmitFile like mechanisms, or by
+	reference/memory mapping in the local case.
  */
 
 struct CbPackageHeader
@@ -43,33 +52,59 @@ enum : uint32_t
 
 struct CbAttachmentEntry
 {
-	uint64_t PayloadSize;
-	uint32_t Flags;
-	IoHash	 AttachmentHash;
+	uint64_t PayloadSize;	  // Size of the associated payload data in the message
+	uint32_t Flags;			  // See flags below
+	IoHash	 AttachmentHash;  // Content Id for the attachment
 
 	enum
 	{
 		kIsCompressed = (1u << 0),	// Is marshaled using compressed buffer storage format
 		kIsObject	  = (1u << 1),	// Is compact binary object
 		kIsError	  = (1u << 2),	// Is error (compact binary formatted) object
+		kIsLocalRef	  = (1u << 3),	// Is "local reference"
 	};
 };
 
+struct CbAttachmentReferenceHeader
+{
+	uint64_t PayloadByteOffset	= 0;
+	uint64_t PayloadByteSize	= ~0u;
+	uint16_t AbsolutePathLength = 0;
+
+	// This header will be followed by UTF8 encoded absolute path to backing file
+};
+
 static_assert(sizeof(CbAttachmentEntry) == 32);
 
-std::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data);
-CompositeBuffer		  FormatPackageMessageBuffer(const CbPackage& Data);
+enum class FormatFlags
+{
+	kDefault			  = 0,
+	kAllowLocalReferences = (1u << 0)
+};
+
+gsl_DEFINE_ENUM_BITMASK_OPERATORS(FormatFlags);
+
+std::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data, FormatFlags Flags);
+CompositeBuffer		  FormatPackageMessageBuffer(const CbPackage& Data, FormatFlags Flags);
 CbPackage			  ParsePackageMessage(
 				IoBuffer												  Payload,
 				std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> CreateBuffer = [](const IoHash&, uint64_t Size) -> IoBuffer {
 		return IoBuffer{Size};
 				});
+bool IsPackageMessage(IoBuffer Payload);
+
+std::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data);
+CompositeBuffer		  FormatPackageMessageBuffer(const CbPackage& Data);
 
 /** Streaming reader for compact binary packages
 
 	The goal is to ultimately support zero-copy I/O, but for now there'll be some
 	copying involved on some platforms at least.
 
+	This approach to deserializing CbPackage data is more efficient than
+	`ParsePackageMessage` since it does not require the entire message to
+	be resident in a memory buffer
+
   */
 class CbPackageReader
 {
@@ -79,11 +114,18 @@ public:
 
 	void SetPayloadBufferCreator(std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> CreateBuffer);
 
-	/** Process header data
+	/** Process compact binary package data stream
+
+		The data stream must be in the serialization format produced by FormatPackageMessage
+
+		\return How many bytes must be fed to this function in the next call
 	 */
-	uint64_t ProcessHeaderData(const void* Data, uint64_t DataBytes);
+	uint64_t ProcessPackageHeaderData(const void* Data, uint64_t DataBytes);
 
-	std::span<IoBuffer> GetPayloadBuffers() { return m_PayloadBuffers; }
+	void							 Finalize();
+	const std::vector<CbAttachment>& GetAttachments() { return m_Attachments; }
+	CbObject						 GetRootObject() { return m_RootObject; }
+	std::span<IoBuffer>				 GetPayloadBuffers() { return m_PayloadBuffers; }
 
 private:
 	enum class State
@@ -97,7 +139,11 @@ private:
 	std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> m_CreateBuffer;
 	std::vector<IoBuffer>									  m_PayloadBuffers;
 	std::vector<CbAttachmentEntry>							  m_AttachmentEntries;
+	std::vector<CbAttachment>								  m_Attachments;
+	CbObject												  m_RootObject;
 	CbPackageHeader											  m_PackageHeader;
+
+	IoBuffer MarshalLocalChunkReference(IoBuffer AttachmentBuffer);
 };
 
 void forcelink_httpshared();
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 07866d4f0..45bbe062b 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -77,7 +77,7 @@ struct PutRequestData
 };
 
 namespace {
-	static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv;
+	static constinit std::string_view HttpZCacheRPCPrefix = "$rpc"sv;
 
 	struct HttpRequestData
 	{
@@ -87,8 +87,8 @@ namespace {
 		std::optional<IoHash>	   ValueContentId;
 	};
 
-	const char* ValidNamespaceNameCharacters = "abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ";
-	const char* ValidBucketNameCharacters	 = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+	constinit AsciiSet ValidNamespaceNameCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
+	constinit AsciiSet ValidBucketNameCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
 
 	std::optional<std::string> GetValidNamespaceName(std::string_view Name)
 	{
@@ -104,7 +104,7 @@ namespace {
 			return {};
 		}
 
-		if (Name.find_first_not_of(ValidNamespaceNameCharacters) != std::string::npos)
+		if (!AsciiSet::HasOnly(Name, ValidNamespaceNameCharactersSet))
 		{
 			ZEN_WARN("Namespace '{}' is invalid, invalid characters detected", Name);
 			return {};
@@ -120,11 +120,13 @@ namespace {
 			ZEN_WARN("Bucket name is invalid, empty bucket name is not allowed");
 			return {};
 		}
-		if (Name.find_first_not_of(ValidBucketNameCharacters) != std::string::npos)
+
+		if (!AsciiSet::HasOnly(Name, ValidBucketNameCharactersSet))
 		{
 			ZEN_WARN("Bucket name '{}' is invalid, invalid characters detected", Name);
 			return {};
 		}
+
 		return ToLower(Name);
 	}
 
diff --git a/zenserver/frontend/frontend.cpp b/zenserver/frontend/frontend.cpp
index 6d576876f..842587708 100644
--- a/zenserver/frontend/frontend.cpp
+++ b/zenserver/frontend/frontend.cpp
@@ -203,51 +203,40 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
 	// Dismiss if the URI contains .. anywhere to prevent arbitrary file reads
 	if (Uri.find("..") != Uri.npos)
 	{
-		Request.WriteResponse(HttpResponseCode::Forbidden);
-		return;
+		return Request.WriteResponse(HttpResponseCode::Forbidden);
 	}
 
 	// Map the file extension to a MIME type. To keep things constrained, only a
-	// small subset of file extensions is allowed.
-	HttpContentType ContentType = HttpContentType::kCOUNT;
-	size_t			DotIndex	= Uri.rfind(".");
-	if (DotIndex != Uri.npos)
+	// small subset of file extensions is allowed
+
+	HttpContentType ContentType = HttpContentType::kUnknownContentType;
+
+	if (const size_t DotIndex = Uri.rfind("."); DotIndex != Uri.npos)
 	{
-		const std::string_view DotExt = Uri.substr(DotIndex);
-		if (DotExt == ".html")
-			ContentType = HttpContentType::kHTML;
-		else if (DotExt == ".js")
-			ContentType = HttpContentType::kJSON;
-		else if (DotExt == ".css")
-			ContentType = HttpContentType::kCSS;
-		else if (DotExt == ".png")
-			ContentType = HttpContentType::kPNG;
-		else if (DotExt == ".ico")
-			ContentType = HttpContentType::kIcon;
+		const std::string_view DotExt = Uri.substr(DotIndex + 1);
+
+		ContentType = ParseContentType(DotExt);
 	}
 
-	if (ContentType == HttpContentType::kCOUNT)
+	if (ContentType == HttpContentType::kUnknownContentType)
 	{
-		Request.WriteResponse(HttpResponseCode::Forbidden);
-		return;
+		return Request.WriteResponse(HttpResponseCode::Forbidden);
 	}
 
 	// The given content directory overrides any zip-fs discovered in the binary
 	if (!m_Directory.empty())
 	{
 		FileContents File = ReadFile(m_Directory / Uri);
+
 		if (!File.ErrorCode)
 		{
-			Request.WriteResponse(HttpResponseCode::OK, ContentType, File.Data[0]);
-			return;
+			return Request.WriteResponse(HttpResponseCode::OK, ContentType, File.Data[0]);
 		}
 	}
 
-	IoBuffer FileBuffer = m_ZipFs.GetFile(Uri);
-	if (FileBuffer)
+	if (IoBuffer FileBuffer = m_ZipFs.GetFile(Uri))
 	{
-		Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer);
-		return;
+		return Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer);
 	}
 
 	Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv);
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index d18ae9e1a..7a55911da 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -450,7 +450,7 @@ ProjectStore::Oplog::IterateOplog(std::function<void(CbObject)>&& Handler)
 }
 
 std::optional<CbObject>
-ProjectStore::Oplog::GetOplog(const Oid& Key)
+ProjectStore::Oplog::GetOpByKey(const Oid& Key)
 {
 	RwLock::SharedLockScope _(m_OplogLock);
 
@@ -465,6 +465,19 @@ ProjectStore::Oplog::GetOplog(const Oid& Key)
 	return {};
 }
 
+std::optional<CbObject>
+ProjectStore::Oplog::GetOpByIndex(int Index)
+{
+	RwLock::SharedLockScope _(m_OplogLock);
+
+	if (const auto AddressEntryIt = m_OpAddressMap.find(Index); AddressEntryIt != m_OpAddressMap.end())
+	{
+		return m_Storage->GetOp(AddressEntryIt->second);
+	}
+
+	return {};
+}
+
 bool
 ProjectStore::Oplog::AddFileMapping(Oid FileId, IoHash Hash, std::string_view ServerPath, std::string_view ClientPath)
 {
@@ -809,7 +822,7 @@ ProjectStore::Project::DiscoverOplogs()
 
 	for (const std::filesystem::path& DirPath : DirContent.Directories)
 	{
-		OpenOplog(PathToUtf8(DirPath.stem()));
+		OpenOplog(PathToUtf8(DirPath.filename()));
 	}
 }
 
@@ -893,7 +906,7 @@ ProjectStore::DiscoverProjects()
 
 	for (const std::filesystem::path& DirPath : DirContent.Directories)
 	{
-		std::string DirName = PathToUtf8(DirPath.stem());
+		std::string DirName = PathToUtf8(DirPath.filename());
 		Project*	Project = OpenProject(DirName);
 
 		if (Project)
@@ -917,8 +930,6 @@ ProjectStore::IterateProjects(std::function<void(Project& Prj)>&& Fn)
 void
 ProjectStore::Flush()
 {
-	// TODO
-
 	RwLock::SharedLockScope _(m_ProjectsLock);
 
 	for (auto& Kv : m_Projects)
@@ -1057,6 +1068,8 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 	m_Router.AddPattern("chunk", "([[:xdigit:]]{24})");
 	m_Router.AddPattern("hash", "([[:xdigit:]]{40})");
 
+	// This would ideally just be the response for the root /prj endpoint but this is
+	// currently not possible for (arbitrary, external) technical reasons
 	m_Router.RegisterRoute(
 		"list",
 		[this](HttpRouterRequest& Req) {
@@ -1064,6 +1077,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 
 			CbWriter Response;
 			Response.BeginArray();
+
 			m_ProjectStore->IterateProjects([&Response](ProjectStore::Project& Prj) {
 				Response.BeginObject();
 				Response << "Id"sv << Prj.Identifier;
@@ -1265,7 +1279,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 
 			ProjectStore::Oplog& Log = *FoundLog;
 
-			Oid Obj = Oid::FromHexString(ChunkId);
+			const Oid Obj = Oid::FromHexString(ChunkId);
 
 			IoBuffer Chunk = Log.FindChunk(Obj);
 			if (!Chunk)
@@ -1606,12 +1620,88 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 
 	m_Router.RegisterRoute(
 		"{project}/oplog/{log}/{op}",
-		[](HttpRouterRequest& Req) {
+		[this](HttpRouterRequest& Req) {
 			HttpServerRequest& HttpReq = Req.ServerRequest();
 
-			// TODO: look up op and respond with the payload!
+			const std::string& ProjectId  = Req.GetCapture(1);
+			const std::string& OplogId	  = Req.GetCapture(2);
+			const std::string& OpIdString = Req.GetCapture(3);
+
+			ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId);
+
+			if (FoundLog == nullptr)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+
+			ProjectStore::Oplog& Oplog = *FoundLog;
+
+			if (const std::optional<int32_t> OpId = zen::ParseInt<uint32_t>(OpIdString))
+			{
+				if (std::optional<CbObject> MaybeOp = Oplog.GetOpByIndex(OpId.value()))
+				{
+					CbObject& Op = MaybeOp.value();
+					if (Req.ServerRequest().AcceptContentType() == ZenContentType::kCbPackage)
+					{
+						CbPackage Package;
+						Package.SetObject(Op);
+
+						Op.IterateAttachments([&](CbFieldView FieldView) {
+							const IoHash AttachmentHash = FieldView.AsAttachment();
+							IoBuffer	 Payload		= m_CidStore.FindChunkByCid(AttachmentHash);
+
+							// We force this for now as content type is not consistently tracked (will
+							// be fixed in CidStore refactor)
+							Payload.SetContentType(ZenContentType::kCompressedBinary);
+
+							if (Payload)
+							{
+								switch (Payload.GetContentType())
+								{
+									case ZenContentType::kCbObject:
+										if (CbObject Object = LoadCompactBinaryObject(Payload))
+										{
+											Package.AddAttachment(CbAttachment(Object));
+										}
+										else
+										{
+											// Error - malformed object
+
+											ZEN_ERROR("malformed object returned for {}", AttachmentHash);
+										}
+										break;
+
+									case ZenContentType::kCompressedBinary:
+										if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload)))
+										{
+											Package.AddAttachment(CbAttachment(Compressed));
+										}
+										else
+										{
+											// Error - not compressed!
+
+											ZEN_ERROR("invalid compressed binary returned for {}", AttachmentHash);
+										}
+										break;
+
+									default:
+										Package.AddAttachment(CbAttachment(SharedBuffer(Payload)));
+										break;
+								}
+							}
+						});
+
+						return HttpReq.WriteResponse(HttpResponseCode::Accepted, Package);
+					}
+					else
+					{
+						// Client cannot accept a package, so we only send the core object
+						return HttpReq.WriteResponse(HttpResponseCode::Accepted, Op);
+					}
+				}
+			}
 
-			HttpReq.WriteResponse(HttpResponseCode::Accepted, HttpContentType::kText, u8"yeee"sv);
+			return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 		},
 		HttpVerb::kGet);
 
@@ -1718,7 +1808,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 				if (auto OpKey = Params.GetValue("opkey"); !OpKey.empty())
 				{
 					Oid						OpKeyId = OpKeyStringAsOId(OpKey);
-					std::optional<CbObject> Op		= FoundLog->GetOplog(OpKeyId);
+					std::optional<CbObject> Op		= FoundLog->GetOpByKey(OpKeyId);
 
 					if (Op.has_value())
 					{
diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h
index f71434783..bf6afa592 100644
--- a/zenserver/projectstore.h
+++ b/zenserver/projectstore.h
@@ -69,7 +69,8 @@ public:
 
 		void IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn);
 		void IterateOplog(std::function<void(CbObject)>&& Fn);
-		std::optional<CbObject> GetOplog(const Oid& Key);
+		std::optional<CbObject> GetOpByKey(const Oid& Key);
+		std::optional<CbObject> GetOpByIndex(int Index);
 
 		IoBuffer FindChunk(Oid ChunkId);
 
diff --git a/zenserver/sos/sos.cpp b/zenserver/sos/sos.cpp
deleted file mode 100644
index 5fa6ffaae..000000000
--- a/zenserver/sos/sos.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "sos.h"
-
-#include <zencore/logging.h>
-
-namespace zen {
-
-HttpCommonStructuredObjectStore::HttpCommonStructuredObjectStore() : m_Log(logging::Get("sos"))
-{
-	m_Router.AddPattern("ns", "([[:alnum:]_-.]+)");
-	m_Router.AddPattern("bucket", "([[:alnum:]_-.]+)");
-	m_Router.AddPattern("hash", "([[:xdigit:]]{40})");
-}
-
-HttpCommonStructuredObjectStore::~HttpCommonStructuredObjectStore()
-{
-}
-
-const char*
-HttpCommonStructuredObjectStore::BaseUri() const
-{
-	return "/sos/";
-}
-
-void
-HttpCommonStructuredObjectStore::HandleRequest(zen::HttpServerRequest& HttpServiceRequest)
-{
-	ZEN_UNUSED(HttpServiceRequest);
-}
-
-}  // namespace zen
diff --git a/zenserver/sos/sos.h b/zenserver/sos/sos.h
deleted file mode 100644
index e602df8c4..000000000
--- a/zenserver/sos/sos.h
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zenhttp/httpserver.h>
-
-#include <zencore/logging.h>
-
-namespace zen {
-
-/** Simple Object Store API
- *
- * Implements an API shared with Jupiter
- *
- * - Objects (compact binary), named and private
- * - Blobs (unstructured binary), named and private
- *
- */
-
-class HttpCommonStructuredObjectStore : public zen::HttpService
-{
-public:
-	HttpCommonStructuredObjectStore();
-	virtual ~HttpCommonStructuredObjectStore();
-
-	virtual const char* BaseUri() const override;
-	virtual void		HandleRequest(zen::HttpServerRequest& HttpServiceRequest) override;
-
-private:
-	spdlog::logger&		   m_Log;
-	zen::HttpRequestRouter m_Router;
-};
-
-}  // namespace zen
diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp
index 0237ec346..cd6a531ca 100644
--- a/zenserver/upstream/zen.cpp
+++ b/zenserver/upstream/zen.cpp
@@ -408,15 +408,14 @@ ZenStructuredCacheSession::CheckHealth()
 }
 
 ZenCacheResult
-ZenStructuredCacheSession::GetCacheRecord(std::string_view, std::string_view BucketId, const IoHash& Key, ZenContentType Type)
+ZenStructuredCacheSession::GetCacheRecord(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType Type)
 {
 	ExtendableStringBuilder<256> Uri;
 	Uri << m_Client.ServiceUrl() << "/z$/";
-	// TODO: DE20220530: Disable adding namespace into URL until we have updated the shared instances with namespace support
-	//	if (Namespace != ZenCacheStore::DefaultNamespace)
-	//	{
-	//		Uri << Namespace << "/";
-	//	}
+	if (Namespace != ZenCacheStore::DefaultNamespace)
+	{
+		Uri << Namespace << "/";
+	}
 	Uri << BucketId << "/" << Key.ToHexString();
 
 	cpr::Session& Session = m_SessionState->GetSession();
@@ -438,15 +437,17 @@ ZenStructuredCacheSession::GetCacheRecord(std::string_view, std::string_view Buc
 }
 
 ZenCacheResult
-ZenStructuredCacheSession::GetCacheValue(std::string_view, std::string_view BucketId, const IoHash& Key, const IoHash& ValueContentId)
+ZenStructuredCacheSession::GetCacheValue(std::string_view Namespace,
+										 std::string_view BucketId,
+										 const IoHash&	  Key,
+										 const IoHash&	  ValueContentId)
 {
 	ExtendableStringBuilder<256> Uri;
 	Uri << m_Client.ServiceUrl() << "/z$/";
-	// TODO: DE20220530: Disable adding namespace into URL until we have updated the shared instances with namespace support
-	//	if (Namespace != ZenCacheStore::DefaultNamespace)
-	//	{
-	//		Uri << Namespace << "/";
-	//	}
+	if (Namespace != ZenCacheStore::DefaultNamespace)
+	{
+		Uri << Namespace << "/";
+	}
 	Uri << BucketId << "/" << Key.ToHexString() << "/" << ValueContentId.ToHexString();
 
 	cpr::Session& Session = m_SessionState->GetSession();
@@ -473,7 +474,7 @@ ZenStructuredCacheSession::GetCacheValue(std::string_view, std::string_view Buck
 }
 
 ZenCacheResult
-ZenStructuredCacheSession::PutCacheRecord(std::string_view,
+ZenStructuredCacheSession::PutCacheRecord(std::string_view Namespace,
 										  std::string_view BucketId,
 										  const IoHash&	   Key,
 										  IoBuffer		   Value,
@@ -481,11 +482,10 @@ ZenStructuredCacheSession::PutCacheRecord(std::string_view,
 {
 	ExtendableStringBuilder<256> Uri;
 	Uri << m_Client.ServiceUrl() << "/z$/";
-	// TODO: DE20220530: Disable adding namespace into URL until we have updated the shared instances with namespace support
-	//	if (Namespace != ZenCacheStore::DefaultNamespace)
-	//	{
-	//		Uri << Namespace << "/";
-	//	}
+	if (Namespace != ZenCacheStore::DefaultNamespace)
+	{
+		Uri << Namespace << "/";
+	}
 	Uri << BucketId << "/" << Key.ToHexString();
 
 	cpr::Session& Session = m_SessionState->GetSession();
@@ -510,7 +510,7 @@ ZenStructuredCacheSession::PutCacheRecord(std::string_view,
 }
 
 ZenCacheResult
-ZenStructuredCacheSession::PutCacheValue(std::string_view,
+ZenStructuredCacheSession::PutCacheValue(std::string_view Namespace,
 										 std::string_view BucketId,
 										 const IoHash&	  Key,
 										 const IoHash&	  ValueContentId,
@@ -518,11 +518,10 @@ ZenStructuredCacheSession::PutCacheValue(std::string_view,
 {
 	ExtendableStringBuilder<256> Uri;
 	Uri << m_Client.ServiceUrl() << "/z$/";
-	// TODO: DE20220530: Disable adding namespace into URL until we have updated the shared instances with namespace support
-	//	if (Namespace != ZenCacheStore::DefaultNamespace)
-	//	{
-	//		Uri << Namespace << "/";
-	//	}
+	if (Namespace != ZenCacheStore::DefaultNamespace)
+	{
+		Uri << Namespace << "/";
+	}
 	Uri << BucketId << "/" << Key.ToHexString() << "/" << ValueContentId.ToHexString();
 
 	cpr::Session& Session = m_SessionState->GetSession();
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 4db69c265..ff7d256f9 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -982,7 +982,7 @@ ZenEntryPoint::Run()
 		}
 		else
 		{
-			printf("sentry_init returned failure!");
+			printf("sentry_init returned failure! (error code: %d)", ErrorCode);
 		}
 	}
 
diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp
index 55bec817f..01eda4697 100644
--- a/zenstore/cidstore.cpp
+++ b/zenstore/cidstore.cpp
@@ -36,6 +36,8 @@ struct CidStore::Impl
 		IoBuffer	 Payload		= ChunkData.GetCompressed().Flatten().AsIoBuffer();
 		IoHash		 CompressedHash = IoHash::HashBuffer(Payload.Data(), Payload.Size());
 
+		Payload.SetContentType(ZenContentType::kCompressedBinary);
+
 		CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, CompressedHash);
 		AddCompressedCid(DecompressedId, CompressedHash);
 
diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp
index dfa3d54ab..bb03b9751 100644
--- a/zenstore/gc.cpp
+++ b/zenstore/gc.cpp
@@ -625,7 +625,7 @@ GcScheduler::SchedulerThread()
 					 NiceBytes(Space.Total),
 					 m_Config.Interval.count()
 						 ? fmt::format("{} until next GC", NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(RemaingTime).count())))
-						 : std::string("next scheduled GC no set"));
+						 : std::string("no GC scheduled"));
 
 			// TODO: Trigger GC if max disk usage water mark is reached