send payloads as duplicated handles (#240)

* send payloads as duplicated handles if requestor provides process id and allows local file references.
* linux/macos fixes
* tests
* fix access rights when duplicating handle
* fix closing of duplicated handles on error
* cleanup
* changelog

1 files changed, 130 insertions, 24 deletions

diff --git a/zenhttp/httpshared.cpp b/zenhttp/httpshared.cpp
index b6346413f..7aade56d2 100644
--- a/zenhttp/httpshared.cpp
+++ b/zenhttp/httpshared.cpp
@@ -10,6 +10,7 @@
 #include <zencore/iobuffer.h>
 #include <zencore/iohash.h>
 #include <zencore/logging.h>
+#include <zencore/scopeguard.h>
 #include <zencore/stream.h>
 #include <zencore/testing.h>
 #include <zencore/testutils.h>
@@ -23,21 +24,23 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen {
 
+const std::string_view HandlePrefix(":?#:");
+
 std::vector<IoBuffer>
-FormatPackageMessage(const CbPackage& Data)
+FormatPackageMessage(const CbPackage& Data, int TargetProcessPid)
 {
-	return FormatPackageMessage(Data, FormatFlags::kDefault);
+	return FormatPackageMessage(Data, FormatFlags::kDefault, TargetProcessPid);
 }
 CompositeBuffer
-FormatPackageMessageBuffer(const CbPackage& Data)
+FormatPackageMessageBuffer(const CbPackage& Data, int TargetProcessPid)
 {
-	return FormatPackageMessageBuffer(Data, FormatFlags::kDefault);
+	return FormatPackageMessageBuffer(Data, FormatFlags::kDefault, TargetProcessPid);
 }
 
 CompositeBuffer
-FormatPackageMessageBuffer(const CbPackage& Data, FormatFlags Flags)
+FormatPackageMessageBuffer(const CbPackage& Data, FormatFlags Flags, int TargetProcessPid)
 {
-	std::vector<IoBuffer> Message = FormatPackageMessage(Data, Flags);
+	std::vector<IoBuffer> Message = FormatPackageMessage(Data, Flags, TargetProcessPid);
 
 	std::vector<SharedBuffer> Buffers;
 
@@ -50,8 +53,44 @@ FormatPackageMessageBuffer(const CbPackage& Data, FormatFlags Flags)
 }
 
 std::vector<IoBuffer>
-FormatPackageMessage(const CbPackage& Data, FormatFlags Flags)
+FormatPackageMessage(const CbPackage& Data, FormatFlags Flags, int TargetProcessPid)
 {
+	void* TargetProcessHandle = nullptr;
+#if ZEN_PLATFORM_WINDOWS
+	std::vector<HANDLE> DuplicatedHandles;
+	auto				_ = MakeGuard([&DuplicatedHandles, &TargetProcessHandle]() {
+		   if (TargetProcessHandle == nullptr)
+		   {
+			   return;
+		   }
+
+		   for (HANDLE DuplicatedHandle : DuplicatedHandles)
+		   {
+			   HANDLE ClosingHandle;
+			   if (::DuplicateHandle((HANDLE)TargetProcessHandle,
+									 DuplicatedHandle,
+									 GetCurrentProcess(),
+									 &ClosingHandle,
+									 0,
+									 FALSE,
+									 DUPLICATE_CLOSE_SOURCE | DUPLICATE_SAME_ACCESS) == TRUE)
+			   {
+				   ::CloseHandle(ClosingHandle);
+			   }
+		   }
+		   ::CloseHandle((HANDLE)TargetProcessHandle);
+		   TargetProcessHandle = nullptr;
+	   });
+
+	if (EnumHasAllFlags(Flags, FormatFlags::kAllowLocalReferences) && TargetProcessPid != 0)
+	{
+		TargetProcessHandle = OpenProcess(PROCESS_DUP_HANDLE, FALSE, TargetProcessPid);
+	}
+#else
+	ZEN_UNUSED(TargetProcessPid);
+	void* DuplicatedHandles = nullptr;
+#endif	// ZEN_PLATFORM_WINDOWS
+
 	const std::span<const CbAttachment>& Attachments = Data.GetAttachments();
 	std::vector<IoBuffer>				 ResponseBuffers;
 
@@ -99,10 +138,11 @@ FormatPackageMessage(const CbPackage& Data, FormatFlags Flags)
 
 	tsl::robin_map<void*, std::string> FileNameMap;
 
-	auto IsLocalRef = [&FileNameMap](const CompositeBuffer&		  AttachmentBinary,
-									 bool						  DenyPartialLocalReferences,
-									 CbAttachmentReferenceHeader& LocalRef,
-									 std::string&				  Path8) -> bool {
+	auto IsLocalRef = [&FileNameMap, &DuplicatedHandles](const CompositeBuffer&		  AttachmentBinary,
+														 bool						  DenyPartialLocalReferences,
+														 void*						  TargetProcessHandle,
+														 CbAttachmentReferenceHeader& LocalRef,
+														 std::string&				  Path8) -> bool {
 		const SharedBuffer&	  Segment = AttachmentBinary.GetSegments().front();
 		IoBufferFileReference Ref;
 		const IoBuffer&		  SegmentBuffer = Segment.AsIoBuffer();
@@ -123,9 +163,36 @@ FormatPackageMessage(const CbPackage& Data, FormatFlags Flags)
 		}
 		else
 		{
-			ExtendablePathBuilder<256> LocalRefFile;
-			LocalRefFile.Append(std::filesystem::absolute(PathFromHandle(Ref.FileHandle)));
-			Path8 = LocalRefFile.ToUtf8();
+			bool UseFilePath = true;
+#if ZEN_PLATFORM_WINDOWS
+			if (TargetProcessHandle != nullptr)
+			{
+				HANDLE TargetHandle = INVALID_HANDLE_VALUE;
+				BOOL   OK			= ::DuplicateHandle(GetCurrentProcess(),
+											Ref.FileHandle,
+											(HANDLE)TargetProcessHandle,
+											&TargetHandle,
+											FILE_GENERIC_READ,
+											FALSE,
+											0);
+				if (OK)
+				{
+					DuplicatedHandles.push_back(TargetHandle);
+					Path8		= fmt::format("{}{}", HandlePrefix, reinterpret_cast<uint64_t>(TargetHandle));
+					UseFilePath = false;
+				}
+			}
+#else	// ZEN_PLATFORM_WINDOWS
+			ZEN_UNUSED(TargetProcessHandle);
+			// Not supported on Linux/Mac. Could potentially use pidfd_getfd() but that requires a fairly new Linux kernel/includes and to
+			// deal with acceess rights etc.
+#endif	// ZEN_PLATFORM_WINDOWS
+			if (UseFilePath)
+			{
+				ExtendablePathBuilder<256> LocalRefFile;
+				LocalRefFile.Append(std::filesystem::absolute(PathFromHandle(Ref.FileHandle)));
+				Path8 = LocalRefFile.ToUtf8();
+			}
 			FileNameMap.insert_or_assign(Ref.FileHandle, Path8);
 		}
 
@@ -159,14 +226,18 @@ FormatPackageMessage(const CbPackage& Data, FormatFlags Flags)
 
 			if (MarshalByLocalRef)
 			{
-				MarshalByLocalRef = IsLocalRef(Compressed, DenyPartialLocalReferences, LocalRef, Path8);
+				MarshalByLocalRef = IsLocalRef(Compressed, DenyPartialLocalReferences, TargetProcessHandle, LocalRef, Path8);
 			}
 
 			if (MarshalByLocalRef)
 			{
 				const bool IsCompressed = true;
+				bool	   IsHandle		= false;
+#if ZEN_PLATFORM_WINDOWS
+				IsHandle = Path8.starts_with(HandlePrefix);
+#endif
 				MarshalLocal(Path8, LocalRef, AttachmentHash, IsCompressed);
-				ZEN_DEBUG("Marshalled '{}' as file of {} bytes", Path8, Compressed.GetSize());
+				ZEN_DEBUG("Marshalled '{}' as file {} of {} bytes", Path8, IsHandle ? "handle" : "path", Compressed.GetSize());
 			}
 			else
 			{
@@ -201,14 +272,18 @@ FormatPackageMessage(const CbPackage& Data, FormatFlags Flags)
 
 			if (MarshalByLocalRef)
 			{
-				MarshalByLocalRef = IsLocalRef(AttachmentBinary, DenyPartialLocalReferences, LocalRef, Path8);
+				MarshalByLocalRef = IsLocalRef(AttachmentBinary, DenyPartialLocalReferences, TargetProcessHandle, LocalRef, Path8);
 			}
 
 			if (MarshalByLocalRef)
 			{
 				const bool IsCompressed = false;
+				bool	   IsHandle		= false;
+#if ZEN_PLATFORM_WINDOWS
+				IsHandle = Path8.starts_with(HandlePrefix);
+#endif
 				MarshalLocal(Path8, LocalRef, AttachmentHash, IsCompressed);
-				ZEN_DEBUG("Marshalled '{}' as file of {} bytes", Path8, AttachmentBinary.GetSize());
+				ZEN_DEBUG("Marshalled '{}' as file {} of {} bytes", Path8, IsHandle ? "handle" : "path", AttachmentBinary.GetSize());
 			}
 			else
 			{
@@ -226,6 +301,9 @@ FormatPackageMessage(const CbPackage& Data, FormatFlags Flags)
 		}
 	}
 	FileNameMap.clear();
+#if ZEN_PLATFORM_WINDOWS
+	DuplicatedHandles.clear();
+#endif	// ZEN_PLATFORM_WINDOWS
 
 	return ResponseBuffers;
 }
@@ -282,6 +360,8 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
 
 	tsl::robin_map<std::string, IoBuffer> PartialFileBuffers;
 
+	// TODO: Throwing before this loop completes could result in leaking handles as we might not have picked up all the handles in the
+	// message
 	for (uint32_t i = 0; i < ChunkCount; ++i)
 	{
 		const CbAttachmentEntry& Entry			= AttachmentEntries[i];
@@ -297,20 +377,46 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
 			ZEN_ASSERT(AttachmentBuffer.Size() >= sizeof(CbAttachmentReferenceHeader));
 
 			const CbAttachmentReferenceHeader* AttachRefHdr = AttachmentBuffer.Data<CbAttachmentReferenceHeader>();
-			const char8_t*					   PathPointer	= reinterpret_cast<const char8_t*>(AttachRefHdr + 1);
+			const char*						   PathPointer	= reinterpret_cast<const char*>(AttachRefHdr + 1);
 
 			ZEN_ASSERT(AttachmentBuffer.Size() >= (sizeof(CbAttachmentReferenceHeader) + AttachRefHdr->AbsolutePathLength));
-
-			std::filesystem::path Path{std::u8string_view(PathPointer, AttachRefHdr->AbsolutePathLength)};
+			std::string_view PathView(PathPointer, AttachRefHdr->AbsolutePathLength);
 
 			IoBuffer FullFileBuffer;
+
+			std::filesystem::path Path(Utf8ToWide(PathView));
 			if (auto It = PartialFileBuffers.find(Path.string()); It != PartialFileBuffers.end())
 			{
 				FullFileBuffer = It->second;
 			}
 			else
 			{
-				FullFileBuffer = PartialFileBuffers.insert_or_assign(Path.string(), IoBufferBuilder::MakeFromFile(Path)).first->second;
+				if (PathView.starts_with(HandlePrefix))
+				{
+#if ZEN_PLATFORM_WINDOWS
+					std::string_view		HandleString(PathView.substr(HandlePrefix.length()));
+					std::optional<uint64_t> HandleNumber(ParseInt<uint64_t>(HandleString));
+					if (HandleNumber.has_value())
+					{
+						HANDLE		   FileHandle = HANDLE(HandleNumber.value());
+						ULARGE_INTEGER liFileSize;
+						liFileSize.LowPart = ::GetFileSize(FileHandle, &liFileSize.HighPart);
+						if (liFileSize.LowPart != INVALID_FILE_SIZE)
+						{
+							FullFileBuffer = IoBuffer(IoBuffer::File, (void*)FileHandle, 0, uint64_t(liFileSize.QuadPart));
+							PartialFileBuffers.insert_or_assign(Path.string(), FullFileBuffer);
+						}
+					}
+#else	// ZEN_PLATFORM_WINDOWS
+		// Not supported on Linux/Mac. Could potentially use pidfd_getfd() but that requires a fairly new Linux kernel/includes
+		// and to deal with acceess rights etc.
+					ZEN_ASSERT(false);
+#endif	// ZEN_PLATFORM_WINDOWS
+				}
+				else
+				{
+					FullFileBuffer = PartialFileBuffers.insert_or_assign(Path.string(), IoBufferBuilder::MakeFromFile(Path)).first->second;
+				}
 			}
 
 			if (!FullFileBuffer)
@@ -318,7 +424,7 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
 				// Unable to open chunk reference
 				throw std::runtime_error(fmt::format("unable to resolve chunk #{} at '{}' (offset {}, size {})",
 													 i,
-													 PathToUtf8(Path),
+													 Path,
 													 AttachRefHdr->PayloadByteOffset,
 													 AttachRefHdr->PayloadByteSize));
 			}
@@ -332,7 +438,7 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
 			{
 				throw std::runtime_error(fmt::format("invalid format for chunk #{} at '{}' (offset {}, size {})",
 													 i,
-													 PathToUtf8(Path),
+													 Path,
 													 AttachRefHdr->PayloadByteOffset,
 													 AttachRefHdr->PayloadByteSize));
 			}