From 4e2649977d034b913413d2cb35d4a88afc30393f Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 13 Sep 2021 12:24:59 +0200 Subject: Changed interface for httpServerRequest::SessionId()/RequestId() so they share storage and lazy eval logic They now call into ParseSessionId()/ParseRequestId() when required Eliminates redundant logic in derived implementations Also moved package transport code into httpshared.(cpp|h) for easier sharing with client code Added some I/O error reporting in http.sys related code Changed IHttpPackageHandler interface to support partially updated handling flow --- zenhttp/httpshared.cpp | 116 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 zenhttp/httpshared.cpp (limited to 'zenhttp/httpshared.cpp') diff --git a/zenhttp/httpshared.cpp b/zenhttp/httpshared.cpp new file mode 100644 index 000000000..f11f91fae --- /dev/null +++ b/zenhttp/httpshared.cpp @@ -0,0 +1,116 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "httpshared.h" + +#include +#include +#include +#include + +#include +#include + +namespace zen { + +std::vector +FormatPackageMessage(const CbPackage& Data) +{ + const std::span& Attachments = Data.GetAttachments(); + + std::vector ResponseBuffers; + ResponseBuffers.reserve(3 + Attachments.size()); // TODO: may want to use an additional fudge factor here to avoid growing since each + // attachment is likely to consist of several buffers + + uint64_t TotalAttachmentsSize = 0; + + // Fixed size header + + CbPackageHeader Hdr{.HeaderMagic = kCbPkgMagic, .AttachmentCount = gsl::narrow(Attachments.size())}; + + ResponseBuffers.push_back(IoBufferBuilder::MakeCloneFromMemory(&Hdr, sizeof Hdr)); + + // Attachment metadata array + + IoBuffer AttachmentMetadataBuffer = IoBuffer{sizeof(CbAttachmentEntry) * (Attachments.size() + /* root */ 1)}; + + CbAttachmentEntry* AttachmentInfo = reinterpret_cast(AttachmentMetadataBuffer.MutableData()); + + ResponseBuffers.push_back(AttachmentMetadataBuffer); // Attachment metadata + + // Root object + + IoBuffer RootIoBuffer = Data.GetObject().GetBuffer().AsIoBuffer(); + ResponseBuffers.push_back(RootIoBuffer); // Root object + + *AttachmentInfo++ = {.AttachmentSize = RootIoBuffer.Size(), .AttachmentHash = Data.GetObjectHash()}; + + // Attachment payloads + + for (const CbAttachment& Attachment : Attachments) + { + CompressedBuffer AttachmentBuffer = Attachment.AsCompressedBinary(); + CompositeBuffer Compressed = AttachmentBuffer.GetCompressed(); + + *AttachmentInfo++ = {.AttachmentSize = AttachmentBuffer.GetCompressedSize(), + .AttachmentHash = IoHash::FromBLAKE3(AttachmentBuffer.GetRawHash())}; + + for (const SharedBuffer& Segment : Compressed.GetSegments()) + { + ResponseBuffers.push_back(Segment.AsIoBuffer()); + TotalAttachmentsSize += Segment.GetSize(); + } + } + + return std::move(ResponseBuffers); +} + +CbPackage +ParsePackageMessage(IoBuffer Payload) +{ + MemoryInStream InStream(Payload); + BinaryReader Reader(InStream); + + if (!Payload) + { + return {}; + } + + CbPackage Package; + + CbPackageHeader Hdr; + Reader.Read(&Hdr, sizeof Hdr); + + if (Hdr.HeaderMagic != kCbPkgMagic) + { + // report error + return {}; + } + + uint32_t ChunkCount = Hdr.AttachmentCount + 1; + + std::unique_ptr AttachmentEntries{new CbAttachmentEntry[ChunkCount]}; + + Reader.Read(AttachmentEntries.get(), sizeof(CbAttachmentEntry) * ChunkCount); + + for (uint32_t i = 0; i < ChunkCount; ++i) + { + const uint64_t AttachmentSize = AttachmentEntries[i].AttachmentSize; + IoBuffer AttachmentBuffer{AttachmentSize}; + Reader.Read(AttachmentBuffer.MutableData(), AttachmentSize); + CompressedBuffer CompBuf(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer))); + + if (i == 0) + { + Package.SetObject(LoadCompactBinaryObject(CompBuf)); + } + else + { + CbAttachment Attachment(CompBuf); + Package.AddAttachment(Attachment); + } + } + + return Package; +} + +} // namespace zen \ No newline at end of file -- cgit v1.2.3 From 11a2c78d69878e40a6ebafd00fd543f1f2eab702 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 13 Sep 2021 21:43:18 +0200 Subject: Introduced FormatPackageMessageBuffer() returning a ComositeBuffer --- zenhttp/httpshared.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'zenhttp/httpshared.cpp') diff --git a/zenhttp/httpshared.cpp b/zenhttp/httpshared.cpp index f11f91fae..85687b60b 100644 --- a/zenhttp/httpshared.cpp +++ b/zenhttp/httpshared.cpp @@ -3,6 +3,7 @@ #include "httpshared.h" #include +#include #include #include #include @@ -12,6 +13,21 @@ namespace zen { +CompositeBuffer +FormatPackageMessageBuffer(const CbPackage& Data) +{ + std::vector Message = FormatPackageMessage(Data); + + std::vector Buffers; + + for (IoBuffer& Buf : Message) + { + Buffers.push_back(SharedBuffer(Buf)); + } + + return CompositeBuffer(std::move(Buffers)); +} + std::vector FormatPackageMessage(const CbPackage& Data) { -- cgit v1.2.3 From f277fad0ea747807021bb92ae8fd026384901fb7 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 13 Sep 2021 22:25:03 +0200 Subject: Implemented intended package streaming API flow (but currently it "streams" from memory) --- zenhttp/httpshared.cpp | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'zenhttp/httpshared.cpp') diff --git a/zenhttp/httpshared.cpp b/zenhttp/httpshared.cpp index 85687b60b..68252a763 100644 --- a/zenhttp/httpshared.cpp +++ b/zenhttp/httpshared.cpp @@ -81,16 +81,16 @@ FormatPackageMessage(const CbPackage& Data) } CbPackage -ParsePackageMessage(IoBuffer Payload) +ParsePackageMessage(IoBuffer Payload, std::function CreateBuffer) { - MemoryInStream InStream(Payload); - BinaryReader Reader(InStream); - if (!Payload) { return {}; } + MemoryInStream InStream(Payload); + BinaryReader Reader(InStream); + CbPackage Package; CbPackageHeader Hdr; @@ -110,18 +110,24 @@ ParsePackageMessage(IoBuffer Payload) for (uint32_t i = 0; i < ChunkCount; ++i) { - const uint64_t AttachmentSize = AttachmentEntries[i].AttachmentSize; - IoBuffer AttachmentBuffer{AttachmentSize}; + const CbAttachmentEntry& Entry = AttachmentEntries[i]; + const uint64_t AttachmentSize = Entry.AttachmentSize; + IoBuffer AttachmentBuffer = CreateBuffer(Entry.AttachmentHash, AttachmentSize); + + ZEN_ASSERT(AttachmentBuffer); + ZEN_ASSERT(AttachmentBuffer.Size() == AttachmentSize); + Reader.Read(AttachmentBuffer.MutableData(), AttachmentSize); + CompressedBuffer CompBuf(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer))); if (i == 0) { - Package.SetObject(LoadCompactBinaryObject(CompBuf)); + Package.SetObject(LoadCompactBinaryObject(std::move(CompBuf))); } else { - CbAttachment Attachment(CompBuf); + CbAttachment Attachment(std::move(CompBuf)); Package.AddAttachment(Attachment); } } -- cgit v1.2.3