diff options
Diffstat (limited to 'src/zenhttp/httpshared.cpp')
| -rw-r--r-- | src/zenhttp/httpshared.cpp | 827 |
1 files changed, 0 insertions, 827 deletions
diff --git a/src/zenhttp/httpshared.cpp b/src/zenhttp/httpshared.cpp deleted file mode 100644 index ca014bf1c..000000000 --- a/src/zenhttp/httpshared.cpp +++ /dev/null @@ -1,827 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zenhttp/httpshared.h> - -#include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinarypackage.h> -#include <zencore/compositebuffer.h> -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/iobuffer.h> -#include <zencore/iohash.h> -#include <zencore/logging.h> -#include <zencore/scopeguard.h> -#include <zencore/stream.h> -#include <zencore/testing.h> -#include <zencore/testutils.h> -#include <zencore/trace.h> - -#include <span> -#include <vector> - -#if ZEN_PLATFORM_WINDOWS -# include <zencore/windows.h> -#endif - -ZEN_THIRD_PARTY_INCLUDES_START -#include <tsl/robin_map.h> -ZEN_THIRD_PARTY_INCLUDES_END - -namespace zen { - -const std::string_view HandlePrefix(":?#:"); - -std::vector<IoBuffer> -FormatPackageMessage(const CbPackage& Data, void* TargetProcessHandle) -{ - return FormatPackageMessage(Data, FormatFlags::kDefault, TargetProcessHandle); -} -CompositeBuffer -FormatPackageMessageBuffer(const CbPackage& Data, void* TargetProcessHandle) -{ - return FormatPackageMessageBuffer(Data, FormatFlags::kDefault, TargetProcessHandle); -} - -CompositeBuffer -FormatPackageMessageBuffer(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle) -{ - std::vector<IoBuffer> Message = FormatPackageMessage(Data, Flags, TargetProcessHandle); - - std::vector<SharedBuffer> Buffers; - - for (IoBuffer& Buf : Message) - { - Buffers.push_back(SharedBuffer(Buf)); - } - - return CompositeBuffer(std::move(Buffers)); -} - -static void -MarshalLocal(CbAttachmentEntry*& AttachmentInfo, - const std::string& Path8, - CbAttachmentReferenceHeader& LocalRef, - const IoHash& AttachmentHash, - bool IsCompressed, - std::vector<IoBuffer>& ResponseBuffers) -{ - IoBuffer RefBuffer(sizeof(CbAttachmentReferenceHeader) + Path8.size()); - - CbAttachmentReferenceHeader* RefHdr = RefBuffer.MutableData<CbAttachmentReferenceHeader>(); - *RefHdr++ = LocalRef; - memcpy(RefHdr, Path8.data(), Path8.size()); - - *AttachmentInfo++ = {.PayloadSize = RefBuffer.GetSize(), - .Flags = (IsCompressed ? uint32_t(CbAttachmentEntry::kIsCompressed) : 0u) | CbAttachmentEntry::kIsLocalRef, - .AttachmentHash = AttachmentHash}; - - ResponseBuffers.push_back(std::move(RefBuffer)); -}; - -static bool -IsLocalRef(tsl::robin_map<void*, std::string>& FileNameMap, - std::vector<void*>& DuplicatedHandles, - const CompositeBuffer& AttachmentBinary, - bool DenyPartialLocalReferences, - void* TargetProcessHandle, - CbAttachmentReferenceHeader& LocalRef, - std::string& Path8) -{ - const SharedBuffer& Segment = AttachmentBinary.GetSegments().front(); - IoBufferFileReference Ref; - const IoBuffer& SegmentBuffer = Segment.AsIoBuffer(); - - if (!SegmentBuffer.GetFileReference(Ref)) - { - return false; - } - - if (DenyPartialLocalReferences && !SegmentBuffer.IsWholeFile()) - { - return false; - } - - if (auto It = FileNameMap.find(Ref.FileHandle); It != FileNameMap.end()) - { - Path8 = It->second; - } - else - { - bool UseFilePath = true; -#if ZEN_PLATFORM_WINDOWS - if (TargetProcessHandle != nullptr) - { - HANDLE TargetHandle = INVALID_HANDLE_VALUE; - BOOL OK = ::DuplicateHandle(GetCurrentProcess(), - Ref.FileHandle, - (HANDLE)TargetProcessHandle, - &TargetHandle, - FILE_GENERIC_READ, - FALSE, - 0); - if (OK) - { - DuplicatedHandles.push_back((void*)TargetHandle); - Path8 = fmt::format("{}{}", HandlePrefix, reinterpret_cast<uint64_t>(TargetHandle)); - UseFilePath = false; - } - } -#else // ZEN_PLATFORM_WINDOWS - ZEN_UNUSED(TargetProcessHandle); - ZEN_UNUSED(DuplicatedHandles); - // Not supported on Linux/Mac. Could potentially use pidfd_getfd() but that requires a fairly new Linux kernel/includes and to - // deal with access rights etc. -#endif // ZEN_PLATFORM_WINDOWS - if (UseFilePath) - { - ExtendablePathBuilder<256> LocalRefFile; - LocalRefFile.Append(std::filesystem::absolute(PathFromHandle(Ref.FileHandle))); - Path8 = LocalRefFile.ToUtf8(); - } - FileNameMap.insert_or_assign(Ref.FileHandle, Path8); - } - - LocalRef.AbsolutePathLength = gsl::narrow<uint16_t>(Path8.size()); - LocalRef.PayloadByteOffset = Ref.FileChunkOffset; - LocalRef.PayloadByteSize = Ref.FileChunkSize; - - return true; -}; - -std::vector<IoBuffer> -FormatPackageMessage(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle) -{ - ZEN_TRACE_CPU("FormatPackageMessage"); - - std::vector<void*> DuplicatedHandles; -#if ZEN_PLATFORM_WINDOWS - auto _ = MakeGuard([&DuplicatedHandles, &TargetProcessHandle]() { - if (TargetProcessHandle == nullptr) - { - return; - } - - for (void* DuplicatedHandle : DuplicatedHandles) - { - HANDLE ClosingHandle; - if (::DuplicateHandle((HANDLE)TargetProcessHandle, - (HANDLE)DuplicatedHandle, - GetCurrentProcess(), - &ClosingHandle, - 0, - FALSE, - DUPLICATE_CLOSE_SOURCE | DUPLICATE_SAME_ACCESS) == TRUE) - { - ::CloseHandle(ClosingHandle); - } - } - }); -#endif // ZEN_PLATFORM_WINDOWS - - const std::span<const CbAttachment>& Attachments = Data.GetAttachments(); - std::vector<IoBuffer> ResponseBuffers; - - ResponseBuffers.reserve(3 + Attachments.size()); // TODO: may want to use an additional fudge factor here to avoid growing since each - // attachment is likely to consist of several buffers - - // Fixed size header - - CbPackageHeader Hdr{.HeaderMagic = kCbPkgMagic, .AttachmentCount = gsl::narrow<uint32_t>(Attachments.size())}; - - ResponseBuffers.push_back(IoBufferBuilder::MakeCloneFromMemory(&Hdr, sizeof Hdr)); - - // Attachment metadata array - - IoBuffer AttachmentMetadataBuffer = IoBuffer{sizeof(CbAttachmentEntry) * (Attachments.size() + /* root */ 1)}; - CbAttachmentEntry* AttachmentInfo = reinterpret_cast<CbAttachmentEntry*>(AttachmentMetadataBuffer.MutableData()); - - ResponseBuffers.push_back(AttachmentMetadataBuffer); // Attachment metadata - - // Root object - - IoBuffer RootIoBuffer = Data.GetObject().GetBuffer().AsIoBuffer(); - ResponseBuffers.push_back(RootIoBuffer); // Root object - - *AttachmentInfo++ = {.PayloadSize = RootIoBuffer.Size(), .Flags = CbAttachmentEntry::kIsObject, .AttachmentHash = Data.GetObjectHash()}; - - // Attachment payloads - tsl::robin_map<void*, std::string> FileNameMap; - - for (const CbAttachment& Attachment : Attachments) - { - if (Attachment.IsNull()) - { - ZEN_NOT_IMPLEMENTED("Null attachments are not supported"); - } - else if (CompressedBuffer AttachmentBuffer = Attachment.AsCompressedBinary()) - { - CompositeBuffer Compressed = AttachmentBuffer.GetCompressed(); - IoHash AttachmentHash = Attachment.GetHash(); - - // If the data is either not backed by a file, or there are multiple - // fragments then we cannot marshal it by local reference. We might - // want/need to extend this in the future to allow multiple chunk - // segments to be marshaled at once - - bool MarshalByLocalRef = EnumHasAllFlags(Flags, FormatFlags::kAllowLocalReferences) && (Compressed.GetSegments().size() == 1); - bool DenyPartialLocalReferences = EnumHasAllFlags(Flags, FormatFlags::kDenyPartialLocalReferences); - CbAttachmentReferenceHeader LocalRef; - std::string Path8; - - if (MarshalByLocalRef) - { - MarshalByLocalRef = IsLocalRef(FileNameMap, - DuplicatedHandles, - Compressed, - DenyPartialLocalReferences, - TargetProcessHandle, - LocalRef, - Path8); - } - - if (MarshalByLocalRef) - { - const bool IsCompressed = true; - bool IsHandle = false; -#if ZEN_PLATFORM_WINDOWS - IsHandle = Path8.starts_with(HandlePrefix); -#endif - MarshalLocal(AttachmentInfo, Path8, LocalRef, AttachmentHash, IsCompressed, ResponseBuffers); - ZEN_DEBUG("Marshalled '{}' as file {} of {} bytes", Path8, IsHandle ? "handle" : "path", Compressed.GetSize()); - } - else - { - *AttachmentInfo++ = {.PayloadSize = AttachmentBuffer.GetCompressedSize(), - .Flags = CbAttachmentEntry::kIsCompressed, - .AttachmentHash = AttachmentHash}; - - for (const SharedBuffer& Segment : Compressed.GetSegments()) - { - ResponseBuffers.push_back(Segment.AsIoBuffer()); - } - } - } - else if (CbObject AttachmentObject = Attachment.AsObject()) - { - IoBuffer ObjIoBuffer = AttachmentObject.GetBuffer().AsIoBuffer(); - ResponseBuffers.push_back(ObjIoBuffer); - - *AttachmentInfo++ = {.PayloadSize = ObjIoBuffer.Size(), - .Flags = CbAttachmentEntry::kIsObject, - .AttachmentHash = Attachment.GetHash()}; - } - else if (CompositeBuffer AttachmentBinary = Attachment.AsCompositeBinary()) - { - IoHash AttachmentHash = Attachment.GetHash(); - bool MarshalByLocalRef = - EnumHasAllFlags(Flags, FormatFlags::kAllowLocalReferences) && (AttachmentBinary.GetSegments().size() == 1); - bool DenyPartialLocalReferences = EnumHasAllFlags(Flags, FormatFlags::kDenyPartialLocalReferences); - - CbAttachmentReferenceHeader LocalRef; - std::string Path8; - - if (MarshalByLocalRef) - { - MarshalByLocalRef = IsLocalRef(FileNameMap, - DuplicatedHandles, - AttachmentBinary, - DenyPartialLocalReferences, - TargetProcessHandle, - LocalRef, - Path8); - } - - if (MarshalByLocalRef) - { - const bool IsCompressed = false; - bool IsHandle = false; -#if ZEN_PLATFORM_WINDOWS - IsHandle = Path8.starts_with(HandlePrefix); -#endif - MarshalLocal(AttachmentInfo, Path8, LocalRef, AttachmentHash, IsCompressed, ResponseBuffers); - ZEN_DEBUG("Marshalled '{}' as file {} of {} bytes", Path8, IsHandle ? "handle" : "path", AttachmentBinary.GetSize()); - } - else - { - *AttachmentInfo++ = {.PayloadSize = AttachmentBinary.GetSize(), .Flags = 0, .AttachmentHash = Attachment.GetHash()}; - - for (const SharedBuffer& Segment : AttachmentBinary.GetSegments()) - { - ResponseBuffers.push_back(Segment.AsIoBuffer()); - } - } - } - else - { - ZEN_NOT_IMPLEMENTED("Unknown attachment kind"); - } - } - FileNameMap.clear(); -#if ZEN_PLATFORM_WINDOWS - DuplicatedHandles.clear(); -#endif // ZEN_PLATFORM_WINDOWS - - return ResponseBuffers; -} - -bool -IsPackageMessage(IoBuffer Payload) -{ - if (!Payload) - { - return false; - } - - BinaryReader Reader(Payload); - - CbPackageHeader Hdr; - Reader.Read(&Hdr, sizeof Hdr); - - if (Hdr.HeaderMagic != kCbPkgMagic) - { - return false; - } - - return true; -} - -CbPackage -ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint64_t)> CreateBuffer) -{ - ZEN_TRACE_CPU("ParsePackageMessage"); - - if (!Payload) - { - return {}; - } - - BinaryReader Reader(Payload); - - CbPackageHeader Hdr; - Reader.Read(&Hdr, sizeof Hdr); - - if (Hdr.HeaderMagic != kCbPkgMagic) - { - throw std::runtime_error("invalid CbPackage header magic"); - } - - const uint32_t ChunkCount = Hdr.AttachmentCount + 1; - - std::unique_ptr<CbAttachmentEntry[]> AttachmentEntries{new CbAttachmentEntry[ChunkCount]}; - - Reader.Read(AttachmentEntries.get(), sizeof(CbAttachmentEntry) * ChunkCount); - - CbPackage Package; - - std::vector<CbAttachment> Attachments; - Attachments.reserve(ChunkCount); // Guessing here... - - tsl::robin_map<std::string, IoBuffer> PartialFileBuffers; - - // TODO: Throwing before this loop completes could result in leaking handles as we might not have picked up all the handles in the - // message - for (uint32_t i = 0; i < ChunkCount; ++i) - { - const CbAttachmentEntry& Entry = AttachmentEntries[i]; - const uint64_t AttachmentSize = Entry.PayloadSize; - - const IoBuffer AttachmentBuffer(Payload, Reader.CurrentOffset(), AttachmentSize); - Reader.Skip(AttachmentSize); - - if (Entry.Flags & CbAttachmentEntry::kIsLocalRef) - { - // Marshal local reference - a "pointer" to the chunk backing file - - ZEN_ASSERT(AttachmentBuffer.Size() >= sizeof(CbAttachmentReferenceHeader)); - - const CbAttachmentReferenceHeader* AttachRefHdr = AttachmentBuffer.Data<CbAttachmentReferenceHeader>(); - const char* PathPointer = reinterpret_cast<const char*>(AttachRefHdr + 1); - - ZEN_ASSERT(AttachmentBuffer.Size() >= (sizeof(CbAttachmentReferenceHeader) + AttachRefHdr->AbsolutePathLength)); - std::string_view PathView(PathPointer, AttachRefHdr->AbsolutePathLength); - - IoBuffer FullFileBuffer; - - std::filesystem::path Path(Utf8ToWide(PathView)); - if (auto It = PartialFileBuffers.find(Path.string()); It != PartialFileBuffers.end()) - { - FullFileBuffer = It->second; - } - else - { - if (PathView.starts_with(HandlePrefix)) - { -#if ZEN_PLATFORM_WINDOWS - std::string_view HandleString(PathView.substr(HandlePrefix.length())); - std::optional<uint64_t> HandleNumber(ParseInt<uint64_t>(HandleString)); - if (HandleNumber.has_value()) - { - HANDLE FileHandle = HANDLE(HandleNumber.value()); - ULARGE_INTEGER liFileSize; - liFileSize.LowPart = ::GetFileSize(FileHandle, &liFileSize.HighPart); - if (liFileSize.LowPart != INVALID_FILE_SIZE) - { - FullFileBuffer = - IoBuffer(IoBuffer::File, (void*)FileHandle, 0, uint64_t(liFileSize.QuadPart), /*IsWholeFile*/ true); - PartialFileBuffers.insert_or_assign(Path.string(), FullFileBuffer); - } - } -#else // ZEN_PLATFORM_WINDOWS - // Not supported on Linux/Mac. Could potentially use pidfd_getfd() but that requires a fairly new Linux kernel/includes - // and to deal with acceess rights etc. - ZEN_ASSERT(false); -#endif // ZEN_PLATFORM_WINDOWS - } - else - { - FullFileBuffer = PartialFileBuffers.insert_or_assign(Path.string(), IoBufferBuilder::MakeFromFile(Path)).first->second; - } - } - - if (!FullFileBuffer) - { - // Unable to open chunk reference - throw std::runtime_error(fmt::format("unable to resolve chunk #{} at '{}' (offset {}, size {})", - i, - Path, - AttachRefHdr->PayloadByteOffset, - AttachRefHdr->PayloadByteSize)); - } - - IoBuffer ChunkReference = AttachRefHdr->PayloadByteOffset == 0 && AttachRefHdr->PayloadByteSize == FullFileBuffer.GetSize() - ? FullFileBuffer - : IoBuffer(FullFileBuffer, AttachRefHdr->PayloadByteOffset, AttachRefHdr->PayloadByteSize); - - CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkReference))); - if (!CompBuf) - { - throw std::runtime_error(fmt::format("invalid format for chunk #{} at '{}' (offset {}, size {})", - i, - Path, - AttachRefHdr->PayloadByteOffset, - AttachRefHdr->PayloadByteSize)); - } - Attachments.emplace_back(CbAttachment(std::move(CompBuf), Entry.AttachmentHash)); - } - else if (Entry.Flags & CbAttachmentEntry::kIsCompressed) - { - if (Entry.Flags & CbAttachmentEntry::kIsObject) - { - if (i == 0) - { - CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer))); - if (!CompBuf) - { - throw std::runtime_error(fmt::format("invalid format for chunk #{} expected compressed buffer for CbObject", i)); - } - // First payload is always a compact binary object - Package.SetObject(LoadCompactBinaryObject(std::move(CompBuf))); - } - else - { - ZEN_NOT_IMPLEMENTED("Object attachments are not currently supported"); - } - } - else - { - CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer))); - if (!CompBuf) - { - throw std::runtime_error(fmt::format("invalid format for chunk #{} expected compressed buffer for attachment", i)); - } - Attachments.emplace_back(CbAttachment(std::move(CompBuf), Entry.AttachmentHash)); - } - } - else /* not compressed */ - { - if (Entry.Flags & CbAttachmentEntry::kIsObject) - { - if (i == 0) - { - Package.SetObject(LoadCompactBinaryObject(AttachmentBuffer)); - } - else - { - ZEN_NOT_IMPLEMENTED("Object attachments are not currently supported"); - } - } - else - { - // Make a copy of the buffer so the attachments don't reference the entire payload - IoBuffer AttachmentBufferCopy = CreateBuffer(Entry.AttachmentHash, AttachmentSize); - ZEN_ASSERT(AttachmentBufferCopy); - ZEN_ASSERT(AttachmentBufferCopy.Size() == AttachmentSize); - AttachmentBufferCopy.GetMutableView().CopyFrom(AttachmentBuffer.GetView()); - - Attachments.emplace_back(SharedBuffer{AttachmentBufferCopy}); - } - } - } - PartialFileBuffers.clear(); - - Package.AddAttachments(Attachments); - - return Package; -} - -bool -ParsePackageMessageWithLegacyFallback(const IoBuffer& Response, CbPackage& OutPackage) -{ - if (IsPackageMessage(Response)) - { - OutPackage = ParsePackageMessage(Response); - return true; - } - return OutPackage.TryLoad(Response); -} - -CbPackageReader::CbPackageReader() : m_CreateBuffer([](const IoHash&, uint64_t Size) -> IoBuffer { return IoBuffer{Size}; }) -{ -} - -CbPackageReader::~CbPackageReader() -{ -} - -void -CbPackageReader::SetPayloadBufferCreator(std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> CreateBuffer) -{ - m_CreateBuffer = CreateBuffer; -} - -uint64_t -CbPackageReader::ProcessPackageHeaderData(const void* Data, uint64_t DataBytes) -{ - ZEN_ASSERT(m_CurrentState != State::kReadingBuffers); - - switch (m_CurrentState) - { - case State::kInitialState: - ZEN_ASSERT(Data == nullptr); - m_CurrentState = State::kReadingHeader; - return sizeof m_PackageHeader; - - case State::kReadingHeader: - ZEN_ASSERT(DataBytes == sizeof m_PackageHeader); - memcpy(&m_PackageHeader, Data, sizeof m_PackageHeader); - ZEN_ASSERT(m_PackageHeader.HeaderMagic == kCbPkgMagic); - m_CurrentState = State::kReadingAttachmentEntries; - m_AttachmentEntries.resize(m_PackageHeader.AttachmentCount + 1); - return (m_PackageHeader.AttachmentCount + 1) * sizeof(CbAttachmentEntry); - - case State::kReadingAttachmentEntries: - ZEN_ASSERT(DataBytes == ((m_PackageHeader.AttachmentCount + 1) * sizeof(CbAttachmentEntry))); - memcpy(m_AttachmentEntries.data(), Data, DataBytes); - - for (CbAttachmentEntry& Entry : m_AttachmentEntries) - { - // This preallocates memory for payloads but note that for the local references - // the caller will need to handle the payload differently (i.e it's a - // CbAttachmentReferenceHeader not the actual payload) - - m_PayloadBuffers.push_back(IoBuffer{Entry.PayloadSize}); - } - - m_CurrentState = State::kReadingBuffers; - return 0; - - default: - ZEN_ASSERT(false); - return 0; - } -} - -IoBuffer -CbPackageReader::MarshalLocalChunkReference(IoBuffer AttachmentBuffer) -{ - // Marshal local reference - a "pointer" to the chunk backing file - - ZEN_ASSERT(AttachmentBuffer.Size() >= sizeof(CbAttachmentReferenceHeader)); - - const CbAttachmentReferenceHeader* AttachRefHdr = AttachmentBuffer.Data<CbAttachmentReferenceHeader>(); - const char8_t* PathPointer = reinterpret_cast<const char8_t*>(AttachRefHdr + 1); - - ZEN_ASSERT(AttachmentBuffer.Size() >= (sizeof(CbAttachmentReferenceHeader) + AttachRefHdr->AbsolutePathLength)); - - std::u8string_view PathView{PathPointer, AttachRefHdr->AbsolutePathLength}; - - std::filesystem::path Path{PathView}; - - IoBuffer ChunkReference = IoBufferBuilder::MakeFromFile(Path, AttachRefHdr->PayloadByteOffset, AttachRefHdr->PayloadByteSize); - - if (!ChunkReference) - { - // Unable to open chunk reference - - throw std::runtime_error(fmt::format("unable to resolve local reference to '{}' (offset {}, size {})", - PathToUtf8(Path), - AttachRefHdr->PayloadByteOffset, - AttachRefHdr->PayloadByteSize)); - } - - return ChunkReference; -}; - -void -CbPackageReader::Finalize() -{ - if (m_AttachmentEntries.empty()) - { - return; - } - - m_Attachments.reserve(m_AttachmentEntries.size() - 1); - - int CurrentAttachmentIndex = 0; - for (CbAttachmentEntry& Entry : m_AttachmentEntries) - { - IoBuffer AttachmentBuffer = m_PayloadBuffers[CurrentAttachmentIndex]; - - if (CurrentAttachmentIndex == 0) - { - // Root object - if (Entry.Flags & CbAttachmentEntry::kIsObject) - { - if (Entry.Flags & CbAttachmentEntry::kIsLocalRef) - { - m_RootObject = LoadCompactBinaryObject(MarshalLocalChunkReference(AttachmentBuffer)); - } - else if (Entry.Flags & CbAttachmentEntry::kIsCompressed) - { - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer), RawHash, RawSize); - if (RawHash == Entry.AttachmentHash) - { - m_RootObject = LoadCompactBinaryObject(Compressed); - } - } - else - { - m_RootObject = LoadCompactBinaryObject(std::move(AttachmentBuffer)); - } - } - else - { - throw std::runtime_error("missing or invalid root object"); - } - } - else if (Entry.Flags & CbAttachmentEntry::kIsLocalRef) - { - IoBuffer ChunkReference = MarshalLocalChunkReference(AttachmentBuffer); - - if (Entry.Flags & CbAttachmentEntry::kIsCompressed) - { - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(ChunkReference), RawHash, RawSize); - if (RawHash == Entry.AttachmentHash) - { - m_Attachments.push_back(CbAttachment(Compressed, Entry.AttachmentHash)); - } - } - else - { - CompressedBuffer Compressed = - CompressedBuffer::Compress(SharedBuffer(ChunkReference), OodleCompressor::NotSet, OodleCompressionLevel::None); - m_Attachments.push_back(CbAttachment(std::move(Compressed), Compressed.DecodeRawHash())); - } - } - - ++CurrentAttachmentIndex; - } -} - -/** - ______________________ _____________________________ - \__ ___/\_ _____// _____/\__ ___/ _____/ - | | | __)_ \_____ \ | | \_____ \ - | | | \/ \ | | / \ - |____| /_______ /_______ / |____| /_______ / - \/ \/ \/ - */ - -#if ZEN_WITH_TESTS - -TEST_CASE("CbPackage.Serialization") -{ - // Make a test package - - CbAttachment Attach1{SharedBuffer::MakeView(MakeMemoryView("abcd"))}; - CbAttachment Attach2{SharedBuffer::MakeView(MakeMemoryView("efgh"))}; - - CbObjectWriter Cbo; - Cbo.AddAttachment("abcd", Attach1); - Cbo.AddAttachment("efgh", Attach2); - - CbPackage Pkg; - Pkg.AddAttachment(Attach1); - Pkg.AddAttachment(Attach2); - Pkg.SetObject(Cbo.Save()); - - SharedBuffer Buffer = FormatPackageMessageBuffer(Pkg).Flatten(); - const uint8_t* CursorPtr = reinterpret_cast<const uint8_t*>(Buffer.GetData()); - uint64_t RemainingBytes = Buffer.GetSize(); - - auto ConsumeBytes = [&](uint64_t ByteCount) { - ZEN_ASSERT(ByteCount <= RemainingBytes); - void* ReturnPtr = (void*)CursorPtr; - CursorPtr += ByteCount; - RemainingBytes -= ByteCount; - return ReturnPtr; - }; - - auto CopyBytes = [&](void* TargetBuffer, uint64_t ByteCount) { - ZEN_ASSERT(ByteCount <= RemainingBytes); - memcpy(TargetBuffer, CursorPtr, ByteCount); - CursorPtr += ByteCount; - RemainingBytes -= ByteCount; - }; - - CbPackageReader Reader; - uint64_t InitialRead = Reader.ProcessPackageHeaderData(nullptr, 0); - uint64_t NextBytes = Reader.ProcessPackageHeaderData(ConsumeBytes(InitialRead), InitialRead); - NextBytes = Reader.ProcessPackageHeaderData(ConsumeBytes(NextBytes), NextBytes); - auto Buffers = Reader.GetPayloadBuffers(); - - for (auto& PayloadBuffer : Buffers) - { - CopyBytes(PayloadBuffer.MutableData(), PayloadBuffer.GetSize()); - } - - Reader.Finalize(); -} - -TEST_CASE("CbPackage.LocalRef") -{ - ScopedTemporaryDirectory TempDir; - - auto Path1 = TempDir.Path() / "abcd"; - auto Path2 = TempDir.Path() / "efgh"; - - { - IoBuffer Buffer1 = IoBufferBuilder::MakeCloneFromMemory(MakeMemoryView("abcd")); - IoBuffer Buffer2 = IoBufferBuilder::MakeCloneFromMemory(MakeMemoryView("efgh")); - - WriteFile(Path1, Buffer1); - WriteFile(Path2, Buffer2); - } - - // Make a test package - - IoBuffer FileBuffer1 = IoBufferBuilder::MakeFromFile(Path1); - IoBuffer FileBuffer2 = IoBufferBuilder::MakeFromFile(Path2); - - CbAttachment Attach1{SharedBuffer(FileBuffer1)}; - CbAttachment Attach2{SharedBuffer(FileBuffer2)}; - - CbObjectWriter Cbo; - Cbo.AddAttachment("abcd", Attach1); - Cbo.AddAttachment("efgh", Attach2); - - CbPackage Pkg; - Pkg.AddAttachment(Attach1); - Pkg.AddAttachment(Attach2); - Pkg.SetObject(Cbo.Save()); - - SharedBuffer Buffer = FormatPackageMessageBuffer(Pkg, FormatFlags::kAllowLocalReferences).Flatten(); - const uint8_t* CursorPtr = reinterpret_cast<const uint8_t*>(Buffer.GetData()); - uint64_t RemainingBytes = Buffer.GetSize(); - - auto ConsumeBytes = [&](uint64_t ByteCount) { - ZEN_ASSERT(ByteCount <= RemainingBytes); - void* ReturnPtr = (void*)CursorPtr; - CursorPtr += ByteCount; - RemainingBytes -= ByteCount; - return ReturnPtr; - }; - - auto CopyBytes = [&](void* TargetBuffer, uint64_t ByteCount) { - ZEN_ASSERT(ByteCount <= RemainingBytes); - memcpy(TargetBuffer, CursorPtr, ByteCount); - CursorPtr += ByteCount; - RemainingBytes -= ByteCount; - }; - - CbPackageReader Reader; - uint64_t InitialRead = Reader.ProcessPackageHeaderData(nullptr, 0); - uint64_t NextBytes = Reader.ProcessPackageHeaderData(ConsumeBytes(InitialRead), InitialRead); - NextBytes = Reader.ProcessPackageHeaderData(ConsumeBytes(NextBytes), NextBytes); - auto Buffers = Reader.GetPayloadBuffers(); - - for (auto& PayloadBuffer : Buffers) - { - CopyBytes(PayloadBuffer.MutableData(), PayloadBuffer.GetSize()); - } - - Reader.Finalize(); -} - -void -forcelink_httpshared() -{ -} - -#endif - -} // namespace zen |