From a5158f9fc806d506590dd9bf0e3282cb76c3ac4e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 16 Jan 2025 09:19:08 +0100 Subject: move basicfile.h/cpp -> zencore (#273) move jupiter.h/cpp -> zenutil move packageformat.h/.cpp -> zenhttp zenutil now depends on zenhttp instead of the inverse --- src/zenutil/packageformat.cpp | 894 ------------------------------------------ 1 file changed, 894 deletions(-) delete mode 100644 src/zenutil/packageformat.cpp (limited to 'src/zenutil/packageformat.cpp') diff --git a/src/zenutil/packageformat.cpp b/src/zenutil/packageformat.cpp deleted file mode 100644 index 579e0d13c..000000000 --- a/src/zenutil/packageformat.cpp +++ /dev/null @@ -1,894 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#if ZEN_PLATFORM_WINDOWS -# include -#endif - -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END - -namespace zen { - -const std::string_view HandlePrefix(":?#:"); - -std::vector -FormatPackageMessage(const CbPackage& Data, void* TargetProcessHandle) -{ - return FormatPackageMessage(Data, FormatFlags::kDefault, TargetProcessHandle); -} -CompositeBuffer -FormatPackageMessageBuffer(const CbPackage& Data, void* TargetProcessHandle) -{ - return FormatPackageMessageBuffer(Data, FormatFlags::kDefault, TargetProcessHandle); -} - -CompositeBuffer -FormatPackageMessageBuffer(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle) -{ - return CompositeBuffer(FormatPackageMessage(Data, Flags, TargetProcessHandle)); -} - -static void -MarshalLocal(CbAttachmentEntry*& AttachmentInfo, - const std::string& Path8, - CbAttachmentReferenceHeader& LocalRef, - const IoHash& AttachmentHash, - bool IsCompressed, - std::vector& ResponseBuffers) -{ - IoBuffer RefBuffer(sizeof(CbAttachmentReferenceHeader) + Path8.size()); - - CbAttachmentReferenceHeader* RefHdr = RefBuffer.MutableData(); - *RefHdr++ = LocalRef; - memcpy(RefHdr, Path8.data(), Path8.size()); - - *AttachmentInfo++ = {.PayloadSize = RefBuffer.GetSize(), - .Flags = (IsCompressed ? uint32_t(CbAttachmentEntry::kIsCompressed) : 0u) | CbAttachmentEntry::kIsLocalRef, - .AttachmentHash = AttachmentHash}; - - ResponseBuffers.emplace_back(std::move(RefBuffer)); -}; - -static bool -IsLocalRef(tsl::robin_map& FileNameMap, - std::vector& DuplicatedHandles, - const CompositeBuffer& AttachmentBinary, - bool DenyPartialLocalReferences, - void* TargetProcessHandle, - CbAttachmentReferenceHeader& LocalRef, - std::string& Path8) -{ - const SharedBuffer& Segment = AttachmentBinary.GetSegments().front(); - IoBufferFileReference Ref; - const IoBuffer& SegmentBuffer = Segment.AsIoBuffer(); - - if (!SegmentBuffer.GetFileReference(Ref)) - { - return false; - } - - if (DenyPartialLocalReferences && !SegmentBuffer.IsWholeFile()) - { - return false; - } - - if (auto It = FileNameMap.find(Ref.FileHandle); It != FileNameMap.end()) - { - Path8 = It->second; - } - else - { - bool UseFilePath = true; -#if ZEN_PLATFORM_WINDOWS - if (TargetProcessHandle != nullptr) - { - HANDLE TargetHandle = INVALID_HANDLE_VALUE; - BOOL OK = ::DuplicateHandle(GetCurrentProcess(), - Ref.FileHandle, - (HANDLE)TargetProcessHandle, - &TargetHandle, - FILE_GENERIC_READ, - FALSE, - 0); - if (OK) - { - DuplicatedHandles.push_back((void*)TargetHandle); - Path8 = fmt::format("{}{}", HandlePrefix, reinterpret_cast(TargetHandle)); - UseFilePath = false; - } - } -#else // ZEN_PLATFORM_WINDOWS - ZEN_UNUSED(TargetProcessHandle); - ZEN_UNUSED(DuplicatedHandles); - // Not supported on Linux/Mac. Could potentially use pidfd_getfd() but that requires a fairly new Linux kernel/includes and to - // deal with access rights etc. -#endif // ZEN_PLATFORM_WINDOWS - if (UseFilePath) - { - ExtendablePathBuilder<256> LocalRefFile; - std::error_code Ec; - std::filesystem::path FilePath = PathFromHandle(Ref.FileHandle, Ec); - if (Ec) - { - ZEN_WARN("Failed to get path for file handle {} in IsLocalRef check, reason '{}'", Ref.FileHandle, Ec.message()); - return false; - } - LocalRefFile.Append(std::filesystem::absolute(FilePath)); - Path8 = LocalRefFile.ToUtf8(); - } - FileNameMap.insert_or_assign(Ref.FileHandle, Path8); - } - - LocalRef.AbsolutePathLength = gsl::narrow(Path8.size()); - LocalRef.PayloadByteOffset = Ref.FileChunkOffset; - LocalRef.PayloadByteSize = Ref.FileChunkSize; - - return true; -}; - -std::vector -FormatPackageMessage(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle) -{ - ZEN_TRACE_CPU("FormatPackageMessage"); - - std::vector DuplicatedHandles; -#if ZEN_PLATFORM_WINDOWS - auto _ = MakeGuard([&DuplicatedHandles, &TargetProcessHandle]() { - if (TargetProcessHandle == nullptr) - { - return; - } - - for (void* DuplicatedHandle : DuplicatedHandles) - { - HANDLE ClosingHandle; - if (::DuplicateHandle((HANDLE)TargetProcessHandle, - (HANDLE)DuplicatedHandle, - GetCurrentProcess(), - &ClosingHandle, - 0, - FALSE, - DUPLICATE_CLOSE_SOURCE | DUPLICATE_SAME_ACCESS) == TRUE) - { - ::CloseHandle(ClosingHandle); - } - } - }); -#endif // ZEN_PLATFORM_WINDOWS - - const std::span& Attachments = Data.GetAttachments(); - std::vector ResponseBuffers; - - ResponseBuffers.reserve(2 + Attachments.size()); // TODO: may want to use an additional fudge factor here to avoid growing since each - // attachment is likely to consist of several buffers - - IoBuffer AttachmentMetadataBuffer = IoBuffer{sizeof(CbPackageHeader) + sizeof(CbAttachmentEntry) * (Attachments.size() + /* root */ 1)}; - MutableMemoryView HeaderView = AttachmentMetadataBuffer.GetMutableView(); - // Fixed size header - - CbPackageHeader* Hdr = (CbPackageHeader*)HeaderView.GetData(); - *Hdr = {.HeaderMagic = kCbPkgMagic, .AttachmentCount = gsl::narrow(Attachments.size())}; - HeaderView.MidInline(sizeof(CbPackageHeader)); - - // Attachment metadata array - CbAttachmentEntry* AttachmentInfo = reinterpret_cast(HeaderView.GetData()); - ResponseBuffers.emplace_back(std::move(AttachmentMetadataBuffer)); // Attachment metadata - - // Root object - - IoBuffer RootIoBuffer = Data.GetObject().GetBuffer().AsIoBuffer(); - ZEN_ASSERT(RootIoBuffer.GetSize() > 0); - *AttachmentInfo++ = {.PayloadSize = RootIoBuffer.Size(), .Flags = CbAttachmentEntry::kIsObject, .AttachmentHash = Data.GetObjectHash()}; - ResponseBuffers.emplace_back(std::move(RootIoBuffer)); // Root object - - // Attachment payloads - tsl::robin_map FileNameMap; - - for (const CbAttachment& Attachment : Attachments) - { - if (Attachment.IsNull()) - { - ZEN_NOT_IMPLEMENTED("Null attachments are not supported"); - } - else if (const CompressedBuffer& AttachmentBuffer = Attachment.AsCompressedBinary()) - { - const CompositeBuffer& Compressed = AttachmentBuffer.GetCompressed(); - IoHash AttachmentHash = Attachment.GetHash(); - - // If the data is either not backed by a file, or there are multiple - // fragments then we cannot marshal it by local reference. We might - // want/need to extend this in the future to allow multiple chunk - // segments to be marshaled at once - - bool MarshalByLocalRef = EnumHasAllFlags(Flags, FormatFlags::kAllowLocalReferences) && (Compressed.GetSegments().size() == 1); - bool DenyPartialLocalReferences = EnumHasAllFlags(Flags, FormatFlags::kDenyPartialLocalReferences); - CbAttachmentReferenceHeader LocalRef; - std::string Path8; - - if (MarshalByLocalRef) - { - MarshalByLocalRef = IsLocalRef(FileNameMap, - DuplicatedHandles, - Compressed, - DenyPartialLocalReferences, - TargetProcessHandle, - LocalRef, - Path8); - } - - if (MarshalByLocalRef) - { - const bool IsCompressed = true; - bool IsHandle = false; -#if ZEN_PLATFORM_WINDOWS - IsHandle = Path8.starts_with(HandlePrefix); -#endif - MarshalLocal(AttachmentInfo, Path8, LocalRef, AttachmentHash, IsCompressed, ResponseBuffers); - ZEN_DEBUG("Marshalled '{}' as file {} of {} bytes", Path8, IsHandle ? "handle" : "path", Compressed.GetSize()); - } - else - { - *AttachmentInfo++ = {.PayloadSize = AttachmentBuffer.GetCompressedSize(), - .Flags = CbAttachmentEntry::kIsCompressed, - .AttachmentHash = AttachmentHash}; - - std::span Segments = Compressed.GetSegments(); - ResponseBuffers.reserve(ResponseBuffers.size() + Segments.size() - 1); - for (const SharedBuffer& Segment : Segments) - { - ZEN_ASSERT(Segment.GetSize() > 0); - ResponseBuffers.emplace_back(Segment.AsIoBuffer()); - } - } - } - else if (CbObject AttachmentObject = Attachment.AsObject()) - { - IoBuffer ObjIoBuffer = AttachmentObject.GetBuffer().AsIoBuffer(); - ZEN_ASSERT(ObjIoBuffer.GetSize() > 0); - ResponseBuffers.emplace_back(std::move(ObjIoBuffer)); - - *AttachmentInfo++ = {.PayloadSize = ObjIoBuffer.Size(), - .Flags = CbAttachmentEntry::kIsObject, - .AttachmentHash = Attachment.GetHash()}; - } - else if (const CompositeBuffer& AttachmentBinary = Attachment.AsCompositeBinary()) - { - IoHash AttachmentHash = Attachment.GetHash(); - bool MarshalByLocalRef = - EnumHasAllFlags(Flags, FormatFlags::kAllowLocalReferences) && (AttachmentBinary.GetSegments().size() == 1); - bool DenyPartialLocalReferences = EnumHasAllFlags(Flags, FormatFlags::kDenyPartialLocalReferences); - - CbAttachmentReferenceHeader LocalRef; - std::string Path8; - - if (MarshalByLocalRef) - { - MarshalByLocalRef = IsLocalRef(FileNameMap, - DuplicatedHandles, - AttachmentBinary, - DenyPartialLocalReferences, - TargetProcessHandle, - LocalRef, - Path8); - } - - if (MarshalByLocalRef) - { - const bool IsCompressed = false; - bool IsHandle = false; -#if ZEN_PLATFORM_WINDOWS - IsHandle = Path8.starts_with(HandlePrefix); -#endif - MarshalLocal(AttachmentInfo, Path8, LocalRef, AttachmentHash, IsCompressed, ResponseBuffers); - ZEN_DEBUG("Marshalled '{}' as file {} of {} bytes", Path8, IsHandle ? "handle" : "path", AttachmentBinary.GetSize()); - } - else - { - *AttachmentInfo++ = {.PayloadSize = AttachmentBinary.GetSize(), .Flags = 0, .AttachmentHash = Attachment.GetHash()}; - - std::span Segments = AttachmentBinary.GetSegments(); - ResponseBuffers.reserve(ResponseBuffers.size() + Segments.size() - 1); - for (const SharedBuffer& Segment : Segments) - { - ZEN_ASSERT(Segment.GetSize() > 0); - ResponseBuffers.emplace_back(Segment.AsIoBuffer()); - } - } - } - else - { - ZEN_NOT_IMPLEMENTED("Unknown attachment kind"); - } - } - FileNameMap.clear(); -#if ZEN_PLATFORM_WINDOWS - DuplicatedHandles.clear(); -#endif // ZEN_PLATFORM_WINDOWS - - return ResponseBuffers; -} - -bool -IsPackageMessage(IoBuffer Payload) -{ - if (Payload.GetSize() < sizeof(CbPackageHeader)) - { - return false; - } - - BinaryReader Reader(Payload); - const CbPackageHeader* Hdr = reinterpret_cast(Reader.GetView(sizeof(CbPackageHeader)).GetData()); - if (Hdr->HeaderMagic != kCbPkgMagic) - { - return false; - } - - return true; -} - -CbPackage -ParsePackageMessage(IoBuffer Payload, std::function CreateBuffer) -{ - ZEN_TRACE_CPU("ParsePackageMessage"); - - if (Payload.GetSize() < sizeof(CbPackageHeader)) - { - throw std::invalid_argument(fmt::format("invalid CbPackage, missing complete header (size {})", Payload.GetSize())); - } - - BinaryReader Reader(Payload); - - const CbPackageHeader* Hdr = reinterpret_cast(Reader.GetView(sizeof(CbPackageHeader)).GetData()); - if (Hdr->HeaderMagic != kCbPkgMagic) - { - throw std::invalid_argument( - fmt::format("invalid CbPackage header magic, expected {0:x}, got {0:x}", static_cast(kCbPkgMagic), Hdr->HeaderMagic)); - } - Reader.Skip(sizeof(CbPackageHeader)); - - const uint32_t ChunkCount = Hdr->AttachmentCount + 1; - - if (Reader.Remaining() < sizeof(CbAttachmentEntry) * ChunkCount) - { - throw std::invalid_argument(fmt::format("invalid CbPackage, missing attachment entry data (need {} bytes, have {} bytes)", - sizeof(CbAttachmentEntry) * ChunkCount, - Reader.Remaining())); - } - const CbAttachmentEntry* AttachmentEntries = - reinterpret_cast(Reader.GetView(sizeof(CbAttachmentEntry) * ChunkCount).GetData()); - Reader.Skip(sizeof(CbAttachmentEntry) * ChunkCount); - - CbPackage Package; - - std::vector Attachments; - Attachments.reserve(ChunkCount); // Guessing here... - - tsl::robin_map PartialFileBuffers; - - std::vector> MalformedAttachments; - - for (uint32_t i = 0; i < ChunkCount; ++i) - { - const CbAttachmentEntry& Entry = AttachmentEntries[i]; - const uint64_t AttachmentSize = Entry.PayloadSize; - - if (Reader.Remaining() < AttachmentSize) - { - throw std::invalid_argument(fmt::format("invalid CbPackage, missing attachment data (need {} bytes, have {} bytes)", - AttachmentSize, - Reader.Remaining())); - } - const IoBuffer AttachmentBuffer(Payload, Reader.CurrentOffset(), AttachmentSize); - Reader.Skip(AttachmentSize); - - if (Entry.Flags & CbAttachmentEntry::kIsLocalRef) - { - // Marshal local reference - a "pointer" to the chunk backing file - - ZEN_ASSERT(AttachmentBuffer.Size() >= sizeof(CbAttachmentReferenceHeader)); - - const CbAttachmentReferenceHeader* AttachRefHdr = AttachmentBuffer.Data(); - const char* PathPointer = reinterpret_cast(AttachRefHdr + 1); - - ZEN_ASSERT(AttachmentBuffer.Size() >= (sizeof(CbAttachmentReferenceHeader) + AttachRefHdr->AbsolutePathLength)); - std::string_view PathView(PathPointer, AttachRefHdr->AbsolutePathLength); - - IoBuffer FullFileBuffer; - - std::filesystem::path Path(Utf8ToWide(PathView)); - if (auto It = PartialFileBuffers.find(Path.string()); It != PartialFileBuffers.end()) - { - FullFileBuffer = It->second; - } - else - { - if (PathView.starts_with(HandlePrefix)) - { -#if ZEN_PLATFORM_WINDOWS - std::string_view HandleString(PathView.substr(HandlePrefix.length())); - std::optional HandleNumber(ParseInt(HandleString)); - if (HandleNumber.has_value()) - { - HANDLE FileHandle = HANDLE(HandleNumber.value()); - ULARGE_INTEGER liFileSize; - liFileSize.LowPart = ::GetFileSize(FileHandle, &liFileSize.HighPart); - if (liFileSize.LowPart != INVALID_FILE_SIZE) - { - FullFileBuffer = - IoBuffer(IoBuffer::File, (void*)FileHandle, 0, uint64_t(liFileSize.QuadPart), /*IsWholeFile*/ true); - PartialFileBuffers.insert_or_assign(Path.string(), FullFileBuffer); - } - } -#else // ZEN_PLATFORM_WINDOWS - // Not supported on Linux/Mac. Could potentially use pidfd_getfd() but that requires a fairly new Linux kernel/includes - // and to deal with acceess rights etc. - ZEN_ASSERT(false); -#endif // ZEN_PLATFORM_WINDOWS - } - else - { - FullFileBuffer = PartialFileBuffers.insert_or_assign(Path.string(), IoBufferBuilder::MakeFromFile(Path)).first->second; - } - } - - if (FullFileBuffer) - { - IoBuffer ChunkReference = AttachRefHdr->PayloadByteOffset == 0 && AttachRefHdr->PayloadByteSize == FullFileBuffer.GetSize() - ? FullFileBuffer - : IoBuffer(FullFileBuffer, AttachRefHdr->PayloadByteOffset, AttachRefHdr->PayloadByteSize); - - CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkReference))); - if (CompBuf) - { - Attachments.emplace_back(CbAttachment(std::move(CompBuf), Entry.AttachmentHash)); - } - else - { - MalformedAttachments.push_back(std::make_pair(i, - fmt::format("Invalid format in '{}' (offset {}, size {}) for {}", - Path, - AttachRefHdr->PayloadByteOffset, - AttachRefHdr->PayloadByteSize, - Entry.AttachmentHash))); - } - } - else - { - MalformedAttachments.push_back(std::make_pair(i, - fmt::format("Unable to resolve chunk at '{}' (offset {}, size {}) for {}", - Path, - AttachRefHdr->PayloadByteOffset, - AttachRefHdr->PayloadByteSize, - Entry.AttachmentHash))); - } - } - else if (Entry.Flags & CbAttachmentEntry::kIsCompressed) - { - if (Entry.Flags & CbAttachmentEntry::kIsObject) - { - if (i == 0) - { - CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer))); - if (CompBuf) - { - Package.SetObject(LoadCompactBinaryObject(std::move(CompBuf))); - } - else - { - // First payload is always a compact binary object - MalformedAttachments.push_back( - std::make_pair(i, - fmt::format("Invalid format, expected compressed buffer for CbObject (size {}) for {}", - AttachmentBuffer.GetSize(), - Entry.AttachmentHash))); - } - } - else - { - MalformedAttachments.push_back(std::make_pair( - i, - fmt::format("Invalid format, compressed object attachments are not currently supported (size {}) for {}", - AttachmentBuffer.GetSize(), - Entry.AttachmentHash))); - } - } - else - { - CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer))); - if (CompBuf) - { - Attachments.emplace_back(CbAttachment(std::move(CompBuf), Entry.AttachmentHash)); - } - else - { - MalformedAttachments.push_back( - std::make_pair(i, - fmt::format("Invalid format, expected compressed buffer for attachment (size {}) for {}", - AttachmentBuffer.GetSize(), - Entry.AttachmentHash))); - } - } - } - else /* not compressed */ - { - if (Entry.Flags & CbAttachmentEntry::kIsObject) - { - if (i == 0) - { - Package.SetObject(LoadCompactBinaryObject(AttachmentBuffer)); - } - else - { - MalformedAttachments.push_back( - std::make_pair(i, - fmt::format("Invalid format, object attachments are not currently supported (size {}) for {}", - AttachmentBuffer.GetSize(), - Entry.AttachmentHash))); - } - } - else if (AttachmentSize > 0) - { - // Make a copy of the buffer so the attachments don't reference the entire payload - IoBuffer AttachmentBufferCopy = CreateBuffer(Entry.AttachmentHash, AttachmentSize); - ZEN_ASSERT(AttachmentBufferCopy); - ZEN_ASSERT(AttachmentBufferCopy.Size() == AttachmentSize); - AttachmentBufferCopy.GetMutableView().CopyFrom(AttachmentBuffer.GetView()); - - Attachments.emplace_back(SharedBuffer{AttachmentBufferCopy}); - } - else - { - MalformedAttachments.push_back( - std::make_pair(i, fmt::format("Invalid format, attachment of size zero detected for {}", Entry.AttachmentHash))); - } - } - } - PartialFileBuffers.clear(); - - Package.AddAttachments(Attachments); - - using namespace std::literals; - - if (!MalformedAttachments.empty()) - { - StringBuilder<1024> SB; - SB << (uint64_t)MalformedAttachments.size() << " malformed attachments in package message:\n"; - for (const auto& It : MalformedAttachments) - { - SB << " #"sv << It.first << ": " << It.second << "\n"; - } - ZEN_WARN("{}", SB.ToView()); - throw std::invalid_argument(SB.ToString()); - } - - return Package; -} - -bool -ParsePackageMessageWithLegacyFallback(const IoBuffer& Response, CbPackage& OutPackage) -{ - if (IsPackageMessage(Response)) - { - OutPackage = ParsePackageMessage(Response); - return true; - } - return OutPackage.TryLoad(Response); -} - -CbPackageReader::CbPackageReader() : m_CreateBuffer([](const IoHash&, uint64_t Size) -> IoBuffer { return IoBuffer{Size}; }) -{ -} - -CbPackageReader::~CbPackageReader() -{ -} - -void -CbPackageReader::SetPayloadBufferCreator(std::function CreateBuffer) -{ - m_CreateBuffer = CreateBuffer; -} - -uint64_t -CbPackageReader::ProcessPackageHeaderData(const void* Data, uint64_t DataBytes) -{ - ZEN_ASSERT(m_CurrentState != State::kReadingBuffers); - - switch (m_CurrentState) - { - case State::kInitialState: - ZEN_ASSERT(Data == nullptr); - m_CurrentState = State::kReadingHeader; - return sizeof m_PackageHeader; - - case State::kReadingHeader: - ZEN_ASSERT(DataBytes == sizeof m_PackageHeader); - memcpy(&m_PackageHeader, Data, sizeof m_PackageHeader); - ZEN_ASSERT(m_PackageHeader.HeaderMagic == kCbPkgMagic); - m_CurrentState = State::kReadingAttachmentEntries; - m_AttachmentEntries.resize(m_PackageHeader.AttachmentCount + 1); - return (m_PackageHeader.AttachmentCount + 1) * sizeof(CbAttachmentEntry); - - case State::kReadingAttachmentEntries: - ZEN_ASSERT(DataBytes == ((m_PackageHeader.AttachmentCount + 1) * sizeof(CbAttachmentEntry))); - memcpy(m_AttachmentEntries.data(), Data, DataBytes); - - for (CbAttachmentEntry& Entry : m_AttachmentEntries) - { - // This preallocates memory for payloads but note that for the local references - // the caller will need to handle the payload differently (i.e it's a - // CbAttachmentReferenceHeader not the actual payload) - - m_PayloadBuffers.emplace_back(IoBuffer{Entry.PayloadSize}); - } - - m_CurrentState = State::kReadingBuffers; - return 0; - - default: - ZEN_ASSERT(false); - return 0; - } -} - -IoBuffer -CbPackageReader::MarshalLocalChunkReference(IoBuffer AttachmentBuffer) -{ - // Marshal local reference - a "pointer" to the chunk backing file - - ZEN_ASSERT(AttachmentBuffer.Size() >= sizeof(CbAttachmentReferenceHeader)); - - const CbAttachmentReferenceHeader* AttachRefHdr = AttachmentBuffer.Data(); - const char8_t* PathPointer = reinterpret_cast(AttachRefHdr + 1); - - ZEN_ASSERT(AttachmentBuffer.Size() >= (sizeof(CbAttachmentReferenceHeader) + AttachRefHdr->AbsolutePathLength)); - - std::u8string_view PathView{PathPointer, AttachRefHdr->AbsolutePathLength}; - - std::filesystem::path Path{PathView}; - - IoBuffer ChunkReference = IoBufferBuilder::MakeFromFile(Path, AttachRefHdr->PayloadByteOffset, AttachRefHdr->PayloadByteSize); - - if (!ChunkReference) - { - // Unable to open chunk reference - - throw std::runtime_error(fmt::format("unable to resolve local reference to '{}' (offset {}, size {})", - PathToUtf8(Path), - AttachRefHdr->PayloadByteOffset, - AttachRefHdr->PayloadByteSize)); - } - - return ChunkReference; -}; - -void -CbPackageReader::Finalize() -{ - if (m_AttachmentEntries.empty()) - { - return; - } - - m_Attachments.reserve(m_AttachmentEntries.size() - 1); - - int CurrentAttachmentIndex = 0; - for (CbAttachmentEntry& Entry : m_AttachmentEntries) - { - IoBuffer AttachmentBuffer = m_PayloadBuffers[CurrentAttachmentIndex]; - - if (CurrentAttachmentIndex == 0) - { - // Root object - if (Entry.Flags & CbAttachmentEntry::kIsObject) - { - if (Entry.Flags & CbAttachmentEntry::kIsLocalRef) - { - m_RootObject = LoadCompactBinaryObject(MarshalLocalChunkReference(AttachmentBuffer)); - } - else if (Entry.Flags & CbAttachmentEntry::kIsCompressed) - { - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer), RawHash, RawSize); - if (RawHash == Entry.AttachmentHash) - { - m_RootObject = LoadCompactBinaryObject(Compressed); - } - } - else - { - m_RootObject = LoadCompactBinaryObject(std::move(AttachmentBuffer)); - } - } - else - { - throw std::runtime_error("missing or invalid root object"); - } - } - else if (Entry.Flags & CbAttachmentEntry::kIsLocalRef) - { - IoBuffer ChunkReference = MarshalLocalChunkReference(AttachmentBuffer); - - if (Entry.Flags & CbAttachmentEntry::kIsCompressed) - { - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(ChunkReference), RawHash, RawSize); - if (RawHash == Entry.AttachmentHash) - { - m_Attachments.emplace_back(CbAttachment(Compressed, Entry.AttachmentHash)); - } - } - else - { - CompressedBuffer Compressed = - CompressedBuffer::Compress(SharedBuffer(ChunkReference), OodleCompressor::NotSet, OodleCompressionLevel::None); - m_Attachments.emplace_back(CbAttachment(std::move(Compressed), Compressed.DecodeRawHash())); - } - } - - ++CurrentAttachmentIndex; - } -} - -/** - ______________________ _____________________________ - \__ ___/\_ _____// _____/\__ ___/ _____/ - | | | __)_ \_____ \ | | \_____ \ - | | | \/ \ | | / \ - |____| /_______ /_______ / |____| /_______ / - \/ \/ \/ - */ - -#if ZEN_WITH_TESTS - -TEST_CASE("CbPackage.Serialization") -{ - // Make a test package - - CbAttachment Attach1{SharedBuffer::MakeView(MakeMemoryView("abcd"))}; - CbAttachment Attach2{SharedBuffer::MakeView(MakeMemoryView("efgh"))}; - - CbObjectWriter Cbo; - Cbo.AddAttachment("abcd", Attach1); - Cbo.AddAttachment("efgh", Attach2); - - CbPackage Pkg; - Pkg.AddAttachment(Attach1); - Pkg.AddAttachment(Attach2); - Pkg.SetObject(Cbo.Save()); - - SharedBuffer Buffer = FormatPackageMessageBuffer(Pkg).Flatten(); - const uint8_t* CursorPtr = reinterpret_cast(Buffer.GetData()); - uint64_t RemainingBytes = Buffer.GetSize(); - - auto ConsumeBytes = [&](uint64_t ByteCount) { - ZEN_ASSERT(ByteCount <= RemainingBytes); - void* ReturnPtr = (void*)CursorPtr; - CursorPtr += ByteCount; - RemainingBytes -= ByteCount; - return ReturnPtr; - }; - - auto CopyBytes = [&](void* TargetBuffer, uint64_t ByteCount) { - ZEN_ASSERT(ByteCount <= RemainingBytes); - memcpy(TargetBuffer, CursorPtr, ByteCount); - CursorPtr += ByteCount; - RemainingBytes -= ByteCount; - }; - - CbPackageReader Reader; - uint64_t InitialRead = Reader.ProcessPackageHeaderData(nullptr, 0); - uint64_t NextBytes = Reader.ProcessPackageHeaderData(ConsumeBytes(InitialRead), InitialRead); - NextBytes = Reader.ProcessPackageHeaderData(ConsumeBytes(NextBytes), NextBytes); - auto Buffers = Reader.GetPayloadBuffers(); - - for (auto& PayloadBuffer : Buffers) - { - CopyBytes(PayloadBuffer.MutableData(), PayloadBuffer.GetSize()); - } - - Reader.Finalize(); -} - -TEST_CASE("CbPackage.EmptyObject") -{ - CbPackage Pkg; - Pkg.SetObject({}); - std::vector Result = FormatPackageMessage(Pkg, nullptr); -} - -TEST_CASE("CbPackage.LocalRef") -{ - ScopedTemporaryDirectory TempDir; - - auto Path1 = TempDir.Path() / "abcd"; - auto Path2 = TempDir.Path() / "efgh"; - - { - IoBuffer Buffer1 = IoBufferBuilder::MakeCloneFromMemory(MakeMemoryView("abcd")); - IoBuffer Buffer2 = IoBufferBuilder::MakeCloneFromMemory(MakeMemoryView("efgh")); - - WriteFile(Path1, Buffer1); - WriteFile(Path2, Buffer2); - } - - // Make a test package - - IoBuffer FileBuffer1 = IoBufferBuilder::MakeFromFile(Path1); - IoBuffer FileBuffer2 = IoBufferBuilder::MakeFromFile(Path2); - - CbAttachment Attach1{SharedBuffer(FileBuffer1)}; - CbAttachment Attach2{SharedBuffer(FileBuffer2)}; - - CbObjectWriter Cbo; - Cbo.AddAttachment("abcd", Attach1); - Cbo.AddAttachment("efgh", Attach2); - - CbPackage Pkg; - Pkg.AddAttachment(Attach1); - Pkg.AddAttachment(Attach2); - Pkg.SetObject(Cbo.Save()); - - SharedBuffer Buffer = FormatPackageMessageBuffer(Pkg, FormatFlags::kAllowLocalReferences).Flatten(); - const uint8_t* CursorPtr = reinterpret_cast(Buffer.GetData()); - uint64_t RemainingBytes = Buffer.GetSize(); - - auto ConsumeBytes = [&](uint64_t ByteCount) { - ZEN_ASSERT(ByteCount <= RemainingBytes); - void* ReturnPtr = (void*)CursorPtr; - CursorPtr += ByteCount; - RemainingBytes -= ByteCount; - return ReturnPtr; - }; - - auto CopyBytes = [&](void* TargetBuffer, uint64_t ByteCount) { - ZEN_ASSERT(ByteCount <= RemainingBytes); - memcpy(TargetBuffer, CursorPtr, ByteCount); - CursorPtr += ByteCount; - RemainingBytes -= ByteCount; - }; - - CbPackageReader Reader; - uint64_t InitialRead = Reader.ProcessPackageHeaderData(nullptr, 0); - uint64_t NextBytes = Reader.ProcessPackageHeaderData(ConsumeBytes(InitialRead), InitialRead); - NextBytes = Reader.ProcessPackageHeaderData(ConsumeBytes(NextBytes), NextBytes); - auto Buffers = Reader.GetPayloadBuffers(); - - for (auto& PayloadBuffer : Buffers) - { - CopyBytes(PayloadBuffer.MutableData(), PayloadBuffer.GetSize()); - } - - Reader.Finalize(); -} - -void -forcelink_packageformat() -{ -} - -#endif - -} // namespace zen -- cgit v1.2.3