From 55225621f018904abf7e212320bb784dc64f8ac3 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 18 Nov 2022 11:35:13 +0100 Subject: Add `import-project` and `export-project` (#183) * Add `import-project` and `export-project` command line parsing --- zenhttp/httpshared.cpp | 91 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 29 deletions(-) (limited to 'zenhttp/httpshared.cpp') diff --git a/zenhttp/httpshared.cpp b/zenhttp/httpshared.cpp index a7dca5441..e2f061a87 100644 --- a/zenhttp/httpshared.cpp +++ b/zenhttp/httpshared.cpp @@ -17,6 +17,10 @@ #include #include +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + namespace zen { std::vector @@ -93,10 +97,12 @@ FormatPackageMessage(const CbPackage& Data, FormatFlags Flags) ResponseBuffers.push_back(std::move(RefBuffer)); }; - auto IsLocalRef = [](const CompositeBuffer& AttachmentBinary, - bool DenyPartialLocalReferences, - CbAttachmentReferenceHeader& LocalRef, - std::string& Path8) -> bool { + tsl::robin_map FileNameMap; + + auto IsLocalRef = [&FileNameMap](const CompositeBuffer& AttachmentBinary, + bool DenyPartialLocalReferences, + CbAttachmentReferenceHeader& LocalRef, + std::string& Path8) -> bool { const SharedBuffer& Segment = AttachmentBinary.GetSegments().front(); IoBufferFileReference Ref; const IoBuffer& SegmentBuffer = Segment.AsIoBuffer(); @@ -111,9 +117,17 @@ FormatPackageMessage(const CbPackage& Data, FormatFlags Flags) return false; } - ExtendablePathBuilder<256> LocalRefFile; - LocalRefFile.Append(std::filesystem::absolute(PathFromHandle(Ref.FileHandle))); - Path8 = LocalRefFile.ToUtf8(); + if (auto It = FileNameMap.find(Ref.FileHandle); It != FileNameMap.end()) + { + Path8 = It->second; + } + else + { + ExtendablePathBuilder<256> LocalRefFile; + LocalRefFile.Append(std::filesystem::absolute(PathFromHandle(Ref.FileHandle))); + Path8 = LocalRefFile.ToUtf8(); + FileNameMap.insert_or_assign(Ref.FileHandle, Path8); + } LocalRef.AbsolutePathLength = gsl::narrow(Path8.size()); LocalRef.PayloadByteOffset = Ref.FileChunkOffset; @@ -131,7 +145,7 @@ FormatPackageMessage(const CbPackage& Data, FormatFlags Flags) else if (CompressedBuffer AttachmentBuffer = Attachment.AsCompressedBinary()) { CompositeBuffer Compressed = AttachmentBuffer.GetCompressed(); - IoHash AttachmentHash = IoHash::FromBLAKE3(AttachmentBuffer.GetRawHash()); + IoHash AttachmentHash = Attachment.GetHash(); // If the data is either not backed by a file, or there are multiple // fragments then we cannot marshal it by local reference. We might @@ -158,7 +172,7 @@ FormatPackageMessage(const CbPackage& Data, FormatFlags Flags) { *AttachmentInfo++ = {.PayloadSize = AttachmentBuffer.GetCompressedSize(), .Flags = CbAttachmentEntry::kIsCompressed, - .AttachmentHash = IoHash::FromBLAKE3(AttachmentBuffer.GetRawHash())}; + .AttachmentHash = AttachmentHash}; for (const SharedBuffer& Segment : Compressed.GetSegments()) { @@ -177,7 +191,7 @@ FormatPackageMessage(const CbPackage& Data, FormatFlags Flags) } else if (CompositeBuffer AttachmentBinary = Attachment.AsCompositeBinary()) { - IoHash AttachmentHash = IoHash::FromBLAKE3(AttachmentBuffer.GetRawHash()); + IoHash AttachmentHash = Attachment.GetHash(); bool MarshalByLocalRef = EnumHasAllFlags(Flags, FormatFlags::kAllowLocalReferences) && (AttachmentBinary.GetSegments().size() == 1); bool DenyPartialLocalReferences = EnumHasAllFlags(Flags, FormatFlags::kDenyPartialLocalReferences); @@ -211,6 +225,7 @@ FormatPackageMessage(const CbPackage& Data, FormatFlags Flags) ZEN_NOT_IMPLEMENTED("Unknown attachment kind"); } } + FileNameMap.clear(); return ResponseBuffers; } @@ -262,6 +277,11 @@ ParsePackageMessage(IoBuffer Payload, std::function Attachments; + Attachments.reserve(ChunkCount); // Guessing here... + + tsl::robin_map PartialFileBuffers; + for (uint32_t i = 0; i < ChunkCount; ++i) { const CbAttachmentEntry& Entry = AttachmentEntries[i]; @@ -283,22 +303,17 @@ ParsePackageMessage(IoBuffer Payload, std::functionAbsolutePathLength)}; - if (IoBuffer ChunkReference = - IoBufferBuilder::MakeFromFile(Path, AttachRefHdr->PayloadByteOffset, AttachRefHdr->PayloadByteSize)) + IoBuffer FullFileBuffer; + if (auto It = PartialFileBuffers.find(Path.string()); It != PartialFileBuffers.end()) { - CompressedBuffer CompBuf(CompressedBuffer::FromCompressed(SharedBuffer(ChunkReference))); - if (!CompBuf) - { - throw std::runtime_error(fmt::format("invalid format for chunk #{} at '{}' (offset {}, size {})", - i, - PathToUtf8(Path), - AttachRefHdr->PayloadByteOffset, - AttachRefHdr->PayloadByteSize)); - } - CbAttachment Attachment(std::move(CompBuf)); - Package.AddAttachment(Attachment); + FullFileBuffer = It->second; } else + { + FullFileBuffer = PartialFileBuffers.insert_or_assign(Path.string(), IoBufferBuilder::MakeFromFile(Path)).first->second; + } + + if (!FullFileBuffer) { // Unable to open chunk reference throw std::runtime_error(fmt::format("unable to resolve chunk #{} at '{}' (offset {}, size {})", @@ -307,6 +322,21 @@ ParsePackageMessage(IoBuffer Payload, std::functionPayloadByteOffset, AttachRefHdr->PayloadByteSize)); } + + IoBuffer ChunkReference = AttachRefHdr->PayloadByteOffset == 0 && AttachRefHdr->PayloadByteSize == FullFileBuffer.GetSize() + ? FullFileBuffer + : IoBuffer(FullFileBuffer, AttachRefHdr->PayloadByteOffset, AttachRefHdr->PayloadByteSize); + + CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkReference))); + if (!CompBuf) + { + throw std::runtime_error(fmt::format("invalid format for chunk #{} at '{}' (offset {}, size {})", + i, + PathToUtf8(Path), + AttachRefHdr->PayloadByteOffset, + AttachRefHdr->PayloadByteSize)); + } + Attachments.emplace_back(CbAttachment(std::move(CompBuf), Entry.AttachmentHash)); } else if (Entry.Flags & CbAttachmentEntry::kIsCompressed) { @@ -341,8 +371,7 @@ ParsePackageMessage(IoBuffer Payload, std::function