diff options
| -rw-r--r-- | CHANGELOG.md | 1 | ||||
| -rw-r--r-- | src/zencore/compress.cpp | 39 |
2 files changed, 37 insertions, 3 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c6413801..4d80ee543 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Improvement: Check available disk space more frequently and trigger GC operation if low water mark is reached - Improvement: Faster oplog validate to reduce GC wall time and disk I/O pressure - Improvement: `zen builds upload` now use the system temp directory for temporary files leaving the source folder untouched +- Improvement: NoneDecoder::DecompressToStream and NoneDecoder::CompressToStream not uses direct disk I/O ## 5.6.8 - Feature: Add per bucket cache configuration (Lua options file only) diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp index ad6b6103c..62b64bc9d 100644 --- a/src/zencore/compress.cpp +++ b/src/zencore/compress.cpp @@ -7,6 +7,7 @@ #include <zencore/compositebuffer.h> #include <zencore/crc32.h> #include <zencore/endian.h> +#include <zencore/filesystem.h> #include <zencore/intmath.h> #include <zencore/iohash.h> #include <zencore/stream.h> @@ -217,6 +218,20 @@ public: { UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), BLAKE3::HashBuffer(RawData)); Callback(0, 0, 0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderData.GetData(), HeaderData.GetSize()))); + + IoBufferFileReference FileRef = {nullptr, 0, 0}; + if ((RawData.GetSegments().size() == 1) && RawData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef)) + { + ZEN_ASSERT(FileRef.FileHandle != nullptr); + uint64_t CallbackOffset = 0; + ScanFile(FileRef.FileHandle, 0, RawData.GetSize(), 512u * 1024u, [&](const void* Data, size_t Size) { + CompositeBuffer Tmp(SharedBuffer(IoBuffer(IoBuffer::Wrap, Data, Size))); + Callback(CallbackOffset, Size, HeaderData.GetSize() + CallbackOffset, Tmp); + CallbackOffset += Size; + }); + return true; + } + Callback(0, RawData.GetSize(), HeaderData.GetSize(), RawData); return true; } @@ -299,11 +314,29 @@ public: Header.TotalCompressedSize == Header.TotalRawSize + sizeof(BufferHeader) && RawOffset < Header.TotalRawSize && (RawOffset + RawSize) <= Header.TotalRawSize) { - if (!Callback(sizeof(BufferHeader) + RawOffset, RawSize, 0, CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize))) + bool Result = true; + IoBufferFileReference FileRef = {nullptr, 0, 0}; + if ((CompressedData.GetSegments().size() == 1) && CompressedData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef)) { - return false; + ZEN_ASSERT(FileRef.FileHandle != nullptr); + uint64_t CallbackOffset = 0; + ScanFile(FileRef.FileHandle, sizeof(BufferHeader) + RawOffset, RawSize, 512u * 1024u, [&](const void* Data, size_t Size) { + if (Result) + { + CompositeBuffer Tmp(SharedBuffer(IoBuffer(IoBuffer::Wrap, Data, Size))); + Result = Callback(sizeof(BufferHeader) + RawOffset + CallbackOffset, Size, CallbackOffset, Tmp); + } + CallbackOffset += Size; + }); + return Result; + } + else + { + return Callback(sizeof(BufferHeader) + RawOffset, + RawSize, + 0, + CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize)); } - return true; } return false; } |