diff options
| author | Devin Doucette <[email protected]> | 2021-07-22 03:05:09 -0600 |
|---|---|---|
| committer | GitHub <[email protected]> | 2021-07-22 11:05:09 +0200 |
| commit | c49d575726a1d62a1adaf50f6a6341e70abb733e (patch) | |
| tree | e562f08b57b5d89ced3b08896d423760b60c63df /zencore/compress.cpp | |
| parent | Ignore tags folder created by ctags et al. (diff) | |
| download | zen-c49d575726a1d62a1adaf50f6a6341e70abb733e.tar.xz zen-c49d575726a1d62a1adaf50f6a6341e70abb733e.zip | |
Added Oodle to CompressedBuffer (#5)
Diffstat (limited to 'zencore/compress.cpp')
| -rw-r--r-- | zencore/compress.cpp | 304 |
1 files changed, 221 insertions, 83 deletions
diff --git a/zencore/compress.cpp b/zencore/compress.cpp index 805e962cd..ba62e7de2 100644 --- a/zencore/compress.cpp +++ b/zencore/compress.cpp @@ -7,6 +7,9 @@ #include <zencore/crc32.h> #include <zencore/endian.h> +#include "../3rdparty/Oodle/include/oodle2.h" +#pragma comment(lib, "oo2core_win64.lib") + #include <doctest/doctest.h> #include <lz4.h> #include <functional> @@ -19,6 +22,19 @@ static constexpr uint64_t DefaultBlockSize = 256 * 1024; /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +/** Method used to compress the data in a compressed buffer. */ +enum class CompressionMethod : uint8_t +{ + /** Header is followed by one uncompressed block. */ + None = 0, + /** Header is followed by an array of compressed block sizes then the compressed blocks. */ + Oodle = 3, + /** Header is followed by an array of compressed block sizes then the compressed blocks. */ + LZ4 = 4, +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /** Header used on every compressed buffer. Always stored in big-endian format. */ struct BufferHeader { @@ -28,7 +44,8 @@ struct BufferHeader uint32_t Crc32 = 0; // A CRC-32 used to check integrity of the buffer. Uses the polynomial 0x04c11db7 CompressionMethod Method = CompressionMethod::None; // The method used to compress the buffer. Affects layout of data following the header - uint8_t Reserved[2]{}; // The reserved bytes must be initialized to zero + uint8_t Compressor = 0; // The method-specific compressor used to compress the buffer. + uint8_t CompressionLevel = 0; // The method-specific compression level used to compress the buffer. uint8_t BlockSizeExponent = 0; // The power of two size of every uncompressed block except the last. Size is 1 << BlockSizeExponent uint32_t BlockCount = 0; // The number of blocks that follow the header uint64_t TotalRawSize = 0; // The total size of the uncompressed data @@ -95,10 +112,15 @@ static_assert(sizeof(BufferHeader) == 64, "BufferHeader is the wrong size."); /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CompressionMethodBase +class BaseEncoder +{ +public: + virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const = 0; +}; + +class BaseDecoder { public: - virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const = 0; virtual CompositeBuffer Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const = 0; virtual bool TryDecompressTo(const BufferHeader& Header, const CompositeBuffer& CompressedData, MutableMemoryView RawView) const = 0; virtual uint64_t GetHeaderSize(const BufferHeader& Header) const = 0; @@ -106,7 +128,7 @@ public: /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class MethodNone final : public CompressionMethodBase +class NoneEncoder final : public BaseEncoder { public: [[nodiscard]] CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t /* BlockSize */) const final @@ -122,7 +144,11 @@ public: Header.Write(HeaderData); return CompositeBuffer(HeaderData.MoveToShared(), RawData.MakeOwned()); } +}; +class NoneDecoder final : public BaseDecoder +{ +public: [[nodiscard]] CompositeBuffer Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const final { if (Header.Method == CompressionMethod::None && Header.TotalCompressedSize == CompressedData.GetSize() && @@ -152,24 +178,17 @@ public: ////////////////////////////////////////////////////////////////////////// -class MethodBlockBase : public CompressionMethodBase +class BlockEncoder : public BaseEncoder { public: - CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const final; - CompositeBuffer Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const final; - [[nodiscard]] bool TryDecompressTo(const BufferHeader& Header, - const CompositeBuffer& CompressedData, - MutableMemoryView RawView) const final; - [[nodiscard]] uint64_t GetHeaderSize(const BufferHeader& Header) const final - { - return sizeof(BufferHeader) + sizeof(uint32_t) * uint64_t(Header.BlockCount); - } + CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const final; protected: - virtual CompressionMethod GetMethod() const = 0; - virtual uint64_t CompressBlockBound(uint64_t RawSize) const = 0; - virtual bool CompressBlock(MutableMemoryView& CompressedData, MemoryView RawData) const = 0; - virtual bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const = 0; + virtual CompressionMethod GetMethod() const = 0; + virtual uint8_t GetCompressor() const = 0; + virtual uint8_t GetCompressionLevel() const = 0; + virtual uint64_t CompressBlockBound(uint64_t RawSize) const = 0; + virtual bool CompressBlock(MutableMemoryView& CompressedData, MemoryView RawData) const = 0; private: uint64_t GetCompressedBlocksBound(uint64_t BlockCount, uint64_t BlockSize, uint64_t RawSize) const @@ -187,7 +206,7 @@ private: }; CompositeBuffer -MethodBlockBase::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize) const +BlockEncoder::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize) const { ZEN_ASSERT(IsPow2(BlockSize) && BlockSize <= (1 << 31)); @@ -242,7 +261,7 @@ MethodBlockBase::Compress(const CompositeBuffer& RawData, const uint64_t BlockSi if (RawSize <= MetaSize + CompressedSize) { CompressedData.Reset(); - return MethodNone().Compress(RawData, BlockSize); + return NoneEncoder().Compress(RawData, BlockSize); } // Write the header and calculate the CRC-32. @@ -254,6 +273,8 @@ MethodBlockBase::Compress(const CompositeBuffer& RawData, const uint64_t BlockSi BufferHeader Header; Header.Method = GetMethod(); + Header.Compressor = GetCompressor(); + Header.CompressionLevel = GetCompressionLevel(); Header.BlockSizeExponent = static_cast<uint8_t>(zen::FloorLog2_64(BlockSize)); Header.BlockCount = static_cast<uint32_t>(BlockCount); Header.TotalRawSize = RawSize; @@ -265,8 +286,24 @@ MethodBlockBase::Compress(const CompositeBuffer& RawData, const uint64_t BlockSi return CompositeBuffer(SharedBuffer::MakeView(CompositeView, CompressedData.MoveToShared())); } +class BlockDecoder : public BaseDecoder +{ +public: + CompositeBuffer Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const final; + [[nodiscard]] bool TryDecompressTo(const BufferHeader& Header, + const CompositeBuffer& CompressedData, + MutableMemoryView RawView) const final; + [[nodiscard]] uint64_t GetHeaderSize(const BufferHeader& Header) const final + { + return sizeof(BufferHeader) + sizeof(uint32_t) * uint64_t(Header.BlockCount); + } + +protected: + virtual bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const = 0; +}; + CompositeBuffer -MethodBlockBase::Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const +BlockDecoder::Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const { if (Header.BlockCount == 0 || Header.TotalCompressedSize != CompressedData.GetSize()) { @@ -383,7 +420,7 @@ MethodBlockBase::Decompress(const BufferHeader& Header, const CompositeBuffer& C } bool -MethodBlockBase::TryDecompressTo(const BufferHeader& Header, const CompositeBuffer& CompressedData, MutableMemoryView RawView) const +BlockDecoder::TryDecompressTo(const BufferHeader& Header, const CompositeBuffer& CompressedData, MutableMemoryView RawView) const { if (Header.TotalRawSize != RawView.GetSize() || Header.TotalCompressedSize != CompressedData.GetSize()) { @@ -436,35 +473,115 @@ MethodBlockBase::TryDecompressTo(const BufferHeader& Header, const CompositeBuff ////////////////////////////////////////////////////////////////////////// -class MethodLZ4 final : public MethodBlockBase +struct OodleInit { + OodleInit() + { + OodleConfigValues Config; + Oodle_GetConfigValues(&Config); + // Always read/write Oodle v9 binary data. + Config.m_OodleLZ_BackwardsCompatible_MajorVersion = 9; + Oodle_SetConfigValues(&Config); + } +}; + +OodleInit InitOodle; + +class OodleEncoder final : public BlockEncoder +{ +public: + OodleEncoder(OodleCompressor InCompressor, OodleCompressionLevel InCompressionLevel) + : Compressor(InCompressor) + , CompressionLevel(InCompressionLevel) + { + } + protected: - CompressionMethod GetMethod() const final { return CompressionMethod::LZ4; } + CompressionMethod GetMethod() const final { return CompressionMethod::Oodle; } + uint8_t GetCompressor() const final { return static_cast<uint8_t>(Compressor); } + uint8_t GetCompressionLevel() const final { return static_cast<uint8_t>(CompressionLevel); } uint64_t CompressBlockBound(uint64_t RawSize) const final { - if (RawSize <= LZ4_MAX_INPUT_SIZE) + return static_cast<uint64_t>(OodleLZ_GetCompressedBufferSizeNeeded(OodleLZ_Compressor_Kraken, static_cast<OO_SINTa>(RawSize))); + } + + bool CompressBlock(MutableMemoryView& CompressedData, MemoryView RawData) const final + { + const OodleLZ_Compressor LZCompressor = GetOodleLZCompressor(Compressor); + const OodleLZ_CompressionLevel LZCompressionLevel = GetOodleLZCompressionLevel(CompressionLevel); + if (LZCompressor == OodleLZ_Compressor_Invalid || LZCompressionLevel == OodleLZ_CompressionLevel_Invalid || + LZCompressionLevel == OodleLZ_CompressionLevel_None) { - return static_cast<uint64_t>(LZ4_compressBound(static_cast<int>(RawSize))); + return false; } - return 0; + + const OO_SINTa RawSize = static_cast<OO_SINTa>(RawData.GetSize()); + if (static_cast<OO_SINTa>(CompressedData.GetSize()) < OodleLZ_GetCompressedBufferSizeNeeded(LZCompressor, RawSize)) + { + return false; + } + + const OO_SINTa Size = OodleLZ_Compress(LZCompressor, RawData.GetData(), RawSize, CompressedData.GetData(), LZCompressionLevel); + CompressedData.LeftInline(static_cast<uint64_t>(Size)); + return Size > 0; } - bool CompressBlock(MutableMemoryView& CompressedData, MemoryView RawData) const final + static OodleLZ_Compressor GetOodleLZCompressor(OodleCompressor Compressor) { - if (RawData.GetSize() <= LZ4_MAX_INPUT_SIZE) + switch (Compressor) { - const int Size = - LZ4_compress_default(static_cast<const char*>(RawData.GetData()), - static_cast<char*>(CompressedData.GetData()), - static_cast<int>(RawData.GetSize()), - static_cast<int>(zen::Min<uint64_t>(CompressedData.GetSize(), std::numeric_limits<int>::max()))); - CompressedData.LeftInline(static_cast<uint64_t>(Size)); - return Size > 0; + case OodleCompressor::Selkie: + return OodleLZ_Compressor_Selkie; + case OodleCompressor::Mermaid: + return OodleLZ_Compressor_Mermaid; + case OodleCompressor::Kraken: + return OodleLZ_Compressor_Kraken; + case OodleCompressor::Leviathan: + return OodleLZ_Compressor_Leviathan; + case OodleCompressor::NotSet: + default: + return OodleLZ_Compressor_Invalid; } - return false; } + static OodleLZ_CompressionLevel GetOodleLZCompressionLevel(OodleCompressionLevel Level) + { + const int IntLevel = (int)Level; + if (IntLevel < (int)OodleLZ_CompressionLevel_Min || IntLevel > (int)OodleLZ_CompressionLevel_Max) + { + return OodleLZ_CompressionLevel_Invalid; + } + return OodleLZ_CompressionLevel(IntLevel); + } + +private: + const OodleCompressor Compressor; + const OodleCompressionLevel CompressionLevel; +}; + +class OodleDecoder final : public BlockDecoder +{ +protected: + bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const final + { + const OO_SINTa RawSize = static_cast<OO_SINTa>(RawData.GetSize()); + const OO_SINTa Size = OodleLZ_Decompress(CompressedData.GetData(), + static_cast<OO_SINTa>(CompressedData.GetSize()), + RawData.GetData(), + RawSize, + OodleLZ_FuzzSafe_Yes, + OodleLZ_CheckCRC_Yes, + OodleLZ_Verbosity_None); + return Size == RawSize; + } +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +class LZ4Decoder final : public BlockDecoder +{ +protected: bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const final { if (CompressedData.GetSize() <= std::numeric_limits<int>::max()) @@ -481,11 +598,12 @@ protected: ////////////////////////////////////////////////////////////////////////// -static const CompressionMethodBase* -GetMethod(CompressionMethod Method) +static const BaseDecoder* +GetDecoder(CompressionMethod Method) { - static MethodNone None; - static MethodLZ4 LZ4; + static NoneDecoder None; + static OodleDecoder Oodle; + static LZ4Decoder LZ4; switch (Method) { @@ -493,25 +611,13 @@ GetMethod(CompressionMethod Method) return nullptr; case CompressionMethod::None: return &None; + case CompressionMethod::Oodle: + return &Oodle; case CompressionMethod::LZ4: return &LZ4; } } -static const char* -GetMethodName(CompressionMethod Method) -{ - switch (Method) - { - default: - return "error"; - case CompressionMethod::None: - return "None"; - case CompressionMethod::LZ4: - return "LZ4"; - } -} - ////////////////////////////////////////////////////////////////////////// bool @@ -522,10 +628,10 @@ BufferHeader::IsValid(const CompositeBuffer& CompressedData) const BufferHeader Header = Read(CompressedData); if (Header.Magic == BufferHeader::ExpectedMagic) { - if (const CompressionMethodBase* const Method = GetMethod(Header.Method)) + if (const BaseDecoder* const Decoder = GetDecoder(Header.Method)) { UniqueBuffer HeaderCopy; - const MemoryView HeaderView = CompressedData.ViewOrCopyRange(0, Method->GetHeaderSize(Header), HeaderCopy); + const MemoryView HeaderView = CompressedData.ViewOrCopyRange(0, Decoder->GetHeaderSize(Header), HeaderCopy); if (Header.Crc32 == BufferHeader::CalculateCrc32(HeaderView)) { return true; @@ -550,22 +656,37 @@ ValidBufferOrEmpty(BufferType&& CompressedData) namespace zen { CompressedBuffer -CompressedBuffer::Compress(CompressionMethod Method, const CompositeBuffer& RawData) +CompressedBuffer::Compress(const CompositeBuffer& RawData, + OodleCompressor Compressor, + OodleCompressionLevel CompressionLevel, + uint64_t BlockSize) { using namespace detail; + if (BlockSize == 0) + { + BlockSize = DefaultBlockSize; + } + CompressedBuffer Local; - if (const CompressionMethodBase* const Impl = GetMethod(Method)) + if (CompressionLevel == OodleCompressionLevel::None) { - Local.CompressedData = Impl->Compress(RawData); + Local.CompressedData = NoneEncoder().Compress(RawData, BlockSize); + } + else + { + Local.CompressedData = OodleEncoder(Compressor, CompressionLevel).Compress(RawData, BlockSize); } return Local; } CompressedBuffer -CompressedBuffer::Compress(CompressionMethod Method, const SharedBuffer& RawData) +CompressedBuffer::Compress(const SharedBuffer& RawData, + OodleCompressor Compressor, + OodleCompressionLevel CompressionLevel, + uint64_t BlockSize) { - return Compress(Method, CompositeBuffer(RawData)); + return Compress(CompositeBuffer(RawData), Compressor, CompressionLevel, BlockSize); } CompressedBuffer @@ -619,9 +740,9 @@ CompressedBuffer::TryDecompressTo(MutableMemoryView RawView) const if (CompressedData) { const BufferHeader Header = BufferHeader::Read(CompressedData); - if (const CompressionMethodBase* const Method = GetMethod(Header.Method)) + if (const BaseDecoder* const Decoder = GetDecoder(Header.Method)) { - return Method->TryDecompressTo(Header, CompressedData, RawView); + return Decoder->TryDecompressTo(Header, CompressedData, RawView); } } return false; @@ -634,14 +755,14 @@ CompressedBuffer::Decompress() const if (CompressedData) { const BufferHeader Header = BufferHeader::Read(CompressedData); - if (const CompressionMethodBase* const Method = GetMethod(Header.Method)) + if (const BaseDecoder* const Decoder = GetDecoder(Header.Method)) { if (Header.Method == CompressionMethod::None) { - return Method->Decompress(Header, CompressedData).Flatten(); + return Decoder->Decompress(Header, CompressedData).Flatten(); } UniqueBuffer RawData = UniqueBuffer::Alloc(Header.TotalRawSize); - if (Method->TryDecompressTo(Header, CompressedData, RawData)) + if (Decoder->TryDecompressTo(Header, CompressedData, RawData)) { return RawData.MoveToShared(); } @@ -657,18 +778,35 @@ CompressedBuffer::DecompressToComposite() const if (CompressedData) { const BufferHeader Header = BufferHeader::Read(CompressedData); - if (const CompressionMethodBase* const Method = GetMethod(Header.Method)) + if (const BaseDecoder* const Decoder = GetDecoder(Header.Method)) { - return Method->Decompress(Header, CompressedData); + return Decoder->Decompress(Header, CompressedData); } } return CompositeBuffer(); } -const char* -CompressedBuffer::GetFormatName() const +bool +CompressedBuffer::TryGetCompressParameters(OodleCompressor& OutCompressor, OodleCompressionLevel& OutCompressionLevel) const { - return detail::GetMethodName(CompressedData ? detail::BufferHeader::Read(CompressedData).Method : CompressionMethod::None); + using namespace detail; + if (CompressedData) + { + switch (const BufferHeader Header = BufferHeader::Read(CompressedData); Header.Method) + { + case CompressionMethod::None: + OutCompressor = OodleCompressor::NotSet; + OutCompressionLevel = OodleCompressionLevel::None; + return true; + case CompressionMethod::Oodle: + OutCompressor = OodleCompressor(Header.Compressor); + OutCompressionLevel = OodleCompressionLevel(Header.CompressionLevel); + return true; + default: + break; + } + } + return false; } /** @@ -682,13 +820,14 @@ CompressedBuffer::GetFormatName() const TEST_CASE("CompressedBuffer") { - uint8_t Zeroes[256]{}; - uint8_t Ones[256]; + uint8_t Zeroes[1024]{}; + uint8_t Ones[1024]; memset(Ones, 1, sizeof Ones); { - CompressedBuffer Buffer = - CompressedBuffer::Compress(CompressionMethod::None, CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)))); + CompressedBuffer Buffer = CompressedBuffer::Compress(CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes))), + OodleCompressor::NotSet, + OodleCompressionLevel::None); CHECK(Buffer.GetRawSize() == sizeof(Zeroes)); CHECK(Buffer.GetCompressedSize() == (sizeof(Zeroes) + sizeof(detail::BufferHeader))); @@ -706,8 +845,9 @@ TEST_CASE("CompressedBuffer") { CompressedBuffer Buffer = CompressedBuffer::Compress( - CompressionMethod::None, - CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)), SharedBuffer::MakeView(MakeMemoryView(Ones)))); + CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)), SharedBuffer::MakeView(MakeMemoryView(Ones))), + OodleCompressor::NotSet, + OodleCompressionLevel::None); CHECK(Buffer.GetRawSize() == (sizeof(Zeroes) + sizeof(Ones))); CHECK(Buffer.GetCompressedSize() == (sizeof(Zeroes) + sizeof(Ones) + sizeof(detail::BufferHeader))); @@ -724,11 +864,10 @@ TEST_CASE("CompressedBuffer") } { - CompressedBuffer Buffer = - CompressedBuffer::Compress(CompressionMethod::LZ4, CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)))); + CompressedBuffer Buffer = CompressedBuffer::Compress(CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)))); CHECK(Buffer.GetRawSize() == sizeof(Zeroes)); - CHECK(Buffer.GetCompressedSize() == (15 + sizeof(detail::BufferHeader))); + CHECK(Buffer.GetCompressedSize() < sizeof(Zeroes)); CompositeBuffer Compressed = Buffer.GetCompressed(); CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed); @@ -743,11 +882,10 @@ TEST_CASE("CompressedBuffer") { CompressedBuffer Buffer = CompressedBuffer::Compress( - CompressionMethod::LZ4, CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)), SharedBuffer::MakeView(MakeMemoryView(Ones)))); CHECK(Buffer.GetRawSize() == (sizeof(Zeroes) + sizeof(Ones))); - CHECK(Buffer.GetCompressedSize() == (20 + sizeof(detail::BufferHeader))); + CHECK(Buffer.GetCompressedSize() < (sizeof(Zeroes) + sizeof(Ones))); CompositeBuffer Compressed = Buffer.GetCompressed(); CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed); |