aboutsummaryrefslogtreecommitdiff
path: root/zencore/compress.cpp
diff options
context:
space:
mode:
authorDevin Doucette <[email protected]>2021-07-22 03:05:09 -0600
committerGitHub <[email protected]>2021-07-22 11:05:09 +0200
commitc49d575726a1d62a1adaf50f6a6341e70abb733e (patch)
treee562f08b57b5d89ced3b08896d423760b60c63df /zencore/compress.cpp
parentIgnore tags folder created by ctags et al. (diff)
downloadzen-c49d575726a1d62a1adaf50f6a6341e70abb733e.tar.xz
zen-c49d575726a1d62a1adaf50f6a6341e70abb733e.zip
Added Oodle to CompressedBuffer (#5)
Diffstat (limited to 'zencore/compress.cpp')
-rw-r--r--zencore/compress.cpp304
1 files changed, 221 insertions, 83 deletions
diff --git a/zencore/compress.cpp b/zencore/compress.cpp
index 805e962cd..ba62e7de2 100644
--- a/zencore/compress.cpp
+++ b/zencore/compress.cpp
@@ -7,6 +7,9 @@
#include <zencore/crc32.h>
#include <zencore/endian.h>
+#include "../3rdparty/Oodle/include/oodle2.h"
+#pragma comment(lib, "oo2core_win64.lib")
+
#include <doctest/doctest.h>
#include <lz4.h>
#include <functional>
@@ -19,6 +22,19 @@ static constexpr uint64_t DefaultBlockSize = 256 * 1024;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+/** Method used to compress the data in a compressed buffer. */
+enum class CompressionMethod : uint8_t
+{
+ /** Header is followed by one uncompressed block. */
+ None = 0,
+ /** Header is followed by an array of compressed block sizes then the compressed blocks. */
+ Oodle = 3,
+ /** Header is followed by an array of compressed block sizes then the compressed blocks. */
+ LZ4 = 4,
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
/** Header used on every compressed buffer. Always stored in big-endian format. */
struct BufferHeader
{
@@ -28,7 +44,8 @@ struct BufferHeader
uint32_t Crc32 = 0; // A CRC-32 used to check integrity of the buffer. Uses the polynomial 0x04c11db7
CompressionMethod Method =
CompressionMethod::None; // The method used to compress the buffer. Affects layout of data following the header
- uint8_t Reserved[2]{}; // The reserved bytes must be initialized to zero
+ uint8_t Compressor = 0; // The method-specific compressor used to compress the buffer.
+ uint8_t CompressionLevel = 0; // The method-specific compression level used to compress the buffer.
uint8_t BlockSizeExponent = 0; // The power of two size of every uncompressed block except the last. Size is 1 << BlockSizeExponent
uint32_t BlockCount = 0; // The number of blocks that follow the header
uint64_t TotalRawSize = 0; // The total size of the uncompressed data
@@ -95,10 +112,15 @@ static_assert(sizeof(BufferHeader) == 64, "BufferHeader is the wrong size.");
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class CompressionMethodBase
+class BaseEncoder
+{
+public:
+ virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const = 0;
+};
+
+class BaseDecoder
{
public:
- virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const = 0;
virtual CompositeBuffer Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const = 0;
virtual bool TryDecompressTo(const BufferHeader& Header, const CompositeBuffer& CompressedData, MutableMemoryView RawView) const = 0;
virtual uint64_t GetHeaderSize(const BufferHeader& Header) const = 0;
@@ -106,7 +128,7 @@ public:
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-class MethodNone final : public CompressionMethodBase
+class NoneEncoder final : public BaseEncoder
{
public:
[[nodiscard]] CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t /* BlockSize */) const final
@@ -122,7 +144,11 @@ public:
Header.Write(HeaderData);
return CompositeBuffer(HeaderData.MoveToShared(), RawData.MakeOwned());
}
+};
+class NoneDecoder final : public BaseDecoder
+{
+public:
[[nodiscard]] CompositeBuffer Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const final
{
if (Header.Method == CompressionMethod::None && Header.TotalCompressedSize == CompressedData.GetSize() &&
@@ -152,24 +178,17 @@ public:
//////////////////////////////////////////////////////////////////////////
-class MethodBlockBase : public CompressionMethodBase
+class BlockEncoder : public BaseEncoder
{
public:
- CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const final;
- CompositeBuffer Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const final;
- [[nodiscard]] bool TryDecompressTo(const BufferHeader& Header,
- const CompositeBuffer& CompressedData,
- MutableMemoryView RawView) const final;
- [[nodiscard]] uint64_t GetHeaderSize(const BufferHeader& Header) const final
- {
- return sizeof(BufferHeader) + sizeof(uint32_t) * uint64_t(Header.BlockCount);
- }
+ CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const final;
protected:
- virtual CompressionMethod GetMethod() const = 0;
- virtual uint64_t CompressBlockBound(uint64_t RawSize) const = 0;
- virtual bool CompressBlock(MutableMemoryView& CompressedData, MemoryView RawData) const = 0;
- virtual bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const = 0;
+ virtual CompressionMethod GetMethod() const = 0;
+ virtual uint8_t GetCompressor() const = 0;
+ virtual uint8_t GetCompressionLevel() const = 0;
+ virtual uint64_t CompressBlockBound(uint64_t RawSize) const = 0;
+ virtual bool CompressBlock(MutableMemoryView& CompressedData, MemoryView RawData) const = 0;
private:
uint64_t GetCompressedBlocksBound(uint64_t BlockCount, uint64_t BlockSize, uint64_t RawSize) const
@@ -187,7 +206,7 @@ private:
};
CompositeBuffer
-MethodBlockBase::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize) const
+BlockEncoder::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize) const
{
ZEN_ASSERT(IsPow2(BlockSize) && BlockSize <= (1 << 31));
@@ -242,7 +261,7 @@ MethodBlockBase::Compress(const CompositeBuffer& RawData, const uint64_t BlockSi
if (RawSize <= MetaSize + CompressedSize)
{
CompressedData.Reset();
- return MethodNone().Compress(RawData, BlockSize);
+ return NoneEncoder().Compress(RawData, BlockSize);
}
// Write the header and calculate the CRC-32.
@@ -254,6 +273,8 @@ MethodBlockBase::Compress(const CompositeBuffer& RawData, const uint64_t BlockSi
BufferHeader Header;
Header.Method = GetMethod();
+ Header.Compressor = GetCompressor();
+ Header.CompressionLevel = GetCompressionLevel();
Header.BlockSizeExponent = static_cast<uint8_t>(zen::FloorLog2_64(BlockSize));
Header.BlockCount = static_cast<uint32_t>(BlockCount);
Header.TotalRawSize = RawSize;
@@ -265,8 +286,24 @@ MethodBlockBase::Compress(const CompositeBuffer& RawData, const uint64_t BlockSi
return CompositeBuffer(SharedBuffer::MakeView(CompositeView, CompressedData.MoveToShared()));
}
+class BlockDecoder : public BaseDecoder
+{
+public:
+ CompositeBuffer Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const final;
+ [[nodiscard]] bool TryDecompressTo(const BufferHeader& Header,
+ const CompositeBuffer& CompressedData,
+ MutableMemoryView RawView) const final;
+ [[nodiscard]] uint64_t GetHeaderSize(const BufferHeader& Header) const final
+ {
+ return sizeof(BufferHeader) + sizeof(uint32_t) * uint64_t(Header.BlockCount);
+ }
+
+protected:
+ virtual bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const = 0;
+};
+
CompositeBuffer
-MethodBlockBase::Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const
+BlockDecoder::Decompress(const BufferHeader& Header, const CompositeBuffer& CompressedData) const
{
if (Header.BlockCount == 0 || Header.TotalCompressedSize != CompressedData.GetSize())
{
@@ -383,7 +420,7 @@ MethodBlockBase::Decompress(const BufferHeader& Header, const CompositeBuffer& C
}
bool
-MethodBlockBase::TryDecompressTo(const BufferHeader& Header, const CompositeBuffer& CompressedData, MutableMemoryView RawView) const
+BlockDecoder::TryDecompressTo(const BufferHeader& Header, const CompositeBuffer& CompressedData, MutableMemoryView RawView) const
{
if (Header.TotalRawSize != RawView.GetSize() || Header.TotalCompressedSize != CompressedData.GetSize())
{
@@ -436,35 +473,115 @@ MethodBlockBase::TryDecompressTo(const BufferHeader& Header, const CompositeBuff
//////////////////////////////////////////////////////////////////////////
-class MethodLZ4 final : public MethodBlockBase
+struct OodleInit
{
+ OodleInit()
+ {
+ OodleConfigValues Config;
+ Oodle_GetConfigValues(&Config);
+ // Always read/write Oodle v9 binary data.
+ Config.m_OodleLZ_BackwardsCompatible_MajorVersion = 9;
+ Oodle_SetConfigValues(&Config);
+ }
+};
+
+OodleInit InitOodle;
+
+class OodleEncoder final : public BlockEncoder
+{
+public:
+ OodleEncoder(OodleCompressor InCompressor, OodleCompressionLevel InCompressionLevel)
+ : Compressor(InCompressor)
+ , CompressionLevel(InCompressionLevel)
+ {
+ }
+
protected:
- CompressionMethod GetMethod() const final { return CompressionMethod::LZ4; }
+ CompressionMethod GetMethod() const final { return CompressionMethod::Oodle; }
+ uint8_t GetCompressor() const final { return static_cast<uint8_t>(Compressor); }
+ uint8_t GetCompressionLevel() const final { return static_cast<uint8_t>(CompressionLevel); }
uint64_t CompressBlockBound(uint64_t RawSize) const final
{
- if (RawSize <= LZ4_MAX_INPUT_SIZE)
+ return static_cast<uint64_t>(OodleLZ_GetCompressedBufferSizeNeeded(OodleLZ_Compressor_Kraken, static_cast<OO_SINTa>(RawSize)));
+ }
+
+ bool CompressBlock(MutableMemoryView& CompressedData, MemoryView RawData) const final
+ {
+ const OodleLZ_Compressor LZCompressor = GetOodleLZCompressor(Compressor);
+ const OodleLZ_CompressionLevel LZCompressionLevel = GetOodleLZCompressionLevel(CompressionLevel);
+ if (LZCompressor == OodleLZ_Compressor_Invalid || LZCompressionLevel == OodleLZ_CompressionLevel_Invalid ||
+ LZCompressionLevel == OodleLZ_CompressionLevel_None)
{
- return static_cast<uint64_t>(LZ4_compressBound(static_cast<int>(RawSize)));
+ return false;
}
- return 0;
+
+ const OO_SINTa RawSize = static_cast<OO_SINTa>(RawData.GetSize());
+ if (static_cast<OO_SINTa>(CompressedData.GetSize()) < OodleLZ_GetCompressedBufferSizeNeeded(LZCompressor, RawSize))
+ {
+ return false;
+ }
+
+ const OO_SINTa Size = OodleLZ_Compress(LZCompressor, RawData.GetData(), RawSize, CompressedData.GetData(), LZCompressionLevel);
+ CompressedData.LeftInline(static_cast<uint64_t>(Size));
+ return Size > 0;
}
- bool CompressBlock(MutableMemoryView& CompressedData, MemoryView RawData) const final
+ static OodleLZ_Compressor GetOodleLZCompressor(OodleCompressor Compressor)
{
- if (RawData.GetSize() <= LZ4_MAX_INPUT_SIZE)
+ switch (Compressor)
{
- const int Size =
- LZ4_compress_default(static_cast<const char*>(RawData.GetData()),
- static_cast<char*>(CompressedData.GetData()),
- static_cast<int>(RawData.GetSize()),
- static_cast<int>(zen::Min<uint64_t>(CompressedData.GetSize(), std::numeric_limits<int>::max())));
- CompressedData.LeftInline(static_cast<uint64_t>(Size));
- return Size > 0;
+ case OodleCompressor::Selkie:
+ return OodleLZ_Compressor_Selkie;
+ case OodleCompressor::Mermaid:
+ return OodleLZ_Compressor_Mermaid;
+ case OodleCompressor::Kraken:
+ return OodleLZ_Compressor_Kraken;
+ case OodleCompressor::Leviathan:
+ return OodleLZ_Compressor_Leviathan;
+ case OodleCompressor::NotSet:
+ default:
+ return OodleLZ_Compressor_Invalid;
}
- return false;
}
+ static OodleLZ_CompressionLevel GetOodleLZCompressionLevel(OodleCompressionLevel Level)
+ {
+ const int IntLevel = (int)Level;
+ if (IntLevel < (int)OodleLZ_CompressionLevel_Min || IntLevel > (int)OodleLZ_CompressionLevel_Max)
+ {
+ return OodleLZ_CompressionLevel_Invalid;
+ }
+ return OodleLZ_CompressionLevel(IntLevel);
+ }
+
+private:
+ const OodleCompressor Compressor;
+ const OodleCompressionLevel CompressionLevel;
+};
+
+class OodleDecoder final : public BlockDecoder
+{
+protected:
+ bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const final
+ {
+ const OO_SINTa RawSize = static_cast<OO_SINTa>(RawData.GetSize());
+ const OO_SINTa Size = OodleLZ_Decompress(CompressedData.GetData(),
+ static_cast<OO_SINTa>(CompressedData.GetSize()),
+ RawData.GetData(),
+ RawSize,
+ OodleLZ_FuzzSafe_Yes,
+ OodleLZ_CheckCRC_Yes,
+ OodleLZ_Verbosity_None);
+ return Size == RawSize;
+ }
+};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+class LZ4Decoder final : public BlockDecoder
+{
+protected:
bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const final
{
if (CompressedData.GetSize() <= std::numeric_limits<int>::max())
@@ -481,11 +598,12 @@ protected:
//////////////////////////////////////////////////////////////////////////
-static const CompressionMethodBase*
-GetMethod(CompressionMethod Method)
+static const BaseDecoder*
+GetDecoder(CompressionMethod Method)
{
- static MethodNone None;
- static MethodLZ4 LZ4;
+ static NoneDecoder None;
+ static OodleDecoder Oodle;
+ static LZ4Decoder LZ4;
switch (Method)
{
@@ -493,25 +611,13 @@ GetMethod(CompressionMethod Method)
return nullptr;
case CompressionMethod::None:
return &None;
+ case CompressionMethod::Oodle:
+ return &Oodle;
case CompressionMethod::LZ4:
return &LZ4;
}
}
-static const char*
-GetMethodName(CompressionMethod Method)
-{
- switch (Method)
- {
- default:
- return "error";
- case CompressionMethod::None:
- return "None";
- case CompressionMethod::LZ4:
- return "LZ4";
- }
-}
-
//////////////////////////////////////////////////////////////////////////
bool
@@ -522,10 +628,10 @@ BufferHeader::IsValid(const CompositeBuffer& CompressedData)
const BufferHeader Header = Read(CompressedData);
if (Header.Magic == BufferHeader::ExpectedMagic)
{
- if (const CompressionMethodBase* const Method = GetMethod(Header.Method))
+ if (const BaseDecoder* const Decoder = GetDecoder(Header.Method))
{
UniqueBuffer HeaderCopy;
- const MemoryView HeaderView = CompressedData.ViewOrCopyRange(0, Method->GetHeaderSize(Header), HeaderCopy);
+ const MemoryView HeaderView = CompressedData.ViewOrCopyRange(0, Decoder->GetHeaderSize(Header), HeaderCopy);
if (Header.Crc32 == BufferHeader::CalculateCrc32(HeaderView))
{
return true;
@@ -550,22 +656,37 @@ ValidBufferOrEmpty(BufferType&& CompressedData)
namespace zen {
CompressedBuffer
-CompressedBuffer::Compress(CompressionMethod Method, const CompositeBuffer& RawData)
+CompressedBuffer::Compress(const CompositeBuffer& RawData,
+ OodleCompressor Compressor,
+ OodleCompressionLevel CompressionLevel,
+ uint64_t BlockSize)
{
using namespace detail;
+ if (BlockSize == 0)
+ {
+ BlockSize = DefaultBlockSize;
+ }
+
CompressedBuffer Local;
- if (const CompressionMethodBase* const Impl = GetMethod(Method))
+ if (CompressionLevel == OodleCompressionLevel::None)
{
- Local.CompressedData = Impl->Compress(RawData);
+ Local.CompressedData = NoneEncoder().Compress(RawData, BlockSize);
+ }
+ else
+ {
+ Local.CompressedData = OodleEncoder(Compressor, CompressionLevel).Compress(RawData, BlockSize);
}
return Local;
}
CompressedBuffer
-CompressedBuffer::Compress(CompressionMethod Method, const SharedBuffer& RawData)
+CompressedBuffer::Compress(const SharedBuffer& RawData,
+ OodleCompressor Compressor,
+ OodleCompressionLevel CompressionLevel,
+ uint64_t BlockSize)
{
- return Compress(Method, CompositeBuffer(RawData));
+ return Compress(CompositeBuffer(RawData), Compressor, CompressionLevel, BlockSize);
}
CompressedBuffer
@@ -619,9 +740,9 @@ CompressedBuffer::TryDecompressTo(MutableMemoryView RawView) const
if (CompressedData)
{
const BufferHeader Header = BufferHeader::Read(CompressedData);
- if (const CompressionMethodBase* const Method = GetMethod(Header.Method))
+ if (const BaseDecoder* const Decoder = GetDecoder(Header.Method))
{
- return Method->TryDecompressTo(Header, CompressedData, RawView);
+ return Decoder->TryDecompressTo(Header, CompressedData, RawView);
}
}
return false;
@@ -634,14 +755,14 @@ CompressedBuffer::Decompress() const
if (CompressedData)
{
const BufferHeader Header = BufferHeader::Read(CompressedData);
- if (const CompressionMethodBase* const Method = GetMethod(Header.Method))
+ if (const BaseDecoder* const Decoder = GetDecoder(Header.Method))
{
if (Header.Method == CompressionMethod::None)
{
- return Method->Decompress(Header, CompressedData).Flatten();
+ return Decoder->Decompress(Header, CompressedData).Flatten();
}
UniqueBuffer RawData = UniqueBuffer::Alloc(Header.TotalRawSize);
- if (Method->TryDecompressTo(Header, CompressedData, RawData))
+ if (Decoder->TryDecompressTo(Header, CompressedData, RawData))
{
return RawData.MoveToShared();
}
@@ -657,18 +778,35 @@ CompressedBuffer::DecompressToComposite() const
if (CompressedData)
{
const BufferHeader Header = BufferHeader::Read(CompressedData);
- if (const CompressionMethodBase* const Method = GetMethod(Header.Method))
+ if (const BaseDecoder* const Decoder = GetDecoder(Header.Method))
{
- return Method->Decompress(Header, CompressedData);
+ return Decoder->Decompress(Header, CompressedData);
}
}
return CompositeBuffer();
}
-const char*
-CompressedBuffer::GetFormatName() const
+bool
+CompressedBuffer::TryGetCompressParameters(OodleCompressor& OutCompressor, OodleCompressionLevel& OutCompressionLevel) const
{
- return detail::GetMethodName(CompressedData ? detail::BufferHeader::Read(CompressedData).Method : CompressionMethod::None);
+ using namespace detail;
+ if (CompressedData)
+ {
+ switch (const BufferHeader Header = BufferHeader::Read(CompressedData); Header.Method)
+ {
+ case CompressionMethod::None:
+ OutCompressor = OodleCompressor::NotSet;
+ OutCompressionLevel = OodleCompressionLevel::None;
+ return true;
+ case CompressionMethod::Oodle:
+ OutCompressor = OodleCompressor(Header.Compressor);
+ OutCompressionLevel = OodleCompressionLevel(Header.CompressionLevel);
+ return true;
+ default:
+ break;
+ }
+ }
+ return false;
}
/**
@@ -682,13 +820,14 @@ CompressedBuffer::GetFormatName() const
TEST_CASE("CompressedBuffer")
{
- uint8_t Zeroes[256]{};
- uint8_t Ones[256];
+ uint8_t Zeroes[1024]{};
+ uint8_t Ones[1024];
memset(Ones, 1, sizeof Ones);
{
- CompressedBuffer Buffer =
- CompressedBuffer::Compress(CompressionMethod::None, CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes))));
+ CompressedBuffer Buffer = CompressedBuffer::Compress(CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes))),
+ OodleCompressor::NotSet,
+ OodleCompressionLevel::None);
CHECK(Buffer.GetRawSize() == sizeof(Zeroes));
CHECK(Buffer.GetCompressedSize() == (sizeof(Zeroes) + sizeof(detail::BufferHeader)));
@@ -706,8 +845,9 @@ TEST_CASE("CompressedBuffer")
{
CompressedBuffer Buffer = CompressedBuffer::Compress(
- CompressionMethod::None,
- CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)), SharedBuffer::MakeView(MakeMemoryView(Ones))));
+ CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)), SharedBuffer::MakeView(MakeMemoryView(Ones))),
+ OodleCompressor::NotSet,
+ OodleCompressionLevel::None);
CHECK(Buffer.GetRawSize() == (sizeof(Zeroes) + sizeof(Ones)));
CHECK(Buffer.GetCompressedSize() == (sizeof(Zeroes) + sizeof(Ones) + sizeof(detail::BufferHeader)));
@@ -724,11 +864,10 @@ TEST_CASE("CompressedBuffer")
}
{
- CompressedBuffer Buffer =
- CompressedBuffer::Compress(CompressionMethod::LZ4, CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes))));
+ CompressedBuffer Buffer = CompressedBuffer::Compress(CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes))));
CHECK(Buffer.GetRawSize() == sizeof(Zeroes));
- CHECK(Buffer.GetCompressedSize() == (15 + sizeof(detail::BufferHeader)));
+ CHECK(Buffer.GetCompressedSize() < sizeof(Zeroes));
CompositeBuffer Compressed = Buffer.GetCompressed();
CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed);
@@ -743,11 +882,10 @@ TEST_CASE("CompressedBuffer")
{
CompressedBuffer Buffer = CompressedBuffer::Compress(
- CompressionMethod::LZ4,
CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)), SharedBuffer::MakeView(MakeMemoryView(Ones))));
CHECK(Buffer.GetRawSize() == (sizeof(Zeroes) + sizeof(Ones)));
- CHECK(Buffer.GetCompressedSize() == (20 + sizeof(detail::BufferHeader)));
+ CHECK(Buffer.GetCompressedSize() < (sizeof(Zeroes) + sizeof(Ones)));
CompositeBuffer Compressed = Buffer.GetCompressed();
CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed);