Implemented support for UE5-style CompressedBuffers

Currently supports None, LZ4 compression types

1 files changed, 98 insertions, 33 deletions

diff --git a/zencore/include/zencore/compress.h b/zencore/include/zencore/compress.h
index 759cf8444..011d4ed64 100644
--- a/zencore/include/zencore/compress.h
+++ b/zencore/include/zencore/compress.h
@@ -4,16 +4,13 @@
 
 #include "zencore/zencore.h"
 
-namespace zen::CompressedBuffer {
+#include "zencore/blake3.h"
+#include "zencore/compositebuffer.h"
 
-///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-static constexpr uint64_t DefaultBlockSize = 256 * 1024;
-
-///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+namespace zen {
 
 /** Method used to compress the data in a compressed buffer. */
-enum class Method : uint8_t
+enum class CompressionMethod : uint8_t
 {
 	/** Header is followed by one uncompressed block. */
 	None = 0,
@@ -21,33 +18,101 @@ enum class Method : uint8_t
 	LZ4 = 4,
 };
 
-///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-/** Header used on every compressed buffer. Always stored in big-endian format */
-struct BufferHeader
+/**
+ * A compressed buffer stores compressed data in a self-contained format.
+ *
+ * A buffer is self-contained in the sense that it can be decompressed without external knowledge
+ * of the compression format or the size of the raw data.
+ */
+class CompressedBuffer
 {
-	static constexpr uint32_t ExpectedMagic = 0xb7756362;
-
-	/** A magic number to identify a compressed buffer. Always 0xb7756362 */
-	uint32_t Magic = ExpectedMagic;
-	/** A CRC-32 used to check integrity of the buffer. Uses the polynomial 0x04c11db7 */
-	uint32_t Crc32 = 0;
-	/** The method used to compress the buffer. Affects layout of data following the header */
-	Method Method = Method::None;
-	/** The reserved bytes must be initialized to zero */
-	uint8_t Reserved[2]{};
-	/** The power of two size of every uncompressed block except the last. Size is 1 << BlockSizeExponent */
-	uint8_t BlockSizeExponent = 0;
-	/** The number of blocks that follow the header */
-	uint32_t BlockCount = 0;
-	/** The total size of the uncompressed data */
-	uint64_t TotalRawSize = 0;
-	/** The total size of the compressed data including the header */
-	uint64_t TotalCompressedSize = 0;
-};
+public:
+	/**
+	 * Compress the buffer using the requested compression format.
+	 *
+	 * @param FormatName   One of NAME_None, NAME_Default, NAME_LZ4.
+	 * @param RawData      Raw data to compress. NAME_None will reference owned raw data.
+	 * @return An owned compressed buffer, or null on error.
+	 */
+	[[nodiscard]] ZENCORE_API static CompressedBuffer Compress(CompressionMethod Method, const CompositeBuffer& RawData);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer Compress(CompressionMethod Method, const SharedBuffer& RawData);
+
+	/**
+	 * Construct from a compressed buffer previously created by Compress().
+	 *
+	 * @return A compressed buffer, or null on error, such as an invalid format or corrupt header.
+	 */
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const CompositeBuffer& CompressedData);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData);
+	[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData);
+
+	/** Reset this to null. */
+	inline void Reset() { CompressedData.Reset(); }
+
+	/** Returns true if the compressed buffer is not null. */
+	[[nodiscard]] inline explicit operator bool() const { return !IsNull(); }
+
+	/** Returns true if the compressed buffer is null. */
+	[[nodiscard]] inline bool IsNull() const { return CompressedData.IsNull(); }
+
+	/** Returns true if the composite buffer is owned. */
+	[[nodiscard]] inline bool IsOwned() const { return CompressedData.IsOwned(); }
+
+	/** Returns a copy of the compressed buffer that owns its underlying memory. */
+	[[nodiscard]] inline CompressedBuffer MakeOwned() const& { return FromCompressed(CompressedData.MakeOwned()); }
+	[[nodiscard]] inline CompressedBuffer MakeOwned() && { return FromCompressed(std::move(CompressedData).MakeOwned()); }
 
-static_assert(sizeof(BufferHeader) == 32, "BufferHeader is the wrong size");
+	/** Returns a composite buffer containing the compressed data. May be null. May not be owned. */
+	[[nodiscard]] inline const CompositeBuffer& GetCompressed() const& { return CompressedData; }
+	[[nodiscard]] inline CompositeBuffer		GetCompressed() && { return std::move(CompressedData); }
+
+	/** Returns the size of the compressed data. Zero if this is null. */
+	[[nodiscard]] inline uint64_t GetCompressedSize() const { return CompressedData.GetSize(); }
+
+	/** Returns the size of the raw data. Zero on error or if this is empty or null. */
+	[[nodiscard]] ZENCORE_API uint64_t GetRawSize() const;
+
+	/** Returns the hash of the raw data. Zero on error or if this is null. */
+	[[nodiscard]] ZENCORE_API BLAKE3 GetRawHash() const;
+
+	/**
+	 * Returns the name of the compression format used by this buffer.
+	 *
+	 * The format name may differ from the format name specified when creating the compressed buffer
+	 * because an incompressible buffer is stored with NAME_None, and a request of NAME_Default will
+	 * be stored in a specific format such as NAME_LZ4.
+	 *
+	 * @return The format name, or NAME_None if this null, or NAME_Error if the format is unknown.
+	 */
+	[[nodiscard]] ZENCORE_API const char* GetFormatName() const;
+
+	/**
+	 * Decompress into a memory view that is exactly GetRawSize() bytes.
+	 */
+	[[nodiscard]] ZENCORE_API bool TryDecompressTo(MutableMemoryView RawView) const;
+
+	/**
+	 * Decompress into an owned buffer.
+	 *
+	 * @return An owned buffer containing the raw data, or null on error.
+	 */
+	[[nodiscard]] ZENCORE_API SharedBuffer Decompress() const;
+
+	/**
+	 * Decompress into an owned composite buffer.
+	 *
+	 * @return An owned buffer containing the raw data, or null on error.
+	 */
+	[[nodiscard]] ZENCORE_API CompositeBuffer DecompressToComposite() const;
+
+	/** A null compressed buffer. */
+	static const CompressedBuffer Null;
+
+private:
+	CompositeBuffer CompressedData;
+};
 
-///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+void compress_forcelink();  // internal
 
-}  // namespace zen::CompressedBuffer
+}  // namespace zen