streaming decompression support (#142)

Added CompressedBufferReader support from UE. This provides some streaming decompression support which can be employed to reduce memory and other resource usage.

5 files changed, 185 insertions, 4 deletions

diff --git a/src/zencore/include/zencore/compositebuffer.h b/src/zencore/include/zencore/compositebuffer.h
index 4e4b4d002..cc03dd156 100644
--- a/src/zencore/include/zencore/compositebuffer.h
+++ b/src/zencore/include/zencore/compositebuffer.h
@@ -62,8 +62,12 @@ public:
 	[[nodiscard]] ZENCORE_API CompositeBuffer MakeOwned() &&;
 
 	/** Returns the concatenation of the segments into a contiguous buffer. */
-	[[nodiscard]] ZENCORE_API SharedBuffer Flatten() const&;
-	[[nodiscard]] ZENCORE_API SharedBuffer Flatten() &&;
+	[[nodiscard]] inline SharedBuffer Flatten() const& { return ToShared(); }
+	[[nodiscard]] inline SharedBuffer Flatten() && { return ToShared(); }
+
+	/** Returns the concatenation of the segments into a contiguous buffer. */
+	[[nodiscard]] ZENCORE_API SharedBuffer ToShared() const&;
+	[[nodiscard]] ZENCORE_API SharedBuffer ToShared() &&;
 
 	/** Returns the middle part of the buffer by taking the size starting at the offset. */
 	[[nodiscard]] ZENCORE_API CompositeBuffer Mid(uint64_t Offset, uint64_t Size = ~uint64_t(0)) const;
@@ -76,8 +80,12 @@ public:
 	 * @param Offset       The byte offset in this buffer that the range starts at.
 	 * @param Size         The number of bytes in the range to view or copy.
 	 * @param CopyBuffer   The buffer to write the copy into if a copy is required.
+	 * @param Allocator    The optional allocator to use when the copy buffer is required.
 	 */
-	[[nodiscard]] ZENCORE_API MemoryView ViewOrCopyRange(uint64_t Offset, uint64_t Size, UniqueBuffer& CopyBuffer) const;
+	[[nodiscard]] ZENCORE_API MemoryView ViewOrCopyRange(uint64_t									Offset,
+														 uint64_t									Size,
+														 UniqueBuffer&								CopyBuffer,
+														 std::function<UniqueBuffer(uint64_t Size)> Allocator = UniqueBuffer::Alloc) const;
 
 	/**
 	 * Copies a range of the buffer to a contiguous region of memory.
diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h
index 99ce20d8a..44431f299 100644
--- a/src/zencore/include/zencore/compress.h
+++ b/src/zencore/include/zencore/compress.h
@@ -6,9 +6,18 @@
 
 #include "zencore/blake3.h"
 #include "zencore/compositebuffer.h"
+#include "zencore/intmath.h"
+
+#include <limits>
 
 namespace zen {
 
+class Archive;
+
+namespace detail {
+	struct BufferHeader;
+}
+
 enum class OodleCompressor : uint8_t
 {
 	NotSet	  = 0,
@@ -160,6 +169,115 @@ private:
 	CompositeBuffer CompressedData;
 };
 
+namespace detail {
+	/** A reusable context for the compressed buffer decoder. */
+	struct DecoderContext
+	{
+		/** The offset in the source at which the compressed buffer begins, otherwise MAX_uint64. */
+		uint64_t HeaderOffset = MAX_uint64;
+		/** The size of the header if known, otherwise 0. */
+		uint64_t HeaderSize = 0;
+		/** The CRC-32 from the header, otherwise 0. */
+		uint32_t HeaderCrc32 = 0;
+		/** Index of the block stored in RawBlock, otherwise MAX_uint32. */
+		uint32_t RawBlockIndex = MAX_uint32;
+
+		/** A buffer used to store the header when HeaderOffset is not MAX_uint64. */
+		UniqueBuffer Header;
+		/** A buffer used to store a raw block when a partial block read is requested. */
+		UniqueBuffer RawBlock;
+		/** A buffer used to store a compressed block when it was not in contiguous memory. */
+		UniqueBuffer CompressedBlock;
+	};
+}  // namespace detail
+
+/**
+ * A type that stores the state needed to decompress a compressed buffer.
+ *
+ * The compressed buffer can be in memory or can be loaded from a seekable archive.
+ *
+ * The reader can be reused across multiple source buffers, which allows its temporary buffers to
+ * be reused if they are the right size.
+ *
+ * It is only safe to use the reader from one thread at a time.
+ *
+ * @see CompressedBuffer
+ */
+class CompressedBufferReader
+{
+public:
+	/** Construct a reader with no source. */
+	CompressedBufferReader() = default;
+
+	/** Construct a reader that will read from an archive as needed. */
+	explicit CompressedBufferReader(Archive& Archive);
+
+	/** Construct a reader from an in-memory compressed buffer. */
+	explicit CompressedBufferReader(const CompressedBuffer& Buffer);
+
+	/** Release any temporary buffers that have been allocated by the reader. */
+	void ResetBuffers();
+
+	/** Clears the reference to the source without releasing temporary buffers. */
+	void ResetSource();
+
+	void SetSource(Archive& Archive);
+	void SetSource(const CompressedBuffer& Buffer);
+
+	[[nodiscard]] inline bool HasSource() const { return SourceArchive || SourceBuffer; }
+
+	/** Returns the size of the compressed data. Zero on error. */
+	[[nodiscard]] uint64_t GetCompressedSize();
+
+	/** Returns the size of the raw data. Zero on error. */
+	[[nodiscard]] uint64_t GetRawSize();
+
+	/** Returns the hash of the raw data. Zero on error. */
+	[[nodiscard]] IoHash GetRawHash();
+
+	/**
+	 * Returns the compressor and compression level used by this buffer.
+	 *
+	 * The compressor and compression level may differ from those specified when creating the buffer
+	 * because an incompressible buffer is stored with no compression. Parameters cannot be accessed
+	 * if this is null or uses a method other than Oodle, in which case this returns false.
+	 *
+	 * @return True if parameters were written, otherwise false.
+	 */
+	[[nodiscard]] bool TryGetCompressParameters(OodleCompressor&	   OutCompressor,
+												OodleCompressionLevel& OutCompressionLevel,
+												uint64_t&			   OutBlockSize);
+
+	/**
+	 * Decompress into a memory view that is less than or equal to the available raw size.
+	 *
+	 * @param RawView     The view to write to. The size to read is equal to the view size.
+	 * @param RawOffset   The offset into the raw data from which to decompress.
+	 * @return True if the requested range was decompressed, otherwise false.
+	 */
+	[[nodiscard]] bool TryDecompressTo(MutableMemoryView RawView, uint64_t RawOffset = 0);
+
+	/**
+	 * Decompress into an owned buffer.
+	 *
+	 * RawOffset must be at most the raw buffer size. RawSize may be MAX_uint64 to read the whole
+	 * buffer from RawOffset, and must otherwise fit within the bounds of the buffer.
+	 *
+	 * @param RawOffset   The offset into the raw data from which to decompress.
+	 * @param RawSize     The size of the raw data to read from the offset.
+	 * @return An owned buffer containing the raw data, or null on error.
+	 */
+	[[nodiscard]] SharedBuffer	  Decompress(uint64_t RawOffset = 0, uint64_t RawSize = MAX_uint64);
+	[[nodiscard]] CompositeBuffer DecompressToComposite(uint64_t RawOffset = 0, uint64_t RawSize = MAX_uint64);
+
+private:
+	bool TryReadHeader(detail::BufferHeader& OutHeader, MemoryView& OutHeaderView);
+
+	Archive*				SourceArchive = nullptr;
+	const CompressedBuffer* SourceBuffer  = nullptr;
+	detail::DecoderContext	Context;
+};
+
 void compress_forcelink();	// internal
 
 }  // namespace zen
diff --git a/src/zencore/include/zencore/intmath.h b/src/zencore/include/zencore/intmath.h
index f24caed6e..0d87e1b78 100644
--- a/src/zencore/include/zencore/intmath.h
+++ b/src/zencore/include/zencore/intmath.h
@@ -7,6 +7,32 @@
 #include <stdint.h>
 
 //////////////////////////////////////////////////////////////////////////
+// UE Numeric constants
+
+#define MIN_uint8  ((uint8_t)0x00)
+#define MIN_uint16 ((uint16_t)0x0000)
+#define MIN_uint32 ((uint32_t)0x00000000)
+#define MIN_uint64 ((uint64_t)0x0000000000000000)
+#define MIN_int8   ((int8_t)-128)
+#define MIN_int16  ((int16_t)-32768)
+#define MIN_int32  ((int32_t)0x80000000)
+#define MIN_int64  ((int64_t)0x8000000000000000)
+
+#define MAX_uint8  ((uint8_t)0xff)
+#define MAX_uint16 ((uint16_t)0xffff)
+#define MAX_uint32 ((uint32_t)0xffffffff)
+#define MAX_uint64 ((uint64_t)0xffffffffffffffff)
+#define MAX_int8   ((int8_t)0x7f)
+#define MAX_int16  ((int16_t)0x7fff)
+#define MAX_int32  ((int32_t)0x7fffffff)
+#define MAX_int64  ((int64_t)0x7fffffffffffffff)
+
+#define MIN_flt (1.175494351e-38F) /* min positive value */
+#define MAX_flt (3.402823466e+38F)
+#define MIN_dbl (2.2250738585072014e-308) /* min positive value */
+#define MAX_dbl (1.7976931348623158e+308)
+
+//////////////////////////////////////////////////////////////////////////
 
 #if ZEN_COMPILER_MSC || ZEN_PLATFORM_WINDOWS
 #	pragma intrinsic(_BitScanReverse)
diff --git a/src/zencore/include/zencore/stream.h b/src/zencore/include/zencore/stream.h
index 9e4996249..a9d35ef1b 100644
--- a/src/zencore/include/zencore/stream.h
+++ b/src/zencore/include/zencore/stream.h
@@ -79,12 +79,37 @@ public:
 	inline uint64_t CurrentOffset() const { return m_Offset; }
 	inline void		Skip(size_t ByteCount) { m_Offset += ByteCount; };
 
-private:
+protected:
 	const uint8_t* m_BufferBase;
 	uint64_t	   m_BufferSize;
 	uint64_t	   m_Offset = 0;
 };
 
+/**
+ * Archive base class implementation
+ *
+ * Roughly mimics UE's FArchive class, but without all the non-essential functionality
+ */
+class Archive
+{
+public:
+	/** Attempts to set the current offset into backing data storage, this will do nothing if there is no storage. */
+	virtual void	Seek(uint64_t InPos)			   = 0;
+	virtual int64_t Tell()							   = 0;
+	virtual void	Serialize(void* V, int64_t Length) = 0;
+	inline bool		IsError() { return false; }
+};
+
+class BufferReader : public Archive, private BinaryReader
+{
+public:
+	BufferReader(const void* Buffer, uint64_t Size) : BinaryReader(Buffer, Size) {}
+
+	virtual void	Seek(uint64_t InPos) override;
+	virtual int64_t Tell() override;
+	virtual void	Serialize(void* V, int64_t Length) override;
+};
+
 void stream_forcelink();  // internal
 
 }  // namespace zen
diff --git a/src/zencore/include/zencore/zencore.h b/src/zencore/include/zencore/zencore.h
index 02e9dd5d6..47ff2ebc2 100644
--- a/src/zencore/include/zencore/zencore.h
+++ b/src/zencore/include/zencore/zencore.h
@@ -68,6 +68,10 @@
 #	pragma warning(disable : 4324)	 // warning C4324: '<type>': structure was padded due to alignment specifier
 #	pragma warning(default : 4668)	 // warning C4668: 'symbol' is not defined as a preprocessor macro, replacing with '0' for 'directives'
 #	pragma warning(default : 4100)	 // warning C4100: 'identifier' : unreferenced formal parameter
+#	pragma warning( \
+		disable : 4373)	 // '%$S': virtual function overrides '%$pS', previous versions of the compiler did not override when parameters
+						 // only differed by const/volatile qualifiers
+						 // https://docs.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-3-c4373
 #endif
 
 #ifndef ZEN_THIRD_PARTY_INCLUDES_START