From db6a98ba01b7e3b0dd79c725dbaadd5f465c6799 Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Mon, 2 Jun 2025 14:55:35 +0200
Subject: streaming none compressor (#414)

* add proper streaming to none compressor type
---
 src/zencore/compress.cpp | 39 ++++++++++++++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

(limited to 'src/zencore/compress.cpp')

diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp
index ad6b6103c..62b64bc9d 100644
--- a/src/zencore/compress.cpp
+++ b/src/zencore/compress.cpp
@@ -7,6 +7,7 @@
 #include <zencore/compositebuffer.h>
 #include <zencore/crc32.h>
 #include <zencore/endian.h>
+#include <zencore/filesystem.h>
 #include <zencore/intmath.h>
 #include <zencore/iohash.h>
 #include <zencore/stream.h>
@@ -217,6 +218,20 @@ public:
 	{
 		UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), BLAKE3::HashBuffer(RawData));
 		Callback(0, 0, 0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderData.GetData(), HeaderData.GetSize())));
+
+		IoBufferFileReference FileRef = {nullptr, 0, 0};
+		if ((RawData.GetSegments().size() == 1) && RawData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef))
+		{
+			ZEN_ASSERT(FileRef.FileHandle != nullptr);
+			uint64_t CallbackOffset = 0;
+			ScanFile(FileRef.FileHandle, 0, RawData.GetSize(), 512u * 1024u, [&](const void* Data, size_t Size) {
+				CompositeBuffer Tmp(SharedBuffer(IoBuffer(IoBuffer::Wrap, Data, Size)));
+				Callback(CallbackOffset, Size, HeaderData.GetSize() + CallbackOffset, Tmp);
+				CallbackOffset += Size;
+			});
+			return true;
+		}
+
 		Callback(0, RawData.GetSize(), HeaderData.GetSize(), RawData);
 		return true;
 	}
@@ -299,11 +314,29 @@ public:
 			Header.TotalCompressedSize == Header.TotalRawSize + sizeof(BufferHeader) && RawOffset < Header.TotalRawSize &&
 			(RawOffset + RawSize) <= Header.TotalRawSize)
 		{
-			if (!Callback(sizeof(BufferHeader) + RawOffset, RawSize, 0, CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize)))
+			bool				  Result  = true;
+			IoBufferFileReference FileRef = {nullptr, 0, 0};
+			if ((CompressedData.GetSegments().size() == 1) && CompressedData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef))
 			{
-				return false;
+				ZEN_ASSERT(FileRef.FileHandle != nullptr);
+				uint64_t CallbackOffset = 0;
+				ScanFile(FileRef.FileHandle, sizeof(BufferHeader) + RawOffset, RawSize, 512u * 1024u, [&](const void* Data, size_t Size) {
+					if (Result)
+					{
+						CompositeBuffer Tmp(SharedBuffer(IoBuffer(IoBuffer::Wrap, Data, Size)));
+						Result = Callback(sizeof(BufferHeader) + RawOffset + CallbackOffset, Size, CallbackOffset, Tmp);
+					}
+					CallbackOffset += Size;
+				});
+				return Result;
+			}
+			else
+			{
+				return Callback(sizeof(BufferHeader) + RawOffset,
+								RawSize,
+								0,
+								CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize));
 			}
-			return true;
 		}
 		return false;
 	}
-- 
cgit v1.2.3