Merge remote-tracking branch 'origin/main' into de/zen-service-command

26 files changed, 1257 insertions, 140 deletions

diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp
index c2a21ae90..95876cff4 100644
--- a/src/zencore/basicfile.cpp
+++ b/src/zencore/basicfile.cpp
@@ -28,6 +28,20 @@ BasicFile::~BasicFile()
 {
 	Close();
 }
+BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode)
+{
+	Open(FileName, Mode);
+}
+
+BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec)
+{
+	Open(FileName, Mode, Ec);
+}
+
+BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode, std::function<bool(std::error_code& Ec)>&& RetryCallback)
+{
+	Open(FileName, Mode, std::move(RetryCallback));
+}
 
 void
 BasicFile::Open(const std::filesystem::path& FileName, Mode Mode)
@@ -267,7 +281,21 @@ BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<voi
 }
 
 uint64_t
-BasicFile::Write(CompositeBuffer Data, uint64_t FileOffset, std::error_code& Ec)
+BasicFile::Write(const CompositeBuffer& Data, uint64_t FileOffset)
+{
+	std::error_code Ec;
+	uint64_t		WrittenBytes = Write(Data, FileOffset, Ec);
+
+	if (Ec)
+	{
+		std::error_code Dummy;
+		throw std::system_error(Ec, fmt::format("Failed to write to file '{}'", zen::PathFromHandle(m_FileHandle, Dummy)));
+	}
+	return WrittenBytes;
+}
+
+uint64_t
+BasicFile::Write(const CompositeBuffer& Data, uint64_t FileOffset, std::error_code& Ec)
 {
 	uint64_t WrittenBytes = 0;
 	for (const SharedBuffer& Buffer : Data.GetSegments())
@@ -295,6 +323,8 @@ BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec)
 void
 BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec)
 {
+	ZEN_ASSERT(m_FileHandle != nullptr);
+
 	Ec.clear();
 
 	const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
@@ -575,7 +605,6 @@ TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::
 void
 TemporaryFile::SafeWriteFile(const std::filesystem::path& Path, MemoryView Data)
 {
-	TemporaryFile	TempFile;
 	std::error_code Ec;
 	SafeWriteFile(Path, Data, Ec);
 	if (Ec)
@@ -804,6 +833,17 @@ BasicFileWriter::Write(const void* Data, uint64_t Size, uint64_t FileOffset)
 }
 
 void
+BasicFileWriter::Write(const CompositeBuffer& Data, uint64_t FileOffset)
+{
+	for (const SharedBuffer& Segment : Data.GetSegments())
+	{
+		const uint64_t SegmentSize = Segment.GetSize();
+		Write(Segment.GetData(), SegmentSize, FileOffset);
+		FileOffset += SegmentSize;
+	}
+}
+
+void
 BasicFileWriter::Flush()
 {
 	const uint64_t BufferedBytes = m_BufferEnd - m_BufferStart;
@@ -817,6 +857,78 @@ BasicFileWriter::Flush()
 	m_Base.Write(m_Buffer, BufferedBytes, WriteOffset);
 }
 
+IoBuffer
+WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path)
+{
+	TemporaryFile	Temp;
+	std::error_code Ec;
+	Temp.CreateTemporary(Path.parent_path(), Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("Failed to create temp file for blob at '{}'", Path));
+	}
+
+	uint64_t BufferSize = Buffer.GetSize();
+	{
+		uint64_t			  Offset		= 0;
+		static const uint64_t BufferingSize = 256u * 1024u;
+		//		BasicFileWriter		  BufferedOutput(BlockFile, BufferingSize / 2);
+		for (const SharedBuffer& Segment : Buffer.GetSegments())
+		{
+			size_t SegmentSize = Segment.GetSize();
+
+			IoBufferFileReference FileRef;
+			if (SegmentSize >= (BufferingSize + BufferingSize / 2) && Segment.GetFileReference(FileRef))
+			{
+				ScanFile(FileRef.FileHandle,
+						 FileRef.FileChunkOffset,
+						 FileRef.FileChunkSize,
+						 BufferingSize,
+						 [&Temp, &Offset](const void* Data, size_t Size) {
+							 Temp.Write(Data, Size, Offset);
+							 Offset += Size;
+						 });
+			}
+			else
+			{
+				Temp.Write(Segment.GetData(), SegmentSize, Offset);
+				Offset += SegmentSize;
+			}
+		}
+	}
+
+	Temp.MoveTemporaryIntoPlace(Path, Ec);
+	if (Ec)
+	{
+		Ec.clear();
+		BasicFile OpenTemp(Path, BasicFile::Mode::kDelete, Ec);
+		if (Ec)
+		{
+			throw std::system_error(Ec, fmt::format("Failed to move temp file to '{}'", Path));
+		}
+		if (OpenTemp.FileSize() != BufferSize)
+		{
+			throw std::runtime_error(fmt::format("Failed to move temp file to '{}' - mismatching file size already exists", Path));
+		}
+		IoBuffer TmpBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BufferSize, true);
+
+		IoHash		 ExistingHash = IoHash::HashBuffer(TmpBuffer);
+		const IoHash ExpectedHash = IoHash::HashBuffer(Buffer);
+		if (ExistingHash != ExpectedHash)
+		{
+			throw std::runtime_error(fmt::format("Failed to move temp file to '{}' - mismatching file hash already exists", Path));
+		}
+		Buffer = CompositeBuffer{};
+		TmpBuffer.SetDeleteOnClose(true);
+		return TmpBuffer;
+	}
+	Buffer = CompositeBuffer{};
+	BasicFile OpenTemp(Path, BasicFile::Mode::kDelete);
+	IoBuffer  TmpBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BufferSize, true);
+	TmpBuffer.SetDeleteOnClose(true);
+	return TmpBuffer;
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 /*
diff --git a/src/zencore/compactbinary.cpp b/src/zencore/compactbinary.cpp
index adccaba70..b43cc18f1 100644
--- a/src/zencore/compactbinary.cpp
+++ b/src/zencore/compactbinary.cpp
@@ -15,6 +15,8 @@
 #include <zencore/testing.h>
 #include <zencore/uid.h>
 
+#include <EASTL/fixed_vector.h>
+
 #include <fmt/format.h>
 #include <string_view>
 
@@ -1376,9 +1378,9 @@ TryMeasureCompactBinary(MemoryView View, CbFieldType& OutType, uint64_t& OutSize
 CbField
 LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator)
 {
-	std::vector<uint8_t> HeaderBytes;
-	CbFieldType			 FieldType;
-	uint64_t			 FieldSize = 1;
+	eastl::fixed_vector<uint8_t, 32> HeaderBytes;
+	CbFieldType						 FieldType;
+	uint64_t						 FieldSize = 1;
 
 	for (const int64_t StartPos = Ar.CurrentOffset(); FieldSize > 0;)
 	{
@@ -1393,7 +1395,7 @@ LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator)
 		HeaderBytes.resize(ReadOffset + ReadSize);
 
 		Ar.Read(HeaderBytes.data() + ReadOffset, ReadSize);
-		if (TryMeasureCompactBinary(MakeMemoryView(HeaderBytes), FieldType, FieldSize))
+		if (TryMeasureCompactBinary(MakeMemoryView(HeaderBytes.data(), HeaderBytes.size()), FieldType, FieldSize))
 		{
 			if (FieldSize <= uint64_t(Ar.Size() - StartPos))
 			{
diff --git a/src/zencore/compactbinarybuilder.cpp b/src/zencore/compactbinarybuilder.cpp
index a60de023d..63c0b9c5c 100644
--- a/src/zencore/compactbinarybuilder.cpp
+++ b/src/zencore/compactbinarybuilder.cpp
@@ -15,23 +15,21 @@
 
 namespace zen {
 
-template<typename T>
 uint64_t
-AddUninitialized(std::vector<T>& Vector, uint64_t Count)
+AddUninitialized(CbWriter::CbWriterData_t& Vector, uint64_t Count)
 {
 	const uint64_t Offset = Vector.size();
 	Vector.resize(Offset + Count);
 	return Offset;
 }
 
-template<typename T>
 uint64_t
-Append(std::vector<T>& Vector, const T* Data, uint64_t Count)
+Append(CbWriter::CbWriterData_t& Vector, const uint8_t* Data, uint64_t Count)
 {
 	const uint64_t Offset = Vector.size();
 	Vector.resize(Offset + Count);
 
-	memcpy(Vector.data() + Offset, Data, sizeof(T) * Count);
+	memcpy(Vector.data() + Offset, Data, sizeof(uint8_t) * Count);
 
 	return Offset;
 }
@@ -76,7 +74,7 @@ IsUniformType(const CbFieldType Type)
 
 /** Append the payload from the compact binary value to the array and return its type. */
 static inline CbFieldType
-AppendCompactBinary(const CbFieldView& Value, std::vector<uint8_t>& OutData)
+AppendCompactBinary(const CbFieldView& Value, CbWriter::CbWriterData_t& OutData)
 {
 	struct FCopy : public CbFieldView
 	{
@@ -93,7 +91,6 @@ AppendCompactBinary(const CbFieldView& Value, std::vector<uint8_t>& OutData)
 
 CbWriter::CbWriter()
 {
-	States.reserve(4);
 	States.emplace_back();
 }
 
diff --git a/src/zencore/compactbinarypackage.cpp b/src/zencore/compactbinarypackage.cpp
index 7de161845..ffe64f2e9 100644
--- a/src/zencore/compactbinarypackage.cpp
+++ b/src/zencore/compactbinarypackage.cpp
@@ -3,10 +3,13 @@
 #include "zencore/compactbinarypackage.h"
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/compactbinaryvalidation.h>
+#include <zencore/eastlutil.h>
 #include <zencore/endian.h>
 #include <zencore/stream.h>
 #include <zencore/testing.h>
 
+#include <EASTL/span.h>
+
 namespace zen {
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -341,6 +344,12 @@ CbPackage::SetObject(CbObject InObject, const IoHash* InObjectHash, AttachmentRe
 }
 
 void
+CbPackage::ReserveAttachments(size_t Count)
+{
+	Attachments.reserve(Count);
+}
+
+void
 CbPackage::AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver)
 {
 	if (!Attachment.IsNull())
@@ -374,17 +383,18 @@ CbPackage::AddAttachments(std::span<const CbAttachment> InAttachments)
 	{
 		ZEN_ASSERT(!Attachment.IsNull());
 	}
+
 	// Assume we have no duplicates!
 	Attachments.insert(Attachments.end(), InAttachments.begin(), InAttachments.end());
 	std::sort(Attachments.begin(), Attachments.end());
-	ZEN_ASSERT_SLOW(std::unique(Attachments.begin(), Attachments.end()) == Attachments.end());
+	ZEN_ASSERT_SLOW(eastl::unique(Attachments.begin(), Attachments.end()) == Attachments.end());
 }
 
 int32_t
 CbPackage::RemoveAttachment(const IoHash& Hash)
 {
 	return gsl::narrow_cast<int32_t>(
-		std::erase_if(Attachments, [&Hash](const CbAttachment& Attachment) -> bool { return Attachment.GetHash() == Hash; }));
+		erase_if(Attachments, [&Hash](const CbAttachment& Attachment) -> bool { return Attachment.GetHash() == Hash; }));
 }
 
 bool
diff --git a/src/zencore/compositebuffer.cpp b/src/zencore/compositebuffer.cpp
index 49870a304..252ac9045 100644
--- a/src/zencore/compositebuffer.cpp
+++ b/src/zencore/compositebuffer.cpp
@@ -275,36 +275,18 @@ CompositeBuffer::IterateRange(uint64_t															  Offset,
 			Visitor(View, Segment);
 			break;
 		}
-		if (Offset < SegmentSize)
+		else if (Offset <= SegmentSize)
 		{
-			if (Offset == 0 && Size >= SegmentSize)
+			const MemoryView View = Segment.GetView().Mid(Offset, Size);
+			Offset				  = 0;
+			if (Size == 0 || !View.IsEmpty())
 			{
-				const MemoryView View = Segment.GetView();
-				if (!View.IsEmpty())
-				{
-					Visitor(View, Segment);
-				}
-				Size -= View.GetSize();
-				if (Size == 0)
-				{
-					break;
-				}
+				Visitor(View, Segment);
 			}
-			else
+			Size -= View.GetSize();
+			if (Size == 0)
 			{
-				// If we only want a section of the segment, do a subrange so we don't have to materialize the entire iobuffer
-				IoBuffer		 SubRange(Segment.AsIoBuffer(), Offset, Min(Size, SegmentSize - Offset));
-				const MemoryView View = SubRange.GetView();
-				if (!View.IsEmpty())
-				{
-					Visitor(View, Segment);
-				}
-				Size -= View.GetSize();
-				if (Size == 0)
-				{
-					break;
-				}
-				Offset = 0;
+				break;
 			}
 		}
 		else
diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp
index 29c1d9256..88c3bb5b9 100644
--- a/src/zencore/compress.cpp
+++ b/src/zencore/compress.cpp
@@ -2,6 +2,7 @@
 
 #include <zencore/compress.h>
 
+#include <zencore/basicfile.h>
 #include <zencore/blake3.h>
 #include <zencore/compositebuffer.h>
 #include <zencore/crc32.h>
@@ -157,6 +158,9 @@ class BaseEncoder
 {
 public:
 	[[nodiscard]] virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const = 0;
+	[[nodiscard]] virtual bool			  CompressToStream(const CompositeBuffer&												RawData,
+														   std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
+														   uint64_t BlockSize = DefaultBlockSize) const							= 0;
 };
 
 class BaseDecoder
@@ -184,6 +188,12 @@ public:
 												  const MemoryView	   HeaderView,
 												  uint64_t			   RawOffset,
 												  uint64_t			   RawSize) const = 0;
+
+	virtual bool DecompressToStream(const BufferHeader&													 Header,
+									const CompositeBuffer&												 CompressedData,
+									uint64_t															 RawOffset,
+									uint64_t															 RawSize,
+									std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const = 0;
 };
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -191,11 +201,21 @@ public:
 class NoneEncoder final : public BaseEncoder
 {
 public:
-	[[nodiscard]] CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t /* BlockSize */) const final
+	[[nodiscard]] virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t /* BlockSize */) const final
 	{
 		UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), BLAKE3::HashBuffer(RawData));
 		return CompositeBuffer(HeaderData.MoveToShared(), RawData.MakeOwned());
 	}
+
+	[[nodiscard]] virtual bool CompressToStream(const CompositeBuffer&												 RawData,
+												std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
+												uint64_t /* BlockSize */) const final
+	{
+		UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), BLAKE3::HashBuffer(RawData));
+		Callback(0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderData.GetData(), HeaderData.GetSize())));
+		Callback(HeaderData.GetSize(), RawData);
+		return true;
+	}
 };
 
 class NoneDecoder final : public BaseDecoder
@@ -262,6 +282,25 @@ public:
 	}
 
 	[[nodiscard]] uint64_t GetHeaderSize(const BufferHeader&) const final { return sizeof(BufferHeader); }
+
+	virtual bool DecompressToStream(const BufferHeader&													 Header,
+									const CompositeBuffer&												 CompressedData,
+									uint64_t															 RawOffset,
+									uint64_t															 RawSize,
+									std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const final
+	{
+		if (Header.Method == CompressionMethod::None && Header.TotalCompressedSize == CompressedData.GetSize() &&
+			Header.TotalCompressedSize == Header.TotalRawSize + sizeof(BufferHeader) && RawOffset < Header.TotalRawSize &&
+			(RawOffset + RawSize) <= Header.TotalRawSize)
+		{
+			if (!Callback(0, CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize)))
+			{
+				return false;
+			}
+			return true;
+		}
+		return false;
+	}
 };
 
 //////////////////////////////////////////////////////////////////////////
@@ -269,7 +308,10 @@ public:
 class BlockEncoder : public BaseEncoder
 {
 public:
-	CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const final;
+	virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize) const final;
+	virtual bool			CompressToStream(const CompositeBuffer&												  RawData,
+											 std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
+											 uint64_t															  BlockSize) const final;
 
 protected:
 	virtual CompressionMethod GetMethod() const															 = 0;
@@ -314,37 +356,77 @@ BlockEncoder::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize)
 	CompressedBlockSizes.reserve(BlockCount);
 	uint64_t CompressedSize = 0;
 	{
-		UniqueBuffer	  RawBlockCopy;
 		MutableMemoryView CompressedBlocksView = CompressedData.GetMutableView() + sizeof(BufferHeader) + MetaSize;
 
-		CompositeBuffer::Iterator It = RawData.GetIterator(0);
-
-		for (uint64_t RawOffset = 0; RawOffset < RawSize;)
+		IoBufferFileReference FileRef = {nullptr, 0, 0};
+		if ((RawData.GetSegments().size() == 1) && RawData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef))
 		{
-			const uint64_t	 RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize);
-			const MemoryView RawBlock	  = RawData.ViewOrCopyRange(It, RawBlockSize, RawBlockCopy);
-			RawHash.Append(RawBlock);
-
-			MutableMemoryView CompressedBlock = CompressedBlocksView;
-			if (!CompressBlock(CompressedBlock, RawBlock))
+			ZEN_ASSERT(FileRef.FileHandle != nullptr);
+			UniqueBuffer RawBlockCopy = UniqueBuffer::Alloc(BlockSize);
+			BasicFile	 Source;
+			Source.Attach(FileRef.FileHandle);
+			for (uint64_t RawOffset = 0; RawOffset < RawSize;)
 			{
-				return CompositeBuffer();
-			}
+				const uint64_t RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize);
+				Source.Read(RawBlockCopy.GetData(), RawBlockSize, FileRef.FileChunkOffset + RawOffset);
+				const MemoryView RawBlock = RawBlockCopy.GetView().Left(RawBlockSize);
+				RawHash.Append(RawBlock);
+				MutableMemoryView CompressedBlock = CompressedBlocksView;
+				if (!CompressBlock(CompressedBlock, RawBlock))
+				{
+					Source.Detach();
+					return CompositeBuffer();
+				}
 
-			uint64_t CompressedBlockSize = CompressedBlock.GetSize();
-			if (RawBlockSize <= CompressedBlockSize)
-			{
-				CompressedBlockSize	 = RawBlockSize;
-				CompressedBlocksView = CompressedBlocksView.CopyFrom(RawBlock);
+				uint64_t CompressedBlockSize = CompressedBlock.GetSize();
+				if (RawBlockSize <= CompressedBlockSize)
+				{
+					CompressedBlockSize	 = RawBlockSize;
+					CompressedBlocksView = CompressedBlocksView.CopyFrom(RawBlock);
+				}
+				else
+				{
+					CompressedBlocksView += CompressedBlockSize;
+				}
+
+				CompressedBlockSizes.push_back(static_cast<uint32_t>(CompressedBlockSize));
+				CompressedSize += CompressedBlockSize;
+				RawOffset += RawBlockSize;
 			}
-			else
+			Source.Detach();
+		}
+		else
+		{
+			UniqueBuffer			  RawBlockCopy;
+			CompositeBuffer::Iterator It = RawData.GetIterator(0);
+
+			for (uint64_t RawOffset = 0; RawOffset < RawSize;)
 			{
-				CompressedBlocksView += CompressedBlockSize;
-			}
+				const uint64_t	 RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize);
+				const MemoryView RawBlock	  = RawData.ViewOrCopyRange(It, RawBlockSize, RawBlockCopy);
+				RawHash.Append(RawBlock);
+
+				MutableMemoryView CompressedBlock = CompressedBlocksView;
+				if (!CompressBlock(CompressedBlock, RawBlock))
+				{
+					return CompositeBuffer();
+				}
+
+				uint64_t CompressedBlockSize = CompressedBlock.GetSize();
+				if (RawBlockSize <= CompressedBlockSize)
+				{
+					CompressedBlockSize	 = RawBlockSize;
+					CompressedBlocksView = CompressedBlocksView.CopyFrom(RawBlock);
+				}
+				else
+				{
+					CompressedBlocksView += CompressedBlockSize;
+				}
 
-			CompressedBlockSizes.push_back(static_cast<uint32_t>(CompressedBlockSize));
-			CompressedSize += CompressedBlockSize;
-			RawOffset += RawBlockSize;
+				CompressedBlockSizes.push_back(static_cast<uint32_t>(CompressedBlockSize));
+				CompressedSize += CompressedBlockSize;
+				RawOffset += RawBlockSize;
+			}
 		}
 	}
 
@@ -377,6 +459,133 @@ BlockEncoder::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize)
 	return CompositeBuffer(SharedBuffer::MakeView(CompositeView, CompressedData.MoveToShared()));
 }
 
+bool
+BlockEncoder::CompressToStream(const CompositeBuffer&												RawData,
+							   std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
+							   uint64_t																BlockSize = DefaultBlockSize) const
+{
+	ZEN_ASSERT(IsPow2(BlockSize) && (BlockSize <= (1u << 31)));
+
+	const uint64_t RawSize = RawData.GetSize();
+	BLAKE3Stream   RawHash;
+
+	const uint64_t BlockCount = RoundUp(RawSize, BlockSize) / BlockSize;
+	ZEN_ASSERT(BlockCount <= ~uint32_t(0));
+
+	const uint64_t MetaSize		  = BlockCount * sizeof(uint32_t);
+	const uint64_t FullHeaderSize = sizeof(BufferHeader) + MetaSize;
+
+	std::vector<uint32_t> CompressedBlockSizes;
+	CompressedBlockSizes.reserve(BlockCount);
+	uint64_t CompressedSize = 0;
+	{
+		UniqueBuffer CompressedBlockBuffer = UniqueBuffer::Alloc(GetCompressedBlocksBound(1, BlockSize, Min(RawSize, BlockSize)));
+
+		IoBufferFileReference FileRef = {nullptr, 0, 0};
+		if ((RawData.GetSegments().size() == 1) && RawData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef))
+		{
+			ZEN_ASSERT(FileRef.FileHandle != nullptr);
+			UniqueBuffer RawBlockCopy = UniqueBuffer::Alloc(BlockSize);
+			BasicFile	 Source;
+			Source.Attach(FileRef.FileHandle);
+			for (uint64_t RawOffset = 0; RawOffset < RawSize;)
+			{
+				const uint64_t RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize);
+				Source.Read(RawBlockCopy.GetData(), RawBlockSize, FileRef.FileChunkOffset + RawOffset);
+				const MemoryView RawBlock = RawBlockCopy.GetView().Left(RawBlockSize);
+				RawHash.Append(RawBlock);
+				MutableMemoryView CompressedBlock = CompressedBlockBuffer.GetMutableView();
+				if (!CompressBlock(CompressedBlock, RawBlock))
+				{
+					Source.Detach();
+					return false;
+				}
+
+				uint64_t CompressedBlockSize = CompressedBlock.GetSize();
+				if (RawBlockSize <= CompressedBlockSize)
+				{
+					Callback(FullHeaderSize + CompressedSize,
+							 CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawBlockCopy.GetView().GetData(), RawBlockSize)));
+					CompressedBlockSize = RawBlockSize;
+				}
+				else
+				{
+					Callback(FullHeaderSize + CompressedSize,
+							 CompositeBuffer(IoBuffer(IoBuffer::Wrap, CompressedBlock.GetData(), CompressedBlockSize)));
+				}
+
+				CompressedBlockSizes.push_back(static_cast<uint32_t>(CompressedBlockSize));
+				CompressedSize += CompressedBlockSize;
+				RawOffset += RawBlockSize;
+			}
+			Source.Detach();
+		}
+		else
+		{
+			UniqueBuffer			  RawBlockCopy;
+			CompositeBuffer::Iterator It = RawData.GetIterator(0);
+
+			for (uint64_t RawOffset = 0; RawOffset < RawSize;)
+			{
+				const uint64_t	 RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize);
+				const MemoryView RawBlock	  = RawData.ViewOrCopyRange(It, RawBlockSize, RawBlockCopy);
+				RawHash.Append(RawBlock);
+
+				MutableMemoryView CompressedBlock = CompressedBlockBuffer.GetMutableView();
+				if (!CompressBlock(CompressedBlock, RawBlock))
+				{
+					return false;
+				}
+
+				uint64_t CompressedBlockSize = CompressedBlock.GetSize();
+				if (RawBlockSize <= CompressedBlockSize)
+				{
+					Callback(FullHeaderSize + CompressedSize, CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawBlock.GetData(), RawBlockSize)));
+					CompressedBlockSize = RawBlockSize;
+				}
+				else
+				{
+					Callback(FullHeaderSize + CompressedSize,
+							 CompositeBuffer(IoBuffer(IoBuffer::Wrap, CompressedBlock.GetData(), CompressedBlockSize)));
+				}
+
+				CompressedBlockSizes.push_back(static_cast<uint32_t>(CompressedBlockSize));
+				CompressedSize += CompressedBlockSize;
+				RawOffset += RawBlockSize;
+			}
+		}
+	}
+
+	// Return failure if the compressed data is larger than the raw data.
+	if (RawSize <= MetaSize + CompressedSize)
+	{
+		return false;
+	}
+
+	// Write the header and calculate the CRC-32.
+	for (uint32_t& Size : CompressedBlockSizes)
+	{
+		Size = ByteSwap(Size);
+	}
+	UniqueBuffer HeaderBuffer = UniqueBuffer::Alloc(sizeof(BufferHeader) + MetaSize);
+
+	BufferHeader Header;
+	Header.Method			   = GetMethod();
+	Header.Compressor		   = GetCompressor();
+	Header.CompressionLevel	   = GetCompressionLevel();
+	Header.BlockSizeExponent   = static_cast<uint8_t>(zen::FloorLog2_64(BlockSize));
+	Header.BlockCount		   = static_cast<uint32_t>(BlockCount);
+	Header.TotalRawSize		   = RawSize;
+	Header.TotalCompressedSize = sizeof(BufferHeader) + MetaSize + CompressedSize;
+	Header.RawHash			   = RawHash.GetHash();
+
+	HeaderBuffer.GetMutableView().Mid(sizeof(BufferHeader), MetaSize).CopyFrom(MakeMemoryView(CompressedBlockSizes));
+	Header.Write(HeaderBuffer.GetMutableView());
+
+	Callback(0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderBuffer.GetData(), HeaderBuffer.GetSize())));
+	return true;
+}
+
 class BlockDecoder : public BaseDecoder
 {
 public:
@@ -406,6 +615,12 @@ public:
 									   MutableMemoryView	  RawView,
 									   uint64_t				  RawOffset) const final;
 
+	virtual bool DecompressToStream(const BufferHeader&													 Header,
+									const CompositeBuffer&												 CompressedData,
+									uint64_t															 RawOffset,
+									uint64_t															 RawSize,
+									std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const final;
+
 protected:
 	virtual bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const = 0;
 };
@@ -528,6 +743,159 @@ BlockDecoder::DecompressToComposite(const BufferHeader& Header, const CompositeB
 }
 
 bool
+BlockDecoder::DecompressToStream(const BufferHeader&												  Header,
+								 const CompositeBuffer&												  CompressedData,
+								 uint64_t															  RawOffset,
+								 uint64_t															  RawSize,
+								 std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const
+{
+	if (Header.TotalCompressedSize != CompressedData.GetSize())
+	{
+		return false;
+	}
+
+	const uint64_t BlockSize = uint64_t(1) << Header.BlockSizeExponent;
+
+	UniqueBuffer BlockSizeBuffer;
+	MemoryView BlockSizeView = CompressedData.ViewOrCopyRange(sizeof(BufferHeader), Header.BlockCount * sizeof(uint32_t), BlockSizeBuffer);
+	std::span<uint32_t const> CompressedBlockSizes(reinterpret_cast<const uint32_t*>(BlockSizeView.GetData()), Header.BlockCount);
+
+	UniqueBuffer CompressedBlockCopy;
+
+	const size_t   FirstBlockIndex	  = uint64_t(RawOffset / BlockSize);
+	const size_t   LastBlockIndex	  = uint64_t((RawOffset + RawSize - 1) / BlockSize);
+	const uint64_t LastBlockSize	  = BlockSize - ((Header.BlockCount * BlockSize) - Header.TotalRawSize);
+	uint64_t	   OffsetInFirstBlock = RawOffset % BlockSize;
+	uint64_t	   CompressedOffset	  = sizeof(BufferHeader) + uint64_t(Header.BlockCount) * sizeof(uint32_t);
+	uint64_t	   RemainingRawSize	  = RawSize;
+
+	for (size_t BlockIndex = 0; BlockIndex < FirstBlockIndex; BlockIndex++)
+	{
+		const uint32_t CompressedBlockSize = ByteSwap(CompressedBlockSizes[BlockIndex]);
+		CompressedOffset += CompressedBlockSize;
+	}
+
+	UniqueBuffer RawDataBuffer;
+
+	IoBufferFileReference FileRef = {nullptr, 0, 0};
+	if ((CompressedData.GetSegments().size() == 1) && CompressedData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef))
+	{
+		ZEN_ASSERT(FileRef.FileHandle != nullptr);
+		BasicFile Source;
+		Source.Attach(FileRef.FileHandle);
+
+		for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++)
+		{
+			const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize;
+			const uint32_t CompressedBlockSize	 = ByteSwap(CompressedBlockSizes[BlockIndex]);
+			const bool	   IsCompressed			 = CompressedBlockSize < UncompressedBlockSize;
+
+			const uint64_t BytesToUncompress = OffsetInFirstBlock > 0 ? zen::Min(RawSize, UncompressedBlockSize - OffsetInFirstBlock)
+																	  : zen::Min(RemainingRawSize, BlockSize);
+
+			if (CompressedBlockCopy.GetSize() < CompressedBlockSize)
+			{
+				CompressedBlockCopy = UniqueBuffer::Alloc(CompressedBlockSize);
+			}
+			Source.Read(CompressedBlockCopy.GetData(), CompressedBlockSize, FileRef.FileChunkOffset + CompressedOffset);
+
+			MemoryView CompressedBlock = CompressedBlockCopy.GetView().Left(CompressedBlockSize);
+
+			if (IsCompressed)
+			{
+				if (RawDataBuffer.IsNull())
+				{
+					RawDataBuffer = UniqueBuffer::Alloc(zen::Min(RawSize, UncompressedBlockSize));
+				}
+				else
+				{
+					ZEN_ASSERT(RawDataBuffer.GetSize() >= UncompressedBlockSize);
+				}
+				MutableMemoryView UncompressedBlock = RawDataBuffer.GetMutableView().Left(UncompressedBlockSize);
+				if (!DecompressBlock(UncompressedBlock, CompressedBlock))
+				{
+					Source.Detach();
+					return false;
+				}
+				if (!Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
+							  CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress))))
+				{
+					Source.Detach();
+					return false;
+				}
+			}
+			else
+			{
+				if (!Callback(
+						BlockIndex * BlockSize + OffsetInFirstBlock,
+						CompositeBuffer(
+							IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress))))
+				{
+					Source.Detach();
+					return false;
+				}
+			}
+
+			OffsetInFirstBlock = 0;
+			RemainingRawSize -= BytesToUncompress;
+			CompressedOffset += CompressedBlockSize;
+		}
+		Source.Detach();
+	}
+	else
+	{
+		for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++)
+		{
+			const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize;
+			const uint32_t CompressedBlockSize	 = ByteSwap(CompressedBlockSizes[BlockIndex]);
+			const bool	   IsCompressed			 = CompressedBlockSize < UncompressedBlockSize;
+
+			const uint64_t BytesToUncompress = OffsetInFirstBlock > 0 ? zen::Min(RawSize, UncompressedBlockSize - OffsetInFirstBlock)
+																	  : zen::Min(RemainingRawSize, BlockSize);
+
+			MemoryView CompressedBlock = CompressedData.ViewOrCopyRange(CompressedOffset, CompressedBlockSize, CompressedBlockCopy);
+
+			if (IsCompressed)
+			{
+				if (RawDataBuffer.IsNull())
+				{
+					RawDataBuffer = UniqueBuffer::Alloc(zen::Min(RawSize, UncompressedBlockSize));
+				}
+				else
+				{
+					ZEN_ASSERT(RawDataBuffer.GetSize() >= UncompressedBlockSize);
+				}
+				MutableMemoryView UncompressedBlock = RawDataBuffer.GetMutableView().Left(UncompressedBlockSize);
+				if (!DecompressBlock(UncompressedBlock, CompressedBlock))
+				{
+					return false;
+				}
+				if (!Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
+							  CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress))))
+				{
+					return false;
+				}
+			}
+			else
+			{
+				if (!Callback(
+						BlockIndex * BlockSize + OffsetInFirstBlock,
+						CompositeBuffer(
+							IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress))))
+				{
+					return false;
+				}
+			}
+
+			OffsetInFirstBlock = 0;
+			RemainingRawSize -= BytesToUncompress;
+			CompressedOffset += CompressedBlockSize;
+		}
+	}
+	return true;
+}
+
+bool
 BlockDecoder::TryDecompressTo(const BufferHeader&	 Header,
 							  const CompositeBuffer& CompressedData,
 							  MutableMemoryView		 RawView,
@@ -560,51 +928,118 @@ BlockDecoder::TryDecompressTo(const BufferHeader&	 Header,
 		CompressedOffset += CompressedBlockSize;
 	}
 
-	for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++)
+	IoBufferFileReference FileRef = {nullptr, 0, 0};
+	if ((CompressedData.GetSegments().size() == 1) && CompressedData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef))
 	{
-		const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize;
-		const uint32_t CompressedBlockSize	 = ByteSwap(CompressedBlockSizes[BlockIndex]);
-		const bool	   IsCompressed			 = CompressedBlockSize < UncompressedBlockSize;
+		ZEN_ASSERT(FileRef.FileHandle != nullptr);
+		BasicFile Source;
+		Source.Attach(FileRef.FileHandle);
+
+		for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++)
+		{
+			const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize;
+			const uint32_t CompressedBlockSize	 = ByteSwap(CompressedBlockSizes[BlockIndex]);
+			const bool	   IsCompressed			 = CompressedBlockSize < UncompressedBlockSize;
 
-		const uint64_t BytesToUncompress = OffsetInFirstBlock > 0 ? zen::Min(RawView.GetSize(), UncompressedBlockSize - OffsetInFirstBlock)
-																  : zen::Min(RemainingRawSize, BlockSize);
+			const uint64_t BytesToUncompress = OffsetInFirstBlock > 0
+												   ? zen::Min(RawView.GetSize(), UncompressedBlockSize - OffsetInFirstBlock)
+												   : zen::Min(RemainingRawSize, BlockSize);
 
-		MemoryView CompressedBlock = CompressedData.ViewOrCopyRange(CompressedOffset, CompressedBlockSize, CompressedBlockCopy);
+			if (CompressedBlockCopy.GetSize() < CompressedBlockSize)
+			{
+				CompressedBlockCopy = UniqueBuffer::Alloc(CompressedBlockSize);
+			}
+			Source.Read(CompressedBlockCopy.GetData(), CompressedBlockSize, FileRef.FileChunkOffset + CompressedOffset);
 
-		if (IsCompressed)
-		{
-			MutableMemoryView UncompressedBlock = RawView.Left(BytesToUncompress);
+			MemoryView CompressedBlock = CompressedBlockCopy.GetView().Left(CompressedBlockSize);
 
-			const bool IsAligned = BytesToUncompress == UncompressedBlockSize;
-			if (!IsAligned)
+			if (IsCompressed)
 			{
-				// Decompress to a temporary buffer when the first or the last block reads are not aligned with the block boundaries.
-				if (UncompressedBlockCopy.IsNull())
+				MutableMemoryView UncompressedBlock = RawView.Left(BytesToUncompress);
+
+				const bool IsAligned = BytesToUncompress == UncompressedBlockSize;
+				if (!IsAligned)
 				{
-					UncompressedBlockCopy = UniqueBuffer::Alloc(BlockSize);
+					// Decompress to a temporary buffer when the first or the last block reads are not aligned with the block boundaries.
+					if (UncompressedBlockCopy.IsNull())
+					{
+						UncompressedBlockCopy = UniqueBuffer::Alloc(BlockSize);
+					}
+					UncompressedBlock = UncompressedBlockCopy.GetMutableView().Mid(0, UncompressedBlockSize);
 				}
-				UncompressedBlock = UncompressedBlockCopy.GetMutableView().Mid(0, UncompressedBlockSize);
-			}
 
-			if (!DecompressBlock(UncompressedBlock, CompressedBlock))
-			{
-				return false;
-			}
+				if (!DecompressBlock(UncompressedBlock, CompressedBlock))
+				{
+					Source.Detach();
+					return false;
+				}
 
-			if (!IsAligned)
+				if (!IsAligned)
+				{
+					RawView.CopyFrom(UncompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
+				}
+			}
+			else
 			{
-				RawView.CopyFrom(UncompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
+				RawView.CopyFrom(CompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
 			}
+
+			OffsetInFirstBlock = 0;
+			RemainingRawSize -= BytesToUncompress;
+			CompressedOffset += CompressedBlockSize;
+			RawView += BytesToUncompress;
 		}
-		else
+		Source.Detach();
+	}
+	else
+	{
+		for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++)
 		{
-			RawView.CopyFrom(CompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
-		}
+			const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize;
+			const uint32_t CompressedBlockSize	 = ByteSwap(CompressedBlockSizes[BlockIndex]);
+			const bool	   IsCompressed			 = CompressedBlockSize < UncompressedBlockSize;
 
-		OffsetInFirstBlock = 0;
-		RemainingRawSize -= BytesToUncompress;
-		CompressedOffset += CompressedBlockSize;
-		RawView += BytesToUncompress;
+			const uint64_t BytesToUncompress = OffsetInFirstBlock > 0
+												   ? zen::Min(RawView.GetSize(), UncompressedBlockSize - OffsetInFirstBlock)
+												   : zen::Min(RemainingRawSize, BlockSize);
+
+			MemoryView CompressedBlock = CompressedData.ViewOrCopyRange(CompressedOffset, CompressedBlockSize, CompressedBlockCopy);
+
+			if (IsCompressed)
+			{
+				MutableMemoryView UncompressedBlock = RawView.Left(BytesToUncompress);
+
+				const bool IsAligned = BytesToUncompress == UncompressedBlockSize;
+				if (!IsAligned)
+				{
+					// Decompress to a temporary buffer when the first or the last block reads are not aligned with the block boundaries.
+					if (UncompressedBlockCopy.IsNull())
+					{
+						UncompressedBlockCopy = UniqueBuffer::Alloc(BlockSize);
+					}
+					UncompressedBlock = UncompressedBlockCopy.GetMutableView().Mid(0, UncompressedBlockSize);
+				}
+
+				if (!DecompressBlock(UncompressedBlock, CompressedBlock))
+				{
+					return false;
+				}
+
+				if (!IsAligned)
+				{
+					RawView.CopyFrom(UncompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
+				}
+			}
+			else
+			{
+				RawView.CopyFrom(CompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
+			}
+
+			OffsetInFirstBlock = 0;
+			RemainingRawSize -= BytesToUncompress;
+			CompressedOffset += CompressedBlockSize;
+			RawView += BytesToUncompress;
+		}
 	}
 
 	return RemainingRawSize == 0;
@@ -1342,6 +1777,30 @@ CompressedBuffer::Compress(const SharedBuffer&	 RawData,
 	return Compress(CompositeBuffer(RawData), Compressor, CompressionLevel, BlockSize);
 }
 
+bool
+CompressedBuffer::CompressToStream(const CompositeBuffer&												RawData,
+								   std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
+								   OodleCompressor														Compressor,
+								   OodleCompressionLevel												CompressionLevel,
+								   uint64_t																BlockSize)
+{
+	using namespace detail;
+
+	if (BlockSize == 0)
+	{
+		BlockSize = DefaultBlockSize;
+	}
+
+	if (CompressionLevel == OodleCompressionLevel::None)
+	{
+		return NoneEncoder().CompressToStream(RawData, std::move(Callback), BlockSize);
+	}
+	else
+	{
+		return OodleEncoder(Compressor, CompressionLevel).CompressToStream(RawData, std::move(Callback), BlockSize);
+	}
+}
+
 CompressedBuffer
 CompressedBuffer::FromCompressed(const CompositeBuffer& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
 {
@@ -1536,6 +1995,27 @@ CompressedBuffer::DecompressToComposite() const
 }
 
 bool
+CompressedBuffer::DecompressToStream(uint64_t															  RawOffset,
+									 uint64_t															  RawSize,
+									 std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const
+{
+	using namespace detail;
+	if (CompressedData)
+	{
+		const BufferHeader Header = BufferHeader::Read(CompressedData);
+		if (Header.Magic == BufferHeader::ExpectedMagic)
+		{
+			if (const BaseDecoder* const Decoder = GetDecoder(Header.Method))
+			{
+				const uint64_t TotalRawSize = RawSize < ~uint64_t(0) ? RawSize : Header.TotalRawSize - RawOffset;
+				return Decoder->DecompressToStream(Header, CompressedData, RawOffset, TotalRawSize, std::move(Callback));
+			}
+		}
+	}
+	return false;
+}
+
+bool
 CompressedBuffer::TryGetCompressParameters(OodleCompressor&		  OutCompressor,
 										   OodleCompressionLevel& OutCompressionLevel,
 										   uint64_t&			  OutBlockSize) const
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index b8c35212f..05e2bf049 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -531,7 +531,10 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 }
 
 void
-CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options, std::error_code& OutErrorCode)
+CopyFile(const std::filesystem::path& FromPath,
+		 const std::filesystem::path& ToPath,
+		 const CopyFileOptions&		  Options,
+		 std::error_code&			  OutErrorCode)
 {
 	OutErrorCode.clear();
 
@@ -544,7 +547,7 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 }
 
 bool
-CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options)
+CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToPath, const CopyFileOptions& Options)
 {
 	bool Success = false;
 
@@ -587,7 +590,7 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 	ScopedFd $From = {FromFd};
 
 	// To file
-	int ToFd = open(ToPath.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0666);
+	int ToFd = open(ToPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0666);
 	if (ToFd < 0)
 	{
 		ThrowLastError(fmt::format("failed to create file {}", ToPath));
@@ -595,9 +598,14 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 	fchmod(ToFd, 0666);
 	ScopedFd $To = {ToFd};
 
+	struct stat Stat;
+	fstat(FromFd, &Stat);
+
+	size_t FileSizeBytes = Stat.st_size;
+
 	// Copy impl
-	static const size_t BufferSize = 64 << 10;
-	void*				Buffer	   = malloc(BufferSize);
+	const size_t BufferSize = Min(FileSizeBytes, 64u << 10);
+	void*		 Buffer		= malloc(BufferSize);
 	while (true)
 	{
 		int BytesRead = read(FromFd, Buffer, BufferSize);
@@ -607,7 +615,7 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 			break;
 		}
 
-		if (write(ToFd, Buffer, BytesRead) != BufferSize)
+		if (write(ToFd, Buffer, BytesRead) != BytesRead)
 		{
 			Success = false;
 			break;
@@ -618,7 +626,7 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 
 	if (!Success)
 	{
-		ThrowLastError("file copy failed"sv);
+		ThrowLastError(fmt::format("file copy from {} to {} failed", FromPath, ToPath));
 	}
 
 	return true;
@@ -683,7 +691,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 		{
 		}
 
-		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override
 		{
 			std::error_code				Ec;
 			const std::filesystem::path Relative = std::filesystem::relative(Parent, BasePath, Ec);
@@ -1236,7 +1244,11 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
 			}
 			else
 			{
-				Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart, gsl::narrow<uint32_t>(DirInfo->FileAttributes));
+				Visitor.VisitFile(RootDir,
+								  FileName,
+								  DirInfo->EndOfFile.QuadPart,
+								  gsl::narrow<uint32_t>(DirInfo->FileAttributes),
+								  (uint64_t)DirInfo->LastWriteTime.QuadPart);
 			}
 
 			const uint64_t NextOffset = DirInfo->NextEntryOffset;
@@ -1285,7 +1297,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
 		}
 		else if (S_ISREG(Stat.st_mode))
 		{
-			Visitor.VisitFile(RootDir, FileName, Stat.st_size, gsl::narrow<uint32_t>(Stat.st_mode));
+			Visitor.VisitFile(RootDir, FileName, Stat.st_size, gsl::narrow<uint32_t>(Stat.st_mode), gsl::narrow<uint64_t>(Stat.st_mtime));
 		}
 		else
 		{
@@ -1465,6 +1477,36 @@ GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec)
 	return 0;
 }
 
+uint64_t
+GetModificationTickFromPath(const std::filesystem::path& Filename)
+{
+	// PathFromHandle
+	void* Handle;
+#if ZEN_PLATFORM_WINDOWS
+	Handle = CreateFileW(Filename.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr);
+	if (Handle == INVALID_HANDLE_VALUE)
+	{
+		ThrowLastError(fmt::format("Failed to open file {} to check modification tick.", Filename));
+	}
+	auto			_ = MakeGuard([Handle]() { CloseHandle(Handle); });
+	std::error_code Ec;
+	uint64_t		ModificatonTick = GetModificationTickFromHandle(Handle, Ec);
+	if (Ec)
+	{
+		ThrowSystemError(Ec.value(), Ec.message());
+	}
+	return ModificatonTick;
+#else
+	struct stat Stat;
+	int			err = stat(Filename.native().c_str(), &Stat);
+	if (err)
+	{
+		ThrowLastError(fmt::format("Failed to get mode of file {}", Filename));
+	}
+	return gsl::narrow<uint64_t>(Stat.st_mtime);
+#endif
+}
+
 std::filesystem::path
 GetRunningExecutablePath()
 {
@@ -1544,7 +1586,8 @@ GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags
 		virtual void VisitFile(const std::filesystem::path& Parent,
 							   const path_view&				File,
 							   uint64_t						FileSize,
-							   uint32_t						NativeModeOrAttributes) override
+							   uint32_t						NativeModeOrAttributes,
+							   uint64_t						NativeModificationTick) override
 		{
 			if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles))
 			{
@@ -1557,6 +1600,10 @@ GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags
 				{
 					Content.FileAttributes.push_back(NativeModeOrAttributes);
 				}
+				if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeModificationTick))
+				{
+					Content.FileModificationTicks.push_back(NativeModificationTick);
+				}
 			}
 		}
 
@@ -1612,7 +1659,8 @@ GetDirectoryContent(const std::filesystem::path& RootDir,
 		virtual void VisitFile(const std::filesystem::path&,
 							   const path_view& File,
 							   uint64_t			FileSize,
-							   uint32_t			NativeModeOrAttributes) override
+							   uint32_t			NativeModeOrAttributes,
+							   uint64_t			NativeModificationTick) override
 		{
 			if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles))
 			{
@@ -1625,6 +1673,10 @@ GetDirectoryContent(const std::filesystem::path& RootDir,
 				{
 					Content.FileAttributes.push_back(NativeModeOrAttributes);
 				}
+				if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeModificationTick))
+				{
+					Content.FileModificationTicks.push_back(NativeModificationTick);
+				}
 			}
 		}
 
@@ -1881,6 +1933,129 @@ PickDefaultSystemRootDirectory()
 #endif	// ZEN_PLATFORM_WINDOWS
 }
 
+#if ZEN_PLATFORM_WINDOWS
+
+uint32_t
+GetFileAttributes(const std::filesystem::path& Filename)
+{
+	DWORD Attributes = ::GetFileAttributes(Filename.native().c_str());
+	if (Attributes == INVALID_FILE_ATTRIBUTES)
+	{
+		ThrowLastError(fmt::format("failed to get attributes of file {}", Filename));
+	}
+	return (uint32_t)Attributes;
+}
+
+void
+SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes)
+{
+	if (::SetFileAttributes(Filename.native().c_str(), Attributes) == 0)
+	{
+		ThrowLastError(fmt::format("failed to set attributes of file {}", Filename));
+	}
+}
+
+#endif	// ZEN_PLATFORM_WINDOWS
+
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+
+uint32_t
+GetFileMode(const std::filesystem::path& Filename)
+{
+	struct stat Stat;
+	int			err = stat(Filename.native().c_str(), &Stat);
+	if (err)
+	{
+		ThrowLastError(fmt::format("Failed to get mode of file {}", Filename));
+	}
+	return (uint32_t)Stat.st_mode;
+}
+
+void
+SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes)
+{
+	int err = chmod(Filename.native().c_str(), (mode_t)Attributes);
+	if (err)
+	{
+		ThrowLastError(fmt::format("Failed to set mode of file {}", Filename));
+	}
+}
+
+#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+
+#if ZEN_PLATFORM_WINDOWS
+const uint32_t FileAttributesSystemReadOnlyFlag = FILE_ATTRIBUTE_READONLY;
+#else
+const uint32_t FileAttributesSystemReadOnlyFlag = 0x00000001;
+#endif	// ZEN_PLATFORM_WINDOWS
+
+const uint32_t FileModeWriteEnableFlags = 0222;
+
+bool
+IsFileAttributeReadOnly(uint32_t FileAttributes)
+{
+#if ZEN_PLATFORM_WINDOWS
+	return (FileAttributes & FileAttributesSystemReadOnlyFlag) != 0;
+#else
+	return (FileAttributes & 0x00000001) != 0;
+#endif	// ZEN_PLATFORM_WINDOWS
+}
+
+bool
+IsFileModeReadOnly(uint32_t FileMode)
+{
+	return (FileMode & FileModeWriteEnableFlags) == 0;
+}
+
+uint32_t
+MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly)
+{
+	return ReadOnly ? (FileAttributes | FileAttributesSystemReadOnlyFlag) : (FileAttributes & ~FileAttributesSystemReadOnlyFlag);
+}
+
+uint32_t
+MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly)
+{
+	return ReadOnly ? (FileMode & ~FileModeWriteEnableFlags) : (FileMode | FileModeWriteEnableFlags);
+}
+
+bool
+SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly)
+{
+#if ZEN_PLATFORM_WINDOWS
+	uint32_t CurrentAttributes = GetFileAttributes(Filename);
+	uint32_t NewAttributes	   = MakeFileAttributeReadOnly(CurrentAttributes, ReadOnly);
+	if (CurrentAttributes != NewAttributes)
+	{
+		SetFileAttributes(Filename, NewAttributes);
+		return true;
+	}
+#endif	// ZEN_PLATFORM_WINDOWS
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	uint32_t CurrentMode = GetFileMode(Filename);
+	uint32_t NewMode	 = MakeFileModeReadOnly(CurrentMode, ReadOnly);
+	if (CurrentMode != NewMode)
+	{
+		SetFileMode(Filename, NewMode);
+		return true;
+	}
+#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	return false;
+}
+
+std::filesystem::path
+StringToPath(const std::string_view& Path)
+{
+	if (Path.length() > 2 && Path.front() == '\"' && Path.back() == '\"')
+	{
+		return std::filesystem::path(Path.substr(1, Path.length() - 2)).make_preferred();
+	}
+	else
+	{
+		return std::filesystem::path(Path).make_preferred();
+	}
+}
+
 //////////////////////////////////////////////////////////////////////////
 //
 // Testing related code follows...
@@ -1928,7 +2103,7 @@ TEST_CASE("filesystem")
 	// Traversal
 	struct : public FileSystemTraversal::TreeVisitor
 	{
-		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t, uint32_t) override
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t, uint32_t, uint64_t) override
 		{
 			bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected);
 		}
diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h
index 03c5605df..57798b6f4 100644
--- a/src/zencore/include/zencore/basicfile.h
+++ b/src/zencore/include/zencore/basicfile.h
@@ -46,6 +46,10 @@ public:
 		kPreventWrite  = 0x2000'0000,  // Do not open with write sharing mode (prevent other processes from writing to file while open)
 	};
 
+	BasicFile(const std::filesystem::path& FileName, Mode Mode);
+	BasicFile(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec);
+	BasicFile(const std::filesystem::path& FileName, Mode Mode, std::function<bool(std::error_code& Ec)>&& RetryCallback);
+
 	void	 Open(const std::filesystem::path& FileName, Mode Mode);
 	void	 Open(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec);
 	void	 Open(const std::filesystem::path& FileName, Mode Mode, std::function<bool(std::error_code& Ec)>&& RetryCallback);
@@ -56,7 +60,8 @@ public:
 	void	 StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
 	void	 Write(MemoryView Data, uint64_t FileOffset);
 	void	 Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec);
-	uint64_t Write(CompositeBuffer Data, uint64_t FileOffset, std::error_code& Ec);
+	uint64_t Write(const CompositeBuffer& Data, uint64_t FileOffset);
+	uint64_t Write(const CompositeBuffer& Data, uint64_t FileOffset, std::error_code& Ec);
 	void	 Write(const void* Data, uint64_t Size, uint64_t FileOffset);
 	void	 Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec);
 	void	 Flush();
@@ -170,6 +175,7 @@ public:
 	~BasicFileWriter();
 
 	void Write(const void* Data, uint64_t Size, uint64_t FileOffset);
+	void Write(const CompositeBuffer& Data, uint64_t FileOffset);
 	void Flush();
 
 private:
@@ -180,6 +186,8 @@ private:
 	uint64_t	   m_BufferEnd;
 };
 
+IoBuffer WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path);
+
 ZENCORE_API void basicfile_forcelink();
 
 }  // namespace zen
diff --git a/src/zencore/include/zencore/compactbinarybuilder.h b/src/zencore/include/zencore/compactbinarybuilder.h
index 1c625cacc..f11717453 100644
--- a/src/zencore/include/zencore/compactbinarybuilder.h
+++ b/src/zencore/include/zencore/compactbinarybuilder.h
@@ -18,6 +18,8 @@
 #include <type_traits>
 #include <vector>
 
+#include <EASTL/fixed_vector.h>
+
 #include <gsl/gsl-lite.hpp>
 
 namespace zen {
@@ -367,6 +369,8 @@ public:
 	/** Private flags that are public to work with ENUM_CLASS_FLAGS. */
 	enum class StateFlags : uint8_t;
 
+	typedef eastl::fixed_vector<uint8_t, 2048> CbWriterData_t;
+
 protected:
 	/** Reserve the specified size up front until the format is optimized. */
 	ZENCORE_API explicit CbWriter(int64_t InitialSize);
@@ -409,8 +413,8 @@ private:
 	// provided externally, such as on the stack. That format will store the offsets that require
 	// object or array sizes to be inserted and field types to be removed, and will perform those
 	// operations only when saving to a buffer.
-	std::vector<uint8_t>	 Data;
-	std::vector<WriterState> States;
+	eastl::fixed_vector<WriterState, 4> States;
+	CbWriterData_t						Data;
 };
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/src/zencore/include/zencore/compactbinaryfmt.h b/src/zencore/include/zencore/compactbinaryfmt.h
new file mode 100644
index 000000000..ae0c3eb42
--- /dev/null
+++ b/src/zencore/include/zencore/compactbinaryfmt.h
@@ -0,0 +1,23 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compactbinary.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#include <string_view>
+
+template<typename T>
+requires DerivedFrom<T, zen::CbObjectView>
+struct fmt::formatter<T> : fmt::formatter<std::string_view>
+{
+	auto format(const zen::CbObject& a, format_context& ctx) const
+	{
+		zen::ExtendableStringBuilder<1024> ObjStr;
+		zen::CompactBinaryToJson(a, ObjStr);
+		return fmt::formatter<std::string_view>::format(ObjStr.ToView(), ctx);
+	}
+};
diff --git a/src/zencore/include/zencore/compactbinarypackage.h b/src/zencore/include/zencore/compactbinarypackage.h
index 12fcc41b7..9ec12cb0f 100644
--- a/src/zencore/include/zencore/compactbinarypackage.h
+++ b/src/zencore/include/zencore/compactbinarypackage.h
@@ -12,6 +12,8 @@
 #include <span>
 #include <variant>
 
+#include <EASTL/fixed_vector.h>
+
 #ifdef GetObject
 #	error "windows.h pollution"
 #	undef GetObject
@@ -265,7 +267,10 @@ public:
 	}
 
 	/** Returns the attachments in this package. */
-	inline std::span<const CbAttachment> GetAttachments() const { return Attachments; }
+	inline std::span<const CbAttachment> GetAttachments() const
+	{
+		return std::span<const CbAttachment>(begin(Attachments), end(Attachments));
+	}
 
 	/**
 	 * Find an attachment by its hash.
@@ -286,6 +291,8 @@ public:
 
 	void AddAttachments(std::span<const CbAttachment> Attachments);
 
+	void ReserveAttachments(size_t Count);
+
 	/**
 	 * Remove an attachment by hash.
 	 *
@@ -324,9 +331,9 @@ private:
 	void GatherAttachments(const CbObject& Object, AttachmentResolver Resolver);
 
 	/** Attachments ordered by their hash. */
-	std::vector<CbAttachment> Attachments;
-	CbObject				  Object;
-	IoHash					  ObjectHash;
+	eastl::fixed_vector<CbAttachment, 32> Attachments;
+	CbObject							  Object;
+	IoHash								  ObjectHash;
 };
 
 namespace legacy {
diff --git a/src/zencore/include/zencore/compositebuffer.h b/src/zencore/include/zencore/compositebuffer.h
index b435c5e74..1e1611de9 100644
--- a/src/zencore/include/zencore/compositebuffer.h
+++ b/src/zencore/include/zencore/compositebuffer.h
@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <zencore/eastlutil.h>
 #include <zencore/sharedbuffer.h>
 #include <zencore/zencore.h>
 
@@ -9,6 +10,8 @@
 #include <span>
 #include <vector>
 
+#include <EASTL/fixed_vector.h>
+
 namespace zen {
 
 /**
@@ -35,7 +38,7 @@ public:
 		{
 			m_Segments.reserve((GetBufferCount(std::forward<BufferTypes>(Buffers)) + ...));
 			(AppendBuffers(std::forward<BufferTypes>(Buffers)), ...);
-			std::erase_if(m_Segments, [](const SharedBuffer& It) { return It.IsNull(); });
+			erase_if(m_Segments, [](const SharedBuffer& It) { return It.IsNull(); });
 		}
 	}
 
@@ -46,7 +49,10 @@ public:
 	[[nodiscard]] ZENCORE_API uint64_t GetSize() const;
 
 	/** Returns the segments that the buffer is composed from. */
-	[[nodiscard]] inline std::span<const SharedBuffer> GetSegments() const { return std::span<const SharedBuffer>{m_Segments}; }
+	[[nodiscard]] inline std::span<const SharedBuffer> GetSegments() const
+	{
+		return std::span<const SharedBuffer>{begin(m_Segments), end(m_Segments)};
+	}
 
 	/** Returns true if the composite buffer is not null. */
 	[[nodiscard]] inline explicit operator bool() const { return !IsNull(); }
@@ -120,6 +126,8 @@ public:
 	static const CompositeBuffer Null;
 
 private:
+	typedef eastl::fixed_vector<SharedBuffer, 4> SharedBufferVector_t;
+
 	static inline size_t GetBufferCount(const CompositeBuffer& Buffer) { return Buffer.m_Segments.size(); }
 	inline void			 AppendBuffers(const CompositeBuffer& Buffer)
 	{
@@ -134,12 +142,25 @@ private:
 	inline void			 AppendBuffers(SharedBuffer&& Buffer) { m_Segments.push_back(std::move(Buffer)); }
 	inline void			 AppendBuffers(IoBuffer&& Buffer) { m_Segments.push_back(SharedBuffer(std::move(Buffer))); }
 
+	static inline size_t GetBufferCount(std::span<IoBuffer>&& Container) { return Container.size(); }
+	inline void			 AppendBuffers(std::span<IoBuffer>&& Container)
+	{
+		m_Segments.reserve(m_Segments.size() + Container.size());
+		for (IoBuffer& Buffer : Container)
+		{
+			m_Segments.emplace_back(SharedBuffer(std::move(Buffer)));
+		}
+	}
+
 	static inline size_t GetBufferCount(std::vector<SharedBuffer>&& Container) { return Container.size(); }
 	static inline size_t GetBufferCount(std::vector<IoBuffer>&& Container) { return Container.size(); }
 	inline void			 AppendBuffers(std::vector<SharedBuffer>&& Container)
 	{
 		m_Segments.reserve(m_Segments.size() + Container.size());
-		m_Segments.insert(m_Segments.end(), std::make_move_iterator(Container.begin()), std::make_move_iterator(Container.end()));
+		for (SharedBuffer& Buffer : Container)
+		{
+			m_Segments.emplace_back(std::move(Buffer));
+		}
 	}
 	inline void AppendBuffers(std::vector<IoBuffer>&& Container)
 	{
@@ -150,8 +171,17 @@ private:
 		}
 	}
 
+	inline void AppendBuffers(SharedBufferVector_t&& Container)
+	{
+		m_Segments.reserve(m_Segments.size() + Container.size());
+		for (SharedBuffer& Buffer : Container)
+		{
+			m_Segments.emplace_back(std::move(Buffer));
+		}
+	}
+
 private:
-	std::vector<SharedBuffer> m_Segments;
+	SharedBufferVector_t m_Segments;
 };
 
 void compositebuffer_forcelink();  // internal
diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h
index 5e761ceef..74fd5f767 100644
--- a/src/zencore/include/zencore/compress.h
+++ b/src/zencore/include/zencore/compress.h
@@ -74,6 +74,11 @@ public:
 															   OodleCompressor		 Compressor		  = OodleCompressor::Mermaid,
 															   OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast,
 															   uint64_t				 BlockSize		  = 0);
+	[[nodiscard]] ZENCORE_API static bool			  CompressToStream(const CompositeBuffer&												RawData,
+																	   std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
+																	   OodleCompressor		 Compressor		  = OodleCompressor::Mermaid,
+																	   OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast,
+																	   uint64_t				 BlockSize		  = 0);
 
 	/**
 	 * Construct from a compressed buffer previously created by Compress().
@@ -196,6 +201,16 @@ public:
 	 */
 	[[nodiscard]] ZENCORE_API CompositeBuffer DecompressToComposite() const;
 
+	/**
+	 * Decompress into and call callback for ranges of decompressed data.
+	 * The buffer in the callback will be overwritten when the callback returns.
+	 *
+	 * @return True if the buffer is valid and can be decompressed.
+	 */
+	[[nodiscard]] ZENCORE_API bool DecompressToStream(uint64_t															   RawOffset,
+													  uint64_t															   RawSize,
+													  std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const;
+
 	/** A null compressed buffer. */
 	static const CompressedBuffer Null;
 
diff --git a/src/zencore/include/zencore/eastlutil.h b/src/zencore/include/zencore/eastlutil.h
new file mode 100644
index 000000000..642321dae
--- /dev/null
+++ b/src/zencore/include/zencore/eastlutil.h
@@ -0,0 +1,20 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <algorithm>
+
+namespace zen {
+
+size_t
+erase_if(auto& _Cont, auto Predicate)
+{
+	auto	   _First	 = _Cont.begin();
+	const auto _Last	 = _Cont.end();
+	const auto _Old_size = _Cont.size();
+	_First				 = std::remove_if(_First, _Last, Predicate);
+	_Cont.erase(_First, _Last);
+	return _Old_size - _Cont.size();
+}
+
+}  // namespace zen
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index ca8682cd7..9a2b15d1d 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -52,6 +52,10 @@ ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle);
  */
 ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec);
 
+/** Get a native time tick of last modification time
+ */
+ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename);
+
 ZENCORE_API std::filesystem::path GetRunningExecutablePath();
 
 /** Set the max open file handle count to max allowed for the current process on Linux and MacOS
@@ -93,11 +97,11 @@ struct CopyFileOptions
 	bool MustClone	 = false;
 };
 
-ZENCORE_API bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options);
-ZENCORE_API void CopyFile(std::filesystem::path	 FromPath,
-						  std::filesystem::path	 ToPath,
-						  const CopyFileOptions& Options,
-						  std::error_code&		 OutError);
+ZENCORE_API bool CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToPath, const CopyFileOptions& Options);
+ZENCORE_API void CopyFile(const std::filesystem::path& FromPath,
+						  const std::filesystem::path& ToPath,
+						  const CopyFileOptions&	   Options,
+						  std::error_code&			   OutError);
 ZENCORE_API void CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options);
 ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path);
 
@@ -203,7 +207,8 @@ public:
 		virtual void VisitFile(const std::filesystem::path& Parent,
 							   const path_view&				File,
 							   uint64_t						FileSize,
-							   uint32_t						NativeModeOrAttributes) = 0;
+							   uint32_t						NativeModeOrAttributes,
+							   uint64_t						NativeModificationTick) = 0;
 
 		// This should return true if we should recurse into the directory
 		virtual bool VisitDirectory(const std::filesystem::path& Parent,
@@ -216,13 +221,14 @@ public:
 
 enum class DirectoryContentFlags : uint8_t
 {
-	None			  = 0,
-	IncludeDirs		  = 1u << 0,
-	IncludeFiles	  = 1u << 1,
-	Recursive		  = 1u << 2,
-	IncludeFileSizes  = 1u << 3,
-	IncludeAttributes = 1u << 4,
-	IncludeAllEntries = IncludeDirs | IncludeFiles | Recursive
+	None					= 0,
+	IncludeDirs				= 1u << 0,
+	IncludeFiles			= 1u << 1,
+	Recursive				= 1u << 2,
+	IncludeFileSizes		= 1u << 3,
+	IncludeAttributes		= 1u << 4,
+	IncludeModificationTick = 1u << 5,
+	IncludeAllEntries		= IncludeDirs | IncludeFiles | Recursive
 };
 
 ENUM_CLASS_FLAGS(DirectoryContentFlags)
@@ -232,6 +238,7 @@ struct DirectoryContent
 	std::vector<std::filesystem::path> Files;
 	std::vector<uint64_t>			   FileSizes;
 	std::vector<uint32_t>			   FileAttributes;
+	std::vector<uint64_t>			   FileModificationTicks;
 	std::vector<std::filesystem::path> Directories;
 	std::vector<uint32_t>			   DirectoryAttributes;
 };
@@ -246,6 +253,7 @@ public:
 		std::vector<std::filesystem::path> FileNames;
 		std::vector<uint64_t>			   FileSizes;
 		std::vector<uint32_t>			   FileAttributes;
+		std::vector<uint64_t>			   FileModificationTicks;
 		std::vector<std::filesystem::path> DirectoryNames;
 		std::vector<uint32_t>			   DirectoryAttributes;
 	};
@@ -267,6 +275,25 @@ std::error_code RotateDirectories(const std::filesystem::path& DirectoryName, st
 
 std::filesystem::path PickDefaultSystemRootDirectory();
 
+#if ZEN_PLATFORM_WINDOWS
+uint32_t GetFileAttributes(const std::filesystem::path& Filename);
+void	 SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes);
+#endif	// ZEN_PLATFORM_WINDOWS
+
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+uint32_t GetFileMode(const std::filesystem::path& Filename);
+void	 SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes);
+#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+
+bool	 IsFileAttributeReadOnly(uint32_t FileAttributes);
+bool	 IsFileModeReadOnly(uint32_t FileMode);
+uint32_t MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly);
+uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly);
+
+bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly);
+
+std::filesystem::path StringToPath(const std::string_view& Path);
+
 //////////////////////////////////////////////////////////////////////////
 
 void filesystem_forcelink();  // internal
diff --git a/src/zencore/include/zencore/iohash.h b/src/zencore/include/zencore/iohash.h
index 8871a5895..7443e17b7 100644
--- a/src/zencore/include/zencore/iohash.h
+++ b/src/zencore/include/zencore/iohash.h
@@ -47,8 +47,8 @@ struct IoHash
 
 	static IoHash	   HashBuffer(const void* data, size_t byteCount);
 	static IoHash	   HashBuffer(MemoryView Data) { return HashBuffer(Data.GetData(), Data.GetSize()); }
-	static IoHash	   HashBuffer(const CompositeBuffer& Buffer);
-	static IoHash	   HashBuffer(const IoBuffer& Buffer);
+	static IoHash	   HashBuffer(const CompositeBuffer& Buffer, std::atomic<uint64_t>* ProcessedBytes = nullptr);
+	static IoHash	   HashBuffer(const IoBuffer& Buffer, std::atomic<uint64_t>* ProcessedBytes = nullptr);
 	static IoHash	   FromHexString(const char* string);
 	static IoHash	   FromHexString(const std::string_view string);
 	static bool		   TryParse(std::string_view Str, IoHash& Hash);
diff --git a/src/zencore/include/zencore/memory/newdelete.h b/src/zencore/include/zencore/memory/newdelete.h
index d22c8604f..059f1d5ea 100644
--- a/src/zencore/include/zencore/memory/newdelete.h
+++ b/src/zencore/include/zencore/memory/newdelete.h
@@ -153,3 +153,29 @@ operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexce
 	return zen_new_aligned_nothrow(n, static_cast<size_t>(al));
 }
 #endif
+
+// EASTL operator new
+
+void*
+operator new[](size_t size, const char* pName, int flags, unsigned debugFlags, const char* file, int line)
+{
+	ZEN_UNUSED(pName, flags, debugFlags, file, line);
+	return zen_new(size);
+}
+
+void*
+operator new[](size_t	   size,
+			   size_t	   alignment,
+			   size_t	   alignmentOffset,
+			   const char* pName,
+			   int		   flags,
+			   unsigned	   debugFlags,
+			   const char* file,
+			   int		   line)
+{
+	ZEN_UNUSED(alignmentOffset, pName, flags, debugFlags, file, line);
+
+	ZEN_ASSERT_SLOW(alignmentOffset == 0);	// currently not supported
+
+	return zen_new_aligned(size, alignment);
+}
diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h
index 36c2a2481..335e3d909 100644
--- a/src/zencore/include/zencore/process.h
+++ b/src/zencore/include/zencore/process.h
@@ -101,6 +101,9 @@ int				 GetProcessId(CreateProcResult ProcId);
 std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc);
 std::error_code		  FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle);
 
+std::vector<std::string> ParseCommandLine(std::string_view CommandLine);
+std::vector<char*>		 StripCommandlineQuotes(std::vector<std::string>& InOutArgs);
+
 #if ZEN_PLATFORM_LINUX
 void IgnoreChildSignals();
 #endif
diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h
index e2ef1c1a0..68129b691 100644
--- a/src/zencore/include/zencore/string.h
+++ b/src/zencore/include/zencore/string.h
@@ -522,6 +522,9 @@ public:
 
 //////////////////////////////////////////////////////////////////////////
 
+bool							 IsValidUtf8(const std::string_view& str);
+std::string_view::const_iterator FindFirstInvalidUtf8Byte(const std::string_view& str);
+
 void		 Utf8ToWide(const char8_t* str, WideStringBuilderBase& out);
 void		 Utf8ToWide(const std::u8string_view& wstr, WideStringBuilderBase& out);
 void		 Utf8ToWide(const std::string_view& wstr, WideStringBuilderBase& out);
diff --git a/src/zencore/include/zencore/timer.h b/src/zencore/include/zencore/timer.h
index e4ddc3505..767dc4314 100644
--- a/src/zencore/include/zencore/timer.h
+++ b/src/zencore/include/zencore/timer.h
@@ -21,6 +21,10 @@ ZENCORE_API uint64_t GetHifreqTimerFrequency();
 ZENCORE_API double	 GetHifreqTimerToSeconds();
 ZENCORE_API uint64_t GetHifreqTimerFrequencySafe();	 // May be used during static init
 
+// Query time since process was spawned (returns time in ms)
+
+uint64_t GetTimeSinceProcessStart();
+
 class Stopwatch
 {
 public:
diff --git a/src/zencore/iohash.cpp b/src/zencore/iohash.cpp
index 7200e6e3f..3b2af0db4 100644
--- a/src/zencore/iohash.cpp
+++ b/src/zencore/iohash.cpp
@@ -30,7 +30,7 @@ IoHash::HashBuffer(const void* data, size_t byteCount)
 }
 
 IoHash
-IoHash::HashBuffer(const CompositeBuffer& Buffer)
+IoHash::HashBuffer(const CompositeBuffer& Buffer, std::atomic<uint64_t>* ProcessedBytes)
 {
 	IoHashStream Hasher;
 
@@ -46,11 +46,21 @@ IoHash::HashBuffer(const CompositeBuffer& Buffer)
 					 FileRef.FileChunkOffset,
 					 FileRef.FileChunkSize,
 					 BufferingSize,
-					 [&Hasher](const void* Data, size_t Size) { Hasher.Append(Data, Size); });
+					 [&Hasher, ProcessedBytes](const void* Data, size_t Size) {
+						 Hasher.Append(Data, Size);
+						 if (ProcessedBytes != nullptr)
+						 {
+							 ProcessedBytes->fetch_add(Size);
+						 }
+					 });
 		}
 		else
 		{
 			Hasher.Append(Segment.GetData(), SegmentSize);
+			if (ProcessedBytes != nullptr)
+			{
+				ProcessedBytes->fetch_add(SegmentSize);
+			}
 		}
 	}
 
@@ -58,7 +68,7 @@ IoHash::HashBuffer(const CompositeBuffer& Buffer)
 }
 
 IoHash
-IoHash::HashBuffer(const IoBuffer& Buffer)
+IoHash::HashBuffer(const IoBuffer& Buffer, std::atomic<uint64_t>* ProcessedBytes)
 {
 	IoHashStream Hasher;
 
@@ -71,11 +81,21 @@ IoHash::HashBuffer(const IoBuffer& Buffer)
 				 FileRef.FileChunkOffset,
 				 FileRef.FileChunkSize,
 				 BufferingSize,
-				 [&Hasher](const void* Data, size_t Size) { Hasher.Append(Data, Size); });
+				 [&Hasher, ProcessedBytes](const void* Data, size_t Size) {
+					 Hasher.Append(Data, Size);
+					 if (ProcessedBytes != nullptr)
+					 {
+						 ProcessedBytes->fetch_add(Size);
+					 }
+				 });
 	}
 	else
 	{
 		Hasher.Append(Buffer.GetData(), BufferSize);
+		if (ProcessedBytes != nullptr)
+		{
+			ProcessedBytes->fetch_add(BufferSize);
+		}
 	}
 
 	return Hasher.GetHash();
diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp
index 079e2db3f..147b00966 100644
--- a/src/zencore/process.cpp
+++ b/src/zencore/process.cpp
@@ -1064,6 +1064,118 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand
 #endif	// ZEN_PLATFORM_LINUX
 }
 
+std::vector<std::string>
+ParseCommandLine(std::string_view CommandLine)
+{
+	auto IsWhitespaceOrEnd = [](std::string_view CommandLine, std::string::size_type Pos) {
+		if (Pos == CommandLine.length())
+		{
+			return true;
+		}
+		if (CommandLine[Pos] == ' ')
+		{
+			return true;
+		}
+		return false;
+	};
+
+	bool IsParsingArg = false;
+	bool IsInQuote	  = false;
+
+	std::string::size_type	 Pos	  = 0;
+	std::string::size_type	 ArgStart = 0;
+	std::vector<std::string> Args;
+	while (Pos < CommandLine.length())
+	{
+		if (IsInQuote)
+		{
+			if (CommandLine[Pos] == '"' && IsWhitespaceOrEnd(CommandLine, Pos + 1))
+			{
+				Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart + 1)));
+				Pos++;
+				IsInQuote	 = false;
+				IsParsingArg = false;
+			}
+			else
+			{
+				Pos++;
+			}
+		}
+		else if (IsParsingArg)
+		{
+			ZEN_ASSERT(Pos > ArgStart);
+			if (CommandLine[Pos] == ' ')
+			{
+				Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart)));
+				Pos++;
+				IsParsingArg = false;
+			}
+			else if (CommandLine[Pos] == '"')
+			{
+				IsInQuote = true;
+				Pos++;
+			}
+			else
+			{
+				Pos++;
+			}
+		}
+		else if (CommandLine[Pos] == '"')
+		{
+			IsInQuote	 = true;
+			IsParsingArg = true;
+			ArgStart	 = Pos;
+			Pos++;
+		}
+		else if (CommandLine[Pos] != ' ')
+		{
+			IsParsingArg = true;
+			ArgStart	 = Pos;
+			Pos++;
+		}
+		else
+		{
+			Pos++;
+		}
+	}
+	if (IsParsingArg)
+	{
+		ZEN_ASSERT(Pos > ArgStart);
+		Args.push_back(std::string(CommandLine.substr(ArgStart)));
+	}
+
+	return Args;
+}
+
+std::vector<char*>
+StripCommandlineQuotes(std::vector<std::string>& InOutArgs)
+{
+	std::vector<char*> RawArgs;
+	RawArgs.reserve(InOutArgs.size());
+	for (std::string& Arg : InOutArgs)
+	{
+		std::string::size_type EscapedQuotePos = Arg.find("\\\"", 1);
+		while (EscapedQuotePos != std::string::npos && Arg.rfind('\"', EscapedQuotePos - 1) != std::string::npos)
+		{
+			Arg.erase(EscapedQuotePos, 1);
+			EscapedQuotePos = Arg.find("\\\"", EscapedQuotePos);
+		}
+
+		if (Arg.starts_with("\""))
+		{
+			if (Arg.find('"', 1) == Arg.length() - 1)
+			{
+				if (Arg.find(' ', 1) == std::string::npos)
+				{
+					Arg = Arg.substr(1, Arg.length() - 2);
+				}
+			}
+		}
+		RawArgs.push_back(const_cast<char*>(Arg.c_str()));
+	}
+	return RawArgs;
+}
+
 #if ZEN_WITH_TESTS
 
 void
@@ -1140,6 +1252,36 @@ TEST_CASE("BuildArgV")
 	}
 }
 
+TEST_CASE("CommandLine")
+{
+	std::vector<std::string> v1 = ParseCommandLine("c:\\my\\exe.exe \"quoted arg\" \"one\",two,\"three\\\"");
+	CHECK_EQ(v1[0], "c:\\my\\exe.exe");
+	CHECK_EQ(v1[1], "\"quoted arg\"");
+	CHECK_EQ(v1[2], "\"one\",two,\"three\\\"");
+
+	std::vector<std::string> v2 = ParseCommandLine(
+		"--tracehost 127.0.0.1 builds download --url=https://jupiter.devtools.epicgames.com --namespace=ue.oplog "
+		"--bucket=citysample.packaged-build.fortnite-main.windows \"c:\\just\\a\\path\" "
+		"--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\" \"D:\\Dev\\Spaced Folder\\Target\\\" "
+		"--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\\\" 07dn23ifiwesnvoasjncasab --build-part-name win64,linux,ps5");
+
+	std::vector<char*> v2Stripped = StripCommandlineQuotes(v2);
+	CHECK_EQ(v2Stripped[0], std::string("--tracehost"));
+	CHECK_EQ(v2Stripped[1], std::string("127.0.0.1"));
+	CHECK_EQ(v2Stripped[2], std::string("builds"));
+	CHECK_EQ(v2Stripped[3], std::string("download"));
+	CHECK_EQ(v2Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com"));
+	CHECK_EQ(v2Stripped[5], std::string("--namespace=ue.oplog"));
+	CHECK_EQ(v2Stripped[6], std::string("--bucket=citysample.packaged-build.fortnite-main.windows"));
+	CHECK_EQ(v2Stripped[7], std::string("c:\\just\\a\\path"));
+	CHECK_EQ(v2Stripped[8], std::string("--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\""));
+	CHECK_EQ(v2Stripped[9], std::string("\"D:\\Dev\\Spaced Folder\\Target\""));
+	CHECK_EQ(v2Stripped[10], std::string("--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\""));
+	CHECK_EQ(v2Stripped[11], std::string("07dn23ifiwesnvoasjncasab"));
+	CHECK_EQ(v2Stripped[12], std::string("--build-part-name"));
+	CHECK_EQ(v2Stripped[13], std::string("win64,linux,ps5"));
+}
+
 TEST_SUITE_END(/* core.process */);
 
 #endif
diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp
index 242d41abe..a0d8c927f 100644
--- a/src/zencore/string.cpp
+++ b/src/zencore/string.cpp
@@ -99,6 +99,20 @@ FilepathFindExtension(const std::string_view& Path, const char* ExtensionToMatch
 
 //////////////////////////////////////////////////////////////////////////
 
+bool
+IsValidUtf8(const std::string_view& str)
+{
+	return utf8::is_valid(begin(str), end(str));
+}
+
+std::string_view::const_iterator
+FindFirstInvalidUtf8Byte(const std::string_view& str)
+{
+	return utf8::find_invalid(begin(str), end(str));
+}
+
+//////////////////////////////////////////////////////////////////////////
+
 void
 Utf8ToWide(const char8_t* Str8, WideStringBuilderBase& OutString)
 {
diff --git a/src/zencore/timer.cpp b/src/zencore/timer.cpp
index 1655e912d..95536cb26 100644
--- a/src/zencore/timer.cpp
+++ b/src/zencore/timer.cpp
@@ -12,9 +12,20 @@
 #	include <unistd.h>
 #endif
 
+#define GTSPS_IMPLEMENTATION
+#include "GetTimeSinceProcessStart.h"
+
 namespace zen {
 
 uint64_t
+GetTimeSinceProcessStart()
+{
+	double TimeInSeconds = ::GetTimeSinceProcessStart();
+
+	return uint64_t(TimeInSeconds * 1000);
+}
+
+uint64_t
 GetHifreqTimerValue()
 {
 	uint64_t Timestamp;
diff --git a/src/zencore/workthreadpool.cpp b/src/zencore/workthreadpool.cpp
index d15fb2e83..445fe939e 100644
--- a/src/zencore/workthreadpool.cpp
+++ b/src/zencore/workthreadpool.cpp
@@ -274,7 +274,7 @@ WorkerThreadPool::ScheduleWork(Ref<IWork> Work)
 void
 WorkerThreadPool::ScheduleWork(std::function<void()>&& Work)
 {
-	ScheduleWork(Ref<IWork>(new detail::LambdaWork(Work)));
+	ScheduleWork(Ref<IWork>(new detail::LambdaWork(std::move(Work))));
 }
 
 [[nodiscard]] size_t
diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua
index 2efa3fdb8..13611a2e9 100644
--- a/src/zencore/xmake.lua
+++ b/src/zencore/xmake.lua
@@ -29,6 +29,7 @@ target('zencore')
     end
 
     add_includedirs("include", {public=true})
+    add_includedirs("$(projectdir)/thirdparty/GetTimeSinceProcessStart")
     add_includedirs("$(projectdir)/thirdparty/utfcpp/source")
     add_includedirs("$(projectdir)/thirdparty/Oodle/include")
     add_includedirs("$(projectdir)/thirdparty/trace", {public=true})
@@ -55,6 +56,7 @@ target('zencore')
 
     add_packages(
         "vcpkg::doctest", 
+        "vcpkg::eastl",
         "vcpkg::fmt",
         "vcpkg::gsl-lite", 
         "vcpkg::lz4",