From 7de3d4218ee5969af6147f9ab20bda538a136d9a Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Mon, 10 Mar 2025 18:33:24 +0100
Subject: pick up existing cache (#299)

- Improvement: Scavenge .zen temp folders for existing data (downloaded, decompressed or written) from previous failed run
- Improvement: Faster abort during stream compression
- Improvement: Try to move downloaded blobs with rename if possible avoiding an extra disk write
- Improvement: Only clean temp folders on successful or cancelled build - keep it if download fails
---
 src/zencore/compress.cpp | 51 +++++++++++++++++++++++++++++++++---------------
 1 file changed, 35 insertions(+), 16 deletions(-)

(limited to 'src/zencore/compress.cpp')

diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp
index 1844f6a63..88c3bb5b9 100644
--- a/src/zencore/compress.cpp
+++ b/src/zencore/compress.cpp
@@ -193,7 +193,7 @@ public:
 									const CompositeBuffer&												 CompressedData,
 									uint64_t															 RawOffset,
 									uint64_t															 RawSize,
-									std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const = 0;
+									std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const = 0;
 };
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -287,13 +287,16 @@ public:
 									const CompositeBuffer&												 CompressedData,
 									uint64_t															 RawOffset,
 									uint64_t															 RawSize,
-									std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const final
+									std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const final
 	{
 		if (Header.Method == CompressionMethod::None && Header.TotalCompressedSize == CompressedData.GetSize() &&
 			Header.TotalCompressedSize == Header.TotalRawSize + sizeof(BufferHeader) && RawOffset < Header.TotalRawSize &&
 			(RawOffset + RawSize) <= Header.TotalRawSize)
 		{
-			Callback(0, CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize));
+			if (!Callback(0, CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize)))
+			{
+				return false;
+			}
 			return true;
 		}
 		return false;
@@ -616,7 +619,7 @@ public:
 									const CompositeBuffer&												 CompressedData,
 									uint64_t															 RawOffset,
 									uint64_t															 RawSize,
-									std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const final;
+									std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const final;
 
 protected:
 	virtual bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const = 0;
@@ -744,7 +747,7 @@ BlockDecoder::DecompressToStream(const BufferHeader&												  Header,
 								 const CompositeBuffer&												  CompressedData,
 								 uint64_t															  RawOffset,
 								 uint64_t															  RawSize,
-								 std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const
+								 std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const
 {
 	if (Header.TotalCompressedSize != CompressedData.GetSize())
 	{
@@ -814,14 +817,23 @@ BlockDecoder::DecompressToStream(const BufferHeader&												  Header,
 					Source.Detach();
 					return false;
 				}
-				Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
-						 CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress)));
+				if (!Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
+							  CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress))))
+				{
+					Source.Detach();
+					return false;
+				}
 			}
 			else
 			{
-				Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
-						 CompositeBuffer(
-							 IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress)));
+				if (!Callback(
+						BlockIndex * BlockSize + OffsetInFirstBlock,
+						CompositeBuffer(
+							IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress))))
+				{
+					Source.Detach();
+					return false;
+				}
 			}
 
 			OffsetInFirstBlock = 0;
@@ -858,14 +870,21 @@ BlockDecoder::DecompressToStream(const BufferHeader&												  Header,
 				{
 					return false;
 				}
-				Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
-						 CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress)));
+				if (!Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
+							  CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress))))
+				{
+					return false;
+				}
 			}
 			else
 			{
-				Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
-						 CompositeBuffer(
-							 IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress)));
+				if (!Callback(
+						BlockIndex * BlockSize + OffsetInFirstBlock,
+						CompositeBuffer(
+							IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress))))
+				{
+					return false;
+				}
 			}
 
 			OffsetInFirstBlock = 0;
@@ -1978,7 +1997,7 @@ CompressedBuffer::DecompressToComposite() const
 bool
 CompressedBuffer::DecompressToStream(uint64_t															  RawOffset,
 									 uint64_t															  RawSize,
-									 std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const
+									 std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const
 {
 	using namespace detail;
 	if (CompressedData)
-- 
cgit v1.2.3