pick up existing cache (#299)

- Improvement: Scavenge .zen temp folders for existing data (downloaded, decompressed or written) from previous failed run
- Improvement: Faster abort during stream compression
- Improvement: Try to move downloaded blobs with rename if possible avoiding an extra disk write
- Improvement: Only clean temp folders on successful or cancelled build - keep it if download fails

5 files changed, 62 insertions, 32 deletions

diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp
index 6e879ca0d..95876cff4 100644
--- a/src/zencore/basicfile.cpp
+++ b/src/zencore/basicfile.cpp
@@ -858,7 +858,7 @@ BasicFileWriter::Flush()
 }
 
 IoBuffer
-WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path)
+WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path)
 {
 	TemporaryFile	Temp;
 	std::error_code Ec;
@@ -868,6 +868,7 @@ WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path
 		throw std::system_error(Ec, fmt::format("Failed to create temp file for blob at '{}'", Path));
 	}
 
+	uint64_t BufferSize = Buffer.GetSize();
 	{
 		uint64_t			  Offset		= 0;
 		static const uint64_t BufferingSize = 256u * 1024u;
@@ -899,21 +900,31 @@ WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path
 	Temp.MoveTemporaryIntoPlace(Path, Ec);
 	if (Ec)
 	{
-		IoBuffer TmpBuffer = IoBufferBuilder::MakeFromFile(Path);
-		if (TmpBuffer)
+		Ec.clear();
+		BasicFile OpenTemp(Path, BasicFile::Mode::kDelete, Ec);
+		if (Ec)
 		{
-			IoHash		 ExistingHash = IoHash::HashBuffer(TmpBuffer);
-			const IoHash ExpectedHash = IoHash::HashBuffer(Buffer);
-			if (ExistingHash == ExpectedHash)
-			{
-				TmpBuffer.SetDeleteOnClose(true);
-				return TmpBuffer;
-			}
+			throw std::system_error(Ec, fmt::format("Failed to move temp file to '{}'", Path));
 		}
-		throw std::system_error(Ec, fmt::format("Failed to move temp file to '{}'", Path));
-	}
+		if (OpenTemp.FileSize() != BufferSize)
+		{
+			throw std::runtime_error(fmt::format("Failed to move temp file to '{}' - mismatching file size already exists", Path));
+		}
+		IoBuffer TmpBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BufferSize, true);
 
-	IoBuffer TmpBuffer = IoBufferBuilder::MakeFromFile(Path);
+		IoHash		 ExistingHash = IoHash::HashBuffer(TmpBuffer);
+		const IoHash ExpectedHash = IoHash::HashBuffer(Buffer);
+		if (ExistingHash != ExpectedHash)
+		{
+			throw std::runtime_error(fmt::format("Failed to move temp file to '{}' - mismatching file hash already exists", Path));
+		}
+		Buffer = CompositeBuffer{};
+		TmpBuffer.SetDeleteOnClose(true);
+		return TmpBuffer;
+	}
+	Buffer = CompositeBuffer{};
+	BasicFile OpenTemp(Path, BasicFile::Mode::kDelete);
+	IoBuffer  TmpBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BufferSize, true);
 	TmpBuffer.SetDeleteOnClose(true);
 	return TmpBuffer;
 }
diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp
index 1844f6a63..88c3bb5b9 100644
--- a/src/zencore/compress.cpp
+++ b/src/zencore/compress.cpp
@@ -193,7 +193,7 @@ public:
 									const CompositeBuffer&												 CompressedData,
 									uint64_t															 RawOffset,
 									uint64_t															 RawSize,
-									std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const = 0;
+									std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const = 0;
 };
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -287,13 +287,16 @@ public:
 									const CompositeBuffer&												 CompressedData,
 									uint64_t															 RawOffset,
 									uint64_t															 RawSize,
-									std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const final
+									std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const final
 	{
 		if (Header.Method == CompressionMethod::None && Header.TotalCompressedSize == CompressedData.GetSize() &&
 			Header.TotalCompressedSize == Header.TotalRawSize + sizeof(BufferHeader) && RawOffset < Header.TotalRawSize &&
 			(RawOffset + RawSize) <= Header.TotalRawSize)
 		{
-			Callback(0, CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize));
+			if (!Callback(0, CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize)))
+			{
+				return false;
+			}
 			return true;
 		}
 		return false;
@@ -616,7 +619,7 @@ public:
 									const CompositeBuffer&												 CompressedData,
 									uint64_t															 RawOffset,
 									uint64_t															 RawSize,
-									std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const final;
+									std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const final;
 
 protected:
 	virtual bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const = 0;
@@ -744,7 +747,7 @@ BlockDecoder::DecompressToStream(const BufferHeader&												  Header,
 								 const CompositeBuffer&												  CompressedData,
 								 uint64_t															  RawOffset,
 								 uint64_t															  RawSize,
-								 std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const
+								 std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const
 {
 	if (Header.TotalCompressedSize != CompressedData.GetSize())
 	{
@@ -814,14 +817,23 @@ BlockDecoder::DecompressToStream(const BufferHeader&												  Header,
 					Source.Detach();
 					return false;
 				}
-				Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
-						 CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress)));
+				if (!Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
+							  CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress))))
+				{
+					Source.Detach();
+					return false;
+				}
 			}
 			else
 			{
-				Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
-						 CompositeBuffer(
-							 IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress)));
+				if (!Callback(
+						BlockIndex * BlockSize + OffsetInFirstBlock,
+						CompositeBuffer(
+							IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress))))
+				{
+					Source.Detach();
+					return false;
+				}
 			}
 
 			OffsetInFirstBlock = 0;
@@ -858,14 +870,21 @@ BlockDecoder::DecompressToStream(const BufferHeader&												  Header,
 				{
 					return false;
 				}
-				Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
-						 CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress)));
+				if (!Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
+							  CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress))))
+				{
+					return false;
+				}
 			}
 			else
 			{
-				Callback(BlockIndex * BlockSize + OffsetInFirstBlock,
-						 CompositeBuffer(
-							 IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress)));
+				if (!Callback(
+						BlockIndex * BlockSize + OffsetInFirstBlock,
+						CompositeBuffer(
+							IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress))))
+				{
+					return false;
+				}
 			}
 
 			OffsetInFirstBlock = 0;
@@ -1978,7 +1997,7 @@ CompressedBuffer::DecompressToComposite() const
 bool
 CompressedBuffer::DecompressToStream(uint64_t															  RawOffset,
 									 uint64_t															  RawSize,
-									 std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const
+									 std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const
 {
 	using namespace detail;
 	if (CompressedData)
diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h
index a78132879..57798b6f4 100644
--- a/src/zencore/include/zencore/basicfile.h
+++ b/src/zencore/include/zencore/basicfile.h
@@ -186,7 +186,7 @@ private:
 	uint64_t	   m_BufferEnd;
 };
 
-IoBuffer WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path);
+IoBuffer WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path);
 
 ZENCORE_API void basicfile_forcelink();
 
diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h
index 3969e9dbd..74fd5f767 100644
--- a/src/zencore/include/zencore/compress.h
+++ b/src/zencore/include/zencore/compress.h
@@ -209,7 +209,7 @@ public:
 	 */
 	[[nodiscard]] ZENCORE_API bool DecompressToStream(uint64_t															   RawOffset,
 													  uint64_t															   RawSize,
-													  std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const;
+													  std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const;
 
 	/** A null compressed buffer. */
 	static const CompressedBuffer Null;
diff --git a/src/zencore/workthreadpool.cpp b/src/zencore/workthreadpool.cpp
index d15fb2e83..445fe939e 100644
--- a/src/zencore/workthreadpool.cpp
+++ b/src/zencore/workthreadpool.cpp
@@ -274,7 +274,7 @@ WorkerThreadPool::ScheduleWork(Ref<IWork> Work)
 void
 WorkerThreadPool::ScheduleWork(std::function<void()>&& Work)
 {
-	ScheduleWork(Ref<IWork>(new detail::LambdaWork(Work)));
+	ScheduleWork(Ref<IWork>(new detail::LambdaWork(std::move(Work))));
 }
 
 [[nodiscard]] size_t