improvements and infrastructure for upcoming builds api command line (#284)

* add modification tick to filesystem traversal
* add ShowDetails option to ProgressBar
* log callstack if we terminate process
* handle chunking if MaxSize > 1MB
* BasicFile write helpers and WriteToTempFile simplifications
* bugfix for CompositeBuffer::IterateRange when using DecompressToComposite for actually comrpessed data
revert of earlier optimization
* faster compress/decompress for large disk-based files
* enable progress feedback in IoHash::HashBuffer
* add payload validation in HttpClient::Get
* fix range requests (range is including end byte)
* remove BuildPartId for blob/block related operations in builds api

8 files changed, 290 insertions, 121 deletions

diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp
index b3bffd34d..6e879ca0d 100644
--- a/src/zencore/basicfile.cpp
+++ b/src/zencore/basicfile.cpp
@@ -281,6 +281,20 @@ BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<voi
 }
 
 uint64_t
+BasicFile::Write(const CompositeBuffer& Data, uint64_t FileOffset)
+{
+	std::error_code Ec;
+	uint64_t		WrittenBytes = Write(Data, FileOffset, Ec);
+
+	if (Ec)
+	{
+		std::error_code Dummy;
+		throw std::system_error(Ec, fmt::format("Failed to write to file '{}'", zen::PathFromHandle(m_FileHandle, Dummy)));
+	}
+	return WrittenBytes;
+}
+
+uint64_t
 BasicFile::Write(const CompositeBuffer& Data, uint64_t FileOffset, std::error_code& Ec)
 {
 	uint64_t WrittenBytes = 0;
@@ -309,6 +323,8 @@ BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec)
 void
 BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec)
 {
+	ZEN_ASSERT(m_FileHandle != nullptr);
+
 	Ec.clear();
 
 	const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
@@ -817,6 +833,17 @@ BasicFileWriter::Write(const void* Data, uint64_t Size, uint64_t FileOffset)
 }
 
 void
+BasicFileWriter::Write(const CompositeBuffer& Data, uint64_t FileOffset)
+{
+	for (const SharedBuffer& Segment : Data.GetSegments())
+	{
+		const uint64_t SegmentSize = Segment.GetSize();
+		Write(Segment.GetData(), SegmentSize, FileOffset);
+		FileOffset += SegmentSize;
+	}
+}
+
+void
 BasicFileWriter::Flush()
 {
 	const uint64_t BufferedBytes = m_BufferEnd - m_BufferStart;
@@ -831,23 +858,20 @@ BasicFileWriter::Flush()
 }
 
 IoBuffer
-WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path, std::function<bool(std::error_code& Ec)>&& RetryCallback)
+WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path)
 {
-	if (std::filesystem::is_regular_file(Path))
+	TemporaryFile	Temp;
+	std::error_code Ec;
+	Temp.CreateTemporary(Path.parent_path(), Ec);
+	if (Ec)
 	{
-		IoBuffer ExistingTempFile = IoBuffer(IoBufferBuilder::MakeFromFile(Path));
-		if (ExistingTempFile && ExistingTempFile.GetSize() == Buffer.GetSize())
-		{
-			ExistingTempFile.SetDeleteOnClose(true);
-			return ExistingTempFile;
-		}
+		throw std::system_error(Ec, fmt::format("Failed to create temp file for blob at '{}'", Path));
 	}
-	BasicFile BlockFile;
-	BlockFile.Open(Path, BasicFile::Mode::kTruncateDelete, std::move(RetryCallback));
-	uint64_t Offset = 0;
+
 	{
+		uint64_t			  Offset		= 0;
 		static const uint64_t BufferingSize = 256u * 1024u;
-		BasicFileWriter		  BufferedOutput(BlockFile, BufferingSize / 2);
+		//		BasicFileWriter		  BufferedOutput(BlockFile, BufferingSize / 2);
 		for (const SharedBuffer& Segment : Buffer.GetSegments())
 		{
 			size_t SegmentSize = Segment.GetSize();
@@ -859,22 +883,39 @@ WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path, std
 						 FileRef.FileChunkOffset,
 						 FileRef.FileChunkSize,
 						 BufferingSize,
-						 [&BufferedOutput, &Offset](const void* Data, size_t Size) {
-							 BufferedOutput.Write(Data, Size, Offset);
+						 [&Temp, &Offset](const void* Data, size_t Size) {
+							 Temp.Write(Data, Size, Offset);
 							 Offset += Size;
 						 });
 			}
 			else
 			{
-				BufferedOutput.Write(Segment.GetData(), SegmentSize, Offset);
+				Temp.Write(Segment.GetData(), SegmentSize, Offset);
 				Offset += SegmentSize;
 			}
 		}
 	}
-	void*	 FileHandle	 = BlockFile.Detach();
-	IoBuffer BlockBuffer = IoBuffer(IoBuffer::File, FileHandle, 0, Offset, /*IsWholeFile*/ true);
-	BlockBuffer.SetDeleteOnClose(true);
-	return BlockBuffer;
+
+	Temp.MoveTemporaryIntoPlace(Path, Ec);
+	if (Ec)
+	{
+		IoBuffer TmpBuffer = IoBufferBuilder::MakeFromFile(Path);
+		if (TmpBuffer)
+		{
+			IoHash		 ExistingHash = IoHash::HashBuffer(TmpBuffer);
+			const IoHash ExpectedHash = IoHash::HashBuffer(Buffer);
+			if (ExistingHash == ExpectedHash)
+			{
+				TmpBuffer.SetDeleteOnClose(true);
+				return TmpBuffer;
+			}
+		}
+		throw std::system_error(Ec, fmt::format("Failed to move temp file to '{}'", Path));
+	}
+
+	IoBuffer TmpBuffer = IoBufferBuilder::MakeFromFile(Path);
+	TmpBuffer.SetDeleteOnClose(true);
+	return TmpBuffer;
 }
 
 //////////////////////////////////////////////////////////////////////////
diff --git a/src/zencore/compositebuffer.cpp b/src/zencore/compositebuffer.cpp
index 49870a304..252ac9045 100644
--- a/src/zencore/compositebuffer.cpp
+++ b/src/zencore/compositebuffer.cpp
@@ -275,36 +275,18 @@ CompositeBuffer::IterateRange(uint64_t															  Offset,
 			Visitor(View, Segment);
 			break;
 		}
-		if (Offset < SegmentSize)
+		else if (Offset <= SegmentSize)
 		{
-			if (Offset == 0 && Size >= SegmentSize)
+			const MemoryView View = Segment.GetView().Mid(Offset, Size);
+			Offset				  = 0;
+			if (Size == 0 || !View.IsEmpty())
 			{
-				const MemoryView View = Segment.GetView();
-				if (!View.IsEmpty())
-				{
-					Visitor(View, Segment);
-				}
-				Size -= View.GetSize();
-				if (Size == 0)
-				{
-					break;
-				}
+				Visitor(View, Segment);
 			}
-			else
+			Size -= View.GetSize();
+			if (Size == 0)
 			{
-				// If we only want a section of the segment, do a subrange so we don't have to materialize the entire iobuffer
-				IoBuffer		 SubRange(Segment.AsIoBuffer(), Offset, Min(Size, SegmentSize - Offset));
-				const MemoryView View = SubRange.GetView();
-				if (!View.IsEmpty())
-				{
-					Visitor(View, Segment);
-				}
-				Size -= View.GetSize();
-				if (Size == 0)
-				{
-					break;
-				}
-				Offset = 0;
+				break;
 			}
 		}
 		else
diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp
index 29c1d9256..0e2ce2b54 100644
--- a/src/zencore/compress.cpp
+++ b/src/zencore/compress.cpp
@@ -2,6 +2,7 @@
 
 #include <zencore/compress.h>
 
+#include <zencore/basicfile.h>
 #include <zencore/blake3.h>
 #include <zencore/compositebuffer.h>
 #include <zencore/crc32.h>
@@ -314,37 +315,77 @@ BlockEncoder::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize)
 	CompressedBlockSizes.reserve(BlockCount);
 	uint64_t CompressedSize = 0;
 	{
-		UniqueBuffer	  RawBlockCopy;
 		MutableMemoryView CompressedBlocksView = CompressedData.GetMutableView() + sizeof(BufferHeader) + MetaSize;
 
-		CompositeBuffer::Iterator It = RawData.GetIterator(0);
-
-		for (uint64_t RawOffset = 0; RawOffset < RawSize;)
+		IoBufferFileReference FileRef = {nullptr, 0, 0};
+		if ((RawData.GetSegments().size() == 1) && RawData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef))
 		{
-			const uint64_t	 RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize);
-			const MemoryView RawBlock	  = RawData.ViewOrCopyRange(It, RawBlockSize, RawBlockCopy);
-			RawHash.Append(RawBlock);
-
-			MutableMemoryView CompressedBlock = CompressedBlocksView;
-			if (!CompressBlock(CompressedBlock, RawBlock))
+			ZEN_ASSERT(FileRef.FileHandle != nullptr);
+			UniqueBuffer RawBlockCopy = UniqueBuffer::Alloc(BlockSize);
+			BasicFile	 Source;
+			Source.Attach(FileRef.FileHandle);
+			for (uint64_t RawOffset = 0; RawOffset < RawSize;)
 			{
-				return CompositeBuffer();
-			}
+				const uint64_t RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize);
+				Source.Read(RawBlockCopy.GetData(), RawBlockSize, FileRef.FileChunkOffset + RawOffset);
+				const MemoryView RawBlock = RawBlockCopy.GetView().Left(RawBlockSize);
+				RawHash.Append(RawBlock);
+				MutableMemoryView CompressedBlock = CompressedBlocksView;
+				if (!CompressBlock(CompressedBlock, RawBlock))
+				{
+					Source.Detach();
+					return CompositeBuffer();
+				}
 
-			uint64_t CompressedBlockSize = CompressedBlock.GetSize();
-			if (RawBlockSize <= CompressedBlockSize)
-			{
-				CompressedBlockSize	 = RawBlockSize;
-				CompressedBlocksView = CompressedBlocksView.CopyFrom(RawBlock);
+				uint64_t CompressedBlockSize = CompressedBlock.GetSize();
+				if (RawBlockSize <= CompressedBlockSize)
+				{
+					CompressedBlockSize	 = RawBlockSize;
+					CompressedBlocksView = CompressedBlocksView.CopyFrom(RawBlock);
+				}
+				else
+				{
+					CompressedBlocksView += CompressedBlockSize;
+				}
+
+				CompressedBlockSizes.push_back(static_cast<uint32_t>(CompressedBlockSize));
+				CompressedSize += CompressedBlockSize;
+				RawOffset += RawBlockSize;
 			}
-			else
+			Source.Detach();
+		}
+		else
+		{
+			UniqueBuffer			  RawBlockCopy;
+			CompositeBuffer::Iterator It = RawData.GetIterator(0);
+
+			for (uint64_t RawOffset = 0; RawOffset < RawSize;)
 			{
-				CompressedBlocksView += CompressedBlockSize;
-			}
+				const uint64_t	 RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize);
+				const MemoryView RawBlock	  = RawData.ViewOrCopyRange(It, RawBlockSize, RawBlockCopy);
+				RawHash.Append(RawBlock);
 
-			CompressedBlockSizes.push_back(static_cast<uint32_t>(CompressedBlockSize));
-			CompressedSize += CompressedBlockSize;
-			RawOffset += RawBlockSize;
+				MutableMemoryView CompressedBlock = CompressedBlocksView;
+				if (!CompressBlock(CompressedBlock, RawBlock))
+				{
+					return CompositeBuffer();
+				}
+
+				uint64_t CompressedBlockSize = CompressedBlock.GetSize();
+				if (RawBlockSize <= CompressedBlockSize)
+				{
+					CompressedBlockSize	 = RawBlockSize;
+					CompressedBlocksView = CompressedBlocksView.CopyFrom(RawBlock);
+				}
+				else
+				{
+					CompressedBlocksView += CompressedBlockSize;
+				}
+
+				CompressedBlockSizes.push_back(static_cast<uint32_t>(CompressedBlockSize));
+				CompressedSize += CompressedBlockSize;
+				RawOffset += RawBlockSize;
+			}
 		}
 	}
 
@@ -560,51 +601,118 @@ BlockDecoder::TryDecompressTo(const BufferHeader&	 Header,
 		CompressedOffset += CompressedBlockSize;
 	}
 
-	for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++)
+	IoBufferFileReference FileRef = {nullptr, 0, 0};
+	if ((CompressedData.GetSegments().size() == 1) && CompressedData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef))
 	{
-		const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize;
-		const uint32_t CompressedBlockSize	 = ByteSwap(CompressedBlockSizes[BlockIndex]);
-		const bool	   IsCompressed			 = CompressedBlockSize < UncompressedBlockSize;
+		ZEN_ASSERT(FileRef.FileHandle != nullptr);
+		BasicFile Source;
+		Source.Attach(FileRef.FileHandle);
 
-		const uint64_t BytesToUncompress = OffsetInFirstBlock > 0 ? zen::Min(RawView.GetSize(), UncompressedBlockSize - OffsetInFirstBlock)
-																  : zen::Min(RemainingRawSize, BlockSize);
+		for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++)
+		{
+			const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize;
+			const uint32_t CompressedBlockSize	 = ByteSwap(CompressedBlockSizes[BlockIndex]);
+			const bool	   IsCompressed			 = CompressedBlockSize < UncompressedBlockSize;
 
-		MemoryView CompressedBlock = CompressedData.ViewOrCopyRange(CompressedOffset, CompressedBlockSize, CompressedBlockCopy);
+			const uint64_t BytesToUncompress = OffsetInFirstBlock > 0
+												   ? zen::Min(RawView.GetSize(), UncompressedBlockSize - OffsetInFirstBlock)
+												   : zen::Min(RemainingRawSize, BlockSize);
 
-		if (IsCompressed)
-		{
-			MutableMemoryView UncompressedBlock = RawView.Left(BytesToUncompress);
+			if (CompressedBlockCopy.GetSize() < CompressedBlockSize)
+			{
+				CompressedBlockCopy = UniqueBuffer::Alloc(CompressedBlockSize);
+			}
+			Source.Read(CompressedBlockCopy.GetData(), CompressedBlockSize, FileRef.FileChunkOffset + CompressedOffset);
 
-			const bool IsAligned = BytesToUncompress == UncompressedBlockSize;
-			if (!IsAligned)
+			MemoryView CompressedBlock = CompressedBlockCopy.GetView().Left(CompressedBlockSize);
+
+			if (IsCompressed)
 			{
-				// Decompress to a temporary buffer when the first or the last block reads are not aligned with the block boundaries.
-				if (UncompressedBlockCopy.IsNull())
+				MutableMemoryView UncompressedBlock = RawView.Left(BytesToUncompress);
+
+				const bool IsAligned = BytesToUncompress == UncompressedBlockSize;
+				if (!IsAligned)
 				{
-					UncompressedBlockCopy = UniqueBuffer::Alloc(BlockSize);
+					// Decompress to a temporary buffer when the first or the last block reads are not aligned with the block boundaries.
+					if (UncompressedBlockCopy.IsNull())
+					{
+						UncompressedBlockCopy = UniqueBuffer::Alloc(BlockSize);
+					}
+					UncompressedBlock = UncompressedBlockCopy.GetMutableView().Mid(0, UncompressedBlockSize);
 				}
-				UncompressedBlock = UncompressedBlockCopy.GetMutableView().Mid(0, UncompressedBlockSize);
-			}
 
-			if (!DecompressBlock(UncompressedBlock, CompressedBlock))
-			{
-				return false;
-			}
+				if (!DecompressBlock(UncompressedBlock, CompressedBlock))
+				{
+					Source.Detach();
+					return false;
+				}
 
-			if (!IsAligned)
+				if (!IsAligned)
+				{
+					RawView.CopyFrom(UncompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
+				}
+			}
+			else
 			{
-				RawView.CopyFrom(UncompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
+				RawView.CopyFrom(CompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
 			}
+
+			OffsetInFirstBlock = 0;
+			RemainingRawSize -= BytesToUncompress;
+			CompressedOffset += CompressedBlockSize;
+			RawView += BytesToUncompress;
 		}
-		else
+		Source.Detach();
+	}
+	else
+	{
+		for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++)
 		{
-			RawView.CopyFrom(CompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
-		}
+			const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize;
+			const uint32_t CompressedBlockSize	 = ByteSwap(CompressedBlockSizes[BlockIndex]);
+			const bool	   IsCompressed			 = CompressedBlockSize < UncompressedBlockSize;
 
-		OffsetInFirstBlock = 0;
-		RemainingRawSize -= BytesToUncompress;
-		CompressedOffset += CompressedBlockSize;
-		RawView += BytesToUncompress;
+			const uint64_t BytesToUncompress = OffsetInFirstBlock > 0
+												   ? zen::Min(RawView.GetSize(), UncompressedBlockSize - OffsetInFirstBlock)
+												   : zen::Min(RemainingRawSize, BlockSize);
+
+			MemoryView CompressedBlock = CompressedData.ViewOrCopyRange(CompressedOffset, CompressedBlockSize, CompressedBlockCopy);
+
+			if (IsCompressed)
+			{
+				MutableMemoryView UncompressedBlock = RawView.Left(BytesToUncompress);
+
+				const bool IsAligned = BytesToUncompress == UncompressedBlockSize;
+				if (!IsAligned)
+				{
+					// Decompress to a temporary buffer when the first or the last block reads are not aligned with the block boundaries.
+					if (UncompressedBlockCopy.IsNull())
+					{
+						UncompressedBlockCopy = UniqueBuffer::Alloc(BlockSize);
+					}
+					UncompressedBlock = UncompressedBlockCopy.GetMutableView().Mid(0, UncompressedBlockSize);
+				}
+
+				if (!DecompressBlock(UncompressedBlock, CompressedBlock))
+				{
+					return false;
+				}
+
+				if (!IsAligned)
+				{
+					RawView.CopyFrom(UncompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
+				}
+			}
+			else
+			{
+				RawView.CopyFrom(CompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
+			}
+
+			OffsetInFirstBlock = 0;
+			RemainingRawSize -= BytesToUncompress;
+			CompressedOffset += CompressedBlockSize;
+			RawView += BytesToUncompress;
+		}
 	}
 
 	return RemainingRawSize == 0;
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index b8c35212f..5716d1255 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -683,7 +683,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 		{
 		}
 
-		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override
 		{
 			std::error_code				Ec;
 			const std::filesystem::path Relative = std::filesystem::relative(Parent, BasePath, Ec);
@@ -1236,7 +1236,11 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
 			}
 			else
 			{
-				Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart, gsl::narrow<uint32_t>(DirInfo->FileAttributes));
+				Visitor.VisitFile(RootDir,
+								  FileName,
+								  DirInfo->EndOfFile.QuadPart,
+								  gsl::narrow<uint32_t>(DirInfo->FileAttributes),
+								  (uint64_t)DirInfo->LastWriteTime.QuadPart);
 			}
 
 			const uint64_t NextOffset = DirInfo->NextEntryOffset;
@@ -1285,7 +1289,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
 		}
 		else if (S_ISREG(Stat.st_mode))
 		{
-			Visitor.VisitFile(RootDir, FileName, Stat.st_size, gsl::narrow<uint32_t>(Stat.st_mode));
+			Visitor.VisitFile(RootDir, FileName, Stat.st_size, gsl::narrow<uint32_t>(Stat.st_mode), gsl::narrow<uint64_t>(Stat.st_mtime));
 		}
 		else
 		{
@@ -1544,7 +1548,8 @@ GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags
 		virtual void VisitFile(const std::filesystem::path& Parent,
 							   const path_view&				File,
 							   uint64_t						FileSize,
-							   uint32_t						NativeModeOrAttributes) override
+							   uint32_t						NativeModeOrAttributes,
+							   uint64_t						NativeModificationTick) override
 		{
 			if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles))
 			{
@@ -1557,6 +1562,10 @@ GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags
 				{
 					Content.FileAttributes.push_back(NativeModeOrAttributes);
 				}
+				if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeModificationTick))
+				{
+					Content.FileModificationTicks.push_back(NativeModificationTick);
+				}
 			}
 		}
 
@@ -1612,7 +1621,8 @@ GetDirectoryContent(const std::filesystem::path& RootDir,
 		virtual void VisitFile(const std::filesystem::path&,
 							   const path_view& File,
 							   uint64_t			FileSize,
-							   uint32_t			NativeModeOrAttributes) override
+							   uint32_t			NativeModeOrAttributes,
+							   uint64_t			NativeModificationTick) override
 		{
 			if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles))
 			{
@@ -1625,6 +1635,10 @@ GetDirectoryContent(const std::filesystem::path& RootDir,
 				{
 					Content.FileAttributes.push_back(NativeModeOrAttributes);
 				}
+				if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeModificationTick))
+				{
+					Content.FileModificationTicks.push_back(NativeModificationTick);
+				}
 			}
 		}
 
@@ -1928,7 +1942,7 @@ TEST_CASE("filesystem")
 	// Traversal
 	struct : public FileSystemTraversal::TreeVisitor
 	{
-		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t, uint32_t) override
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t, uint32_t, uint64_t) override
 		{
 			bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected);
 		}
diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h
index 7edd40c9c..a78132879 100644
--- a/src/zencore/include/zencore/basicfile.h
+++ b/src/zencore/include/zencore/basicfile.h
@@ -60,6 +60,7 @@ public:
 	void	 StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
 	void	 Write(MemoryView Data, uint64_t FileOffset);
 	void	 Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec);
+	uint64_t Write(const CompositeBuffer& Data, uint64_t FileOffset);
 	uint64_t Write(const CompositeBuffer& Data, uint64_t FileOffset, std::error_code& Ec);
 	void	 Write(const void* Data, uint64_t Size, uint64_t FileOffset);
 	void	 Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec);
@@ -174,6 +175,7 @@ public:
 	~BasicFileWriter();
 
 	void Write(const void* Data, uint64_t Size, uint64_t FileOffset);
+	void Write(const CompositeBuffer& Data, uint64_t FileOffset);
 	void Flush();
 
 private:
@@ -184,9 +186,7 @@ private:
 	uint64_t	   m_BufferEnd;
 };
 
-IoBuffer WriteToTempFile(CompositeBuffer&&							Buffer,
-						 const std::filesystem::path&				Path,
-						 std::function<bool(std::error_code& Ec)>&& RetryCallback);
+IoBuffer WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path);
 
 ZENCORE_API void basicfile_forcelink();
 
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index ca8682cd7..250745e86 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -203,7 +203,8 @@ public:
 		virtual void VisitFile(const std::filesystem::path& Parent,
 							   const path_view&				File,
 							   uint64_t						FileSize,
-							   uint32_t						NativeModeOrAttributes) = 0;
+							   uint32_t						NativeModeOrAttributes,
+							   uint64_t						NativeModificationTick) = 0;
 
 		// This should return true if we should recurse into the directory
 		virtual bool VisitDirectory(const std::filesystem::path& Parent,
@@ -216,13 +217,14 @@ public:
 
 enum class DirectoryContentFlags : uint8_t
 {
-	None			  = 0,
-	IncludeDirs		  = 1u << 0,
-	IncludeFiles	  = 1u << 1,
-	Recursive		  = 1u << 2,
-	IncludeFileSizes  = 1u << 3,
-	IncludeAttributes = 1u << 4,
-	IncludeAllEntries = IncludeDirs | IncludeFiles | Recursive
+	None					= 0,
+	IncludeDirs				= 1u << 0,
+	IncludeFiles			= 1u << 1,
+	Recursive				= 1u << 2,
+	IncludeFileSizes		= 1u << 3,
+	IncludeAttributes		= 1u << 4,
+	IncludeModificationTick = 1u << 5,
+	IncludeAllEntries		= IncludeDirs | IncludeFiles | Recursive
 };
 
 ENUM_CLASS_FLAGS(DirectoryContentFlags)
@@ -232,6 +234,7 @@ struct DirectoryContent
 	std::vector<std::filesystem::path> Files;
 	std::vector<uint64_t>			   FileSizes;
 	std::vector<uint32_t>			   FileAttributes;
+	std::vector<uint64_t>			   FileModificationTicks;
 	std::vector<std::filesystem::path> Directories;
 	std::vector<uint32_t>			   DirectoryAttributes;
 };
@@ -246,6 +249,7 @@ public:
 		std::vector<std::filesystem::path> FileNames;
 		std::vector<uint64_t>			   FileSizes;
 		std::vector<uint32_t>			   FileAttributes;
+		std::vector<uint64_t>			   FileModificationTicks;
 		std::vector<std::filesystem::path> DirectoryNames;
 		std::vector<uint32_t>			   DirectoryAttributes;
 	};
diff --git a/src/zencore/include/zencore/iohash.h b/src/zencore/include/zencore/iohash.h
index 8871a5895..7443e17b7 100644
--- a/src/zencore/include/zencore/iohash.h
+++ b/src/zencore/include/zencore/iohash.h
@@ -47,8 +47,8 @@ struct IoHash
 
 	static IoHash	   HashBuffer(const void* data, size_t byteCount);
 	static IoHash	   HashBuffer(MemoryView Data) { return HashBuffer(Data.GetData(), Data.GetSize()); }
-	static IoHash	   HashBuffer(const CompositeBuffer& Buffer);
-	static IoHash	   HashBuffer(const IoBuffer& Buffer);
+	static IoHash	   HashBuffer(const CompositeBuffer& Buffer, std::atomic<uint64_t>* ProcessedBytes = nullptr);
+	static IoHash	   HashBuffer(const IoBuffer& Buffer, std::atomic<uint64_t>* ProcessedBytes = nullptr);
 	static IoHash	   FromHexString(const char* string);
 	static IoHash	   FromHexString(const std::string_view string);
 	static bool		   TryParse(std::string_view Str, IoHash& Hash);
diff --git a/src/zencore/iohash.cpp b/src/zencore/iohash.cpp
index 7200e6e3f..3b2af0db4 100644
--- a/src/zencore/iohash.cpp
+++ b/src/zencore/iohash.cpp
@@ -30,7 +30,7 @@ IoHash::HashBuffer(const void* data, size_t byteCount)
 }
 
 IoHash
-IoHash::HashBuffer(const CompositeBuffer& Buffer)
+IoHash::HashBuffer(const CompositeBuffer& Buffer, std::atomic<uint64_t>* ProcessedBytes)
 {
 	IoHashStream Hasher;
 
@@ -46,11 +46,21 @@ IoHash::HashBuffer(const CompositeBuffer& Buffer)
 					 FileRef.FileChunkOffset,
 					 FileRef.FileChunkSize,
 					 BufferingSize,
-					 [&Hasher](const void* Data, size_t Size) { Hasher.Append(Data, Size); });
+					 [&Hasher, ProcessedBytes](const void* Data, size_t Size) {
+						 Hasher.Append(Data, Size);
+						 if (ProcessedBytes != nullptr)
+						 {
+							 ProcessedBytes->fetch_add(Size);
+						 }
+					 });
 		}
 		else
 		{
 			Hasher.Append(Segment.GetData(), SegmentSize);
+			if (ProcessedBytes != nullptr)
+			{
+				ProcessedBytes->fetch_add(SegmentSize);
+			}
 		}
 	}
 
@@ -58,7 +68,7 @@ IoHash::HashBuffer(const CompositeBuffer& Buffer)
 }
 
 IoHash
-IoHash::HashBuffer(const IoBuffer& Buffer)
+IoHash::HashBuffer(const IoBuffer& Buffer, std::atomic<uint64_t>* ProcessedBytes)
 {
 	IoHashStream Hasher;
 
@@ -71,11 +81,21 @@ IoHash::HashBuffer(const IoBuffer& Buffer)
 				 FileRef.FileChunkOffset,
 				 FileRef.FileChunkSize,
 				 BufferingSize,
-				 [&Hasher](const void* Data, size_t Size) { Hasher.Append(Data, Size); });
+				 [&Hasher, ProcessedBytes](const void* Data, size_t Size) {
+					 Hasher.Append(Data, Size);
+					 if (ProcessedBytes != nullptr)
+					 {
+						 ProcessedBytes->fetch_add(Size);
+					 }
+				 });
 	}
 	else
 	{
 		Hasher.Append(Buffer.GetData(), BufferSize);
+		if (ProcessedBytes != nullptr)
+		{
+			ProcessedBytes->fetch_add(BufferSize);
+		}
 	}
 
 	return Hasher.GetHash();