From 5bc5b0dd59c0f02afe553e5074dfe57951b19044 Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Tue, 25 Feb 2025 15:48:43 +0100
Subject: improvements and infrastructure for upcoming builds api command line
 (#284)

* add modification tick to filesystem traversal
* add ShowDetails option to ProgressBar
* log callstack if we terminate process
* handle chunking if MaxSize > 1MB
* BasicFile write helpers and WriteToTempFile simplifications
* bugfix for CompositeBuffer::IterateRange when using DecompressToComposite for actually comrpessed data
revert of earlier optimization
* faster compress/decompress for large disk-based files
* enable progress feedback in IoHash::HashBuffer
* add payload validation in HttpClient::Get
* fix range requests (range is including end byte)
* remove BuildPartId for blob/block related operations in builds api
---
 src/zencore/basicfile.cpp | 79 +++++++++++++++++++++++++++++++++++------------
 1 file changed, 60 insertions(+), 19 deletions(-)

(limited to 'src/zencore/basicfile.cpp')

diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp
index b3bffd34d..6e879ca0d 100644
--- a/src/zencore/basicfile.cpp
+++ b/src/zencore/basicfile.cpp
@@ -280,6 +280,20 @@ BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<voi
 	}
 }
 
+uint64_t
+BasicFile::Write(const CompositeBuffer& Data, uint64_t FileOffset)
+{
+	std::error_code Ec;
+	uint64_t		WrittenBytes = Write(Data, FileOffset, Ec);
+
+	if (Ec)
+	{
+		std::error_code Dummy;
+		throw std::system_error(Ec, fmt::format("Failed to write to file '{}'", zen::PathFromHandle(m_FileHandle, Dummy)));
+	}
+	return WrittenBytes;
+}
+
 uint64_t
 BasicFile::Write(const CompositeBuffer& Data, uint64_t FileOffset, std::error_code& Ec)
 {
@@ -309,6 +323,8 @@ BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec)
 void
 BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec)
 {
+	ZEN_ASSERT(m_FileHandle != nullptr);
+
 	Ec.clear();
 
 	const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
@@ -816,6 +832,17 @@ BasicFileWriter::Write(const void* Data, uint64_t Size, uint64_t FileOffset)
 	}
 }
 
+void
+BasicFileWriter::Write(const CompositeBuffer& Data, uint64_t FileOffset)
+{
+	for (const SharedBuffer& Segment : Data.GetSegments())
+	{
+		const uint64_t SegmentSize = Segment.GetSize();
+		Write(Segment.GetData(), SegmentSize, FileOffset);
+		FileOffset += SegmentSize;
+	}
+}
+
 void
 BasicFileWriter::Flush()
 {
@@ -831,23 +858,20 @@ BasicFileWriter::Flush()
 }
 
 IoBuffer
-WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path, std::function<bool(std::error_code& Ec)>&& RetryCallback)
+WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path)
 {
-	if (std::filesystem::is_regular_file(Path))
+	TemporaryFile	Temp;
+	std::error_code Ec;
+	Temp.CreateTemporary(Path.parent_path(), Ec);
+	if (Ec)
 	{
-		IoBuffer ExistingTempFile = IoBuffer(IoBufferBuilder::MakeFromFile(Path));
-		if (ExistingTempFile && ExistingTempFile.GetSize() == Buffer.GetSize())
-		{
-			ExistingTempFile.SetDeleteOnClose(true);
-			return ExistingTempFile;
-		}
+		throw std::system_error(Ec, fmt::format("Failed to create temp file for blob at '{}'", Path));
 	}
-	BasicFile BlockFile;
-	BlockFile.Open(Path, BasicFile::Mode::kTruncateDelete, std::move(RetryCallback));
-	uint64_t Offset = 0;
+
 	{
+		uint64_t			  Offset		= 0;
 		static const uint64_t BufferingSize = 256u * 1024u;
-		BasicFileWriter		  BufferedOutput(BlockFile, BufferingSize / 2);
+		//		BasicFileWriter		  BufferedOutput(BlockFile, BufferingSize / 2);
 		for (const SharedBuffer& Segment : Buffer.GetSegments())
 		{
 			size_t SegmentSize = Segment.GetSize();
@@ -859,22 +883,39 @@ WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path, std
 						 FileRef.FileChunkOffset,
 						 FileRef.FileChunkSize,
 						 BufferingSize,
-						 [&BufferedOutput, &Offset](const void* Data, size_t Size) {
-							 BufferedOutput.Write(Data, Size, Offset);
+						 [&Temp, &Offset](const void* Data, size_t Size) {
+							 Temp.Write(Data, Size, Offset);
 							 Offset += Size;
 						 });
 			}
 			else
 			{
-				BufferedOutput.Write(Segment.GetData(), SegmentSize, Offset);
+				Temp.Write(Segment.GetData(), SegmentSize, Offset);
 				Offset += SegmentSize;
 			}
 		}
 	}
-	void*	 FileHandle	 = BlockFile.Detach();
-	IoBuffer BlockBuffer = IoBuffer(IoBuffer::File, FileHandle, 0, Offset, /*IsWholeFile*/ true);
-	BlockBuffer.SetDeleteOnClose(true);
-	return BlockBuffer;
+
+	Temp.MoveTemporaryIntoPlace(Path, Ec);
+	if (Ec)
+	{
+		IoBuffer TmpBuffer = IoBufferBuilder::MakeFromFile(Path);
+		if (TmpBuffer)
+		{
+			IoHash		 ExistingHash = IoHash::HashBuffer(TmpBuffer);
+			const IoHash ExpectedHash = IoHash::HashBuffer(Buffer);
+			if (ExistingHash == ExpectedHash)
+			{
+				TmpBuffer.SetDeleteOnClose(true);
+				return TmpBuffer;
+			}
+		}
+		throw std::system_error(Ec, fmt::format("Failed to move temp file to '{}'", Path));
+	}
+
+	IoBuffer TmpBuffer = IoBufferBuilder::MakeFromFile(Path);
+	TmpBuffer.SetDeleteOnClose(true);
+	return TmpBuffer;
 }
 
 //////////////////////////////////////////////////////////////////////////
-- 
cgit v1.2.3