Merge branch 'main' into zs/put-overwrite-policy

1 files changed, 172 insertions, 138 deletions

diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp
index c2a21ae90..6989da67e 100644
--- a/src/zencore/basicfile.cpp
+++ b/src/zencore/basicfile.cpp
@@ -28,6 +28,20 @@ BasicFile::~BasicFile()
 {
 	Close();
 }
+BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode)
+{
+	Open(FileName, Mode);
+}
+
+BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec)
+{
+	Open(FileName, Mode, Ec);
+}
+
+BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode, std::function<bool(std::error_code& Ec)>&& RetryCallback)
+{
+	Open(FileName, Mode, std::move(RetryCallback));
+}
 
 void
 BasicFile::Open(const std::filesystem::path& FileName, Mode Mode)
@@ -167,58 +181,18 @@ BasicFile::ReadRange(uint64_t FileOffset, uint64_t ByteCount)
 void
 BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset)
 {
-	const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
-
-	while (BytesToRead)
+	const uint64_t	MaxChunkSize = 2u * 1024 * 1024 * 1024;
+	std::error_code Ec;
+	ReadFile(m_FileHandle, Data, BytesToRead, FileOffset, MaxChunkSize, Ec);
+	if (Ec)
 	{
-		const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize);
-		int32_t		   Error			   = 0;
-		size_t		   BytesRead		   = 0;
-
-#if ZEN_PLATFORM_WINDOWS
-		OVERLAPPED Ovl{};
-
-		Ovl.Offset	   = DWORD(FileOffset & 0xffff'ffffu);
-		Ovl.OffsetHigh = DWORD(FileOffset >> 32);
-
-		DWORD dwNumberOfBytesRead = 0;
-		BOOL  Success			  = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl);
-		if (Success)
-		{
-			BytesRead = size_t(dwNumberOfBytesRead);
-		}
-		else
-		{
-			Error = zen::GetLastError();
-		}
-#else
-		static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
-		int		Fd		   = int(uintptr_t(m_FileHandle));
-		ssize_t ReadResult = pread(Fd, Data, NumberOfBytesToRead, FileOffset);
-		if (ReadResult != -1)
-		{
-			BytesRead = size_t(ReadResult);
-		}
-		else
-		{
-			Error = zen::GetLastError();
-		}
-#endif
-
-		if (Error || (BytesRead != NumberOfBytesToRead))
-		{
-			std::error_code DummyEc;
-			throw std::system_error(std::error_code(Error, std::system_category()),
-									fmt::format("ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})",
-												FileOffset,
-												NumberOfBytesToRead,
-												PathFromHandle(m_FileHandle, DummyEc),
-												FileSizeFromHandle(m_FileHandle)));
-		}
-
-		BytesToRead -= NumberOfBytesToRead;
-		FileOffset += NumberOfBytesToRead;
-		Data = reinterpret_cast<uint8_t*>(Data) + NumberOfBytesToRead;
+		std::error_code DummyEc;
+		throw std::system_error(Ec,
+								fmt::format("BasicFile::Read: ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})",
+											FileOffset,
+											BytesToRead,
+											PathFromHandle(m_FileHandle, DummyEc),
+											FileSizeFromHandle(m_FileHandle)));
 	}
 }
 
@@ -267,7 +241,21 @@ BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<voi
 }
 
 uint64_t
-BasicFile::Write(CompositeBuffer Data, uint64_t FileOffset, std::error_code& Ec)
+BasicFile::Write(const CompositeBuffer& Data, uint64_t FileOffset)
+{
+	std::error_code Ec;
+	uint64_t		WrittenBytes = Write(Data, FileOffset, Ec);
+
+	if (Ec)
+	{
+		std::error_code Dummy;
+		throw std::system_error(Ec, fmt::format("Failed to write to file '{}'", zen::PathFromHandle(m_FileHandle, Dummy)));
+	}
+	return WrittenBytes;
+}
+
+uint64_t
+BasicFile::Write(const CompositeBuffer& Data, uint64_t FileOffset, std::error_code& Ec)
 {
 	uint64_t WrittenBytes = 0;
 	for (const SharedBuffer& Buffer : Data.GetSegments())
@@ -295,41 +283,9 @@ BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec)
 void
 BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec)
 {
-	Ec.clear();
-
-	const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
-
-	while (Size)
-	{
-		const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize);
-
-#if ZEN_PLATFORM_WINDOWS
-		OVERLAPPED Ovl{};
-
-		Ovl.Offset	   = DWORD(FileOffset & 0xffff'ffffu);
-		Ovl.OffsetHigh = DWORD(FileOffset >> 32);
-
-		DWORD dwNumberOfBytesWritten = 0;
-
-		BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl);
-#else
-		static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
-		int	 Fd			  = int(uintptr_t(m_FileHandle));
-		int	 BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, FileOffset);
-		bool Success	  = (BytesWritten > 0);
-#endif
-
-		if (!Success)
-		{
-			Ec = MakeErrorCodeFromLastError();
+	const uint64_t MaxChunkSize = 2u * 1024 * 1024;
 
-			return;
-		}
-
-		Size -= NumberOfBytesToWrite;
-		FileOffset += NumberOfBytesToWrite;
-		Data = reinterpret_cast<const uint8_t*>(Data) + NumberOfBytesToWrite;
-	}
+	WriteFile(m_FileHandle, Data, Size, FileOffset, MaxChunkSize, Ec);
 }
 
 void
@@ -375,59 +331,20 @@ BasicFile::Flush()
 uint64_t
 BasicFile::FileSize() const
 {
-#if ZEN_PLATFORM_WINDOWS
-	ULARGE_INTEGER liFileSize;
-	liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart);
-	if (liFileSize.LowPart == INVALID_FILE_SIZE)
-	{
-		int Error = zen::GetLastError();
-		if (Error)
-		{
-			std::error_code Dummy;
-			ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy)));
-		}
-	}
-	return uint64_t(liFileSize.QuadPart);
-#else
-	int Fd = int(uintptr_t(m_FileHandle));
-	static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
-	struct stat Stat;
-	if (fstat(Fd, &Stat) == -1)
+	std::error_code Ec;
+	uint64_t		FileSize = FileSizeFromHandle(m_FileHandle, Ec);
+	if (Ec)
 	{
 		std::error_code Dummy;
-		ThrowSystemError(GetLastError(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy)));
+		ThrowSystemError(Ec.value(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy)));
 	}
-	return uint64_t(Stat.st_size);
-#endif
+	return FileSize;
 }
 
 uint64_t
 BasicFile::FileSize(std::error_code& Ec) const
 {
-#if ZEN_PLATFORM_WINDOWS
-	ULARGE_INTEGER liFileSize;
-	liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart);
-	if (liFileSize.LowPart == INVALID_FILE_SIZE)
-	{
-		int Error = zen::GetLastError();
-		if (Error)
-		{
-			Ec = MakeErrorCode(Error);
-			return 0;
-		}
-	}
-	return uint64_t(liFileSize.QuadPart);
-#else
-	int Fd = int(uintptr_t(m_FileHandle));
-	static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
-	struct stat Stat;
-	if (fstat(Fd, &Stat) == -1)
-	{
-		Ec = MakeErrorCodeFromLastError();
-		return 0;
-	}
-	return uint64_t(Stat.st_size);
-#endif
+	return FileSizeFromHandle(m_FileHandle, Ec);
 }
 
 void
@@ -560,7 +477,7 @@ TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::
 	// deleting the temporary file
 	BasicFile::Close();
 
-	std::filesystem::rename(m_TempPath, FinalFileName, Ec);
+	RenameFile(m_TempPath, FinalFileName, Ec);
 
 	if (Ec)
 	{
@@ -575,7 +492,6 @@ TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std::
 void
 TemporaryFile::SafeWriteFile(const std::filesystem::path& Path, MemoryView Data)
 {
-	TemporaryFile	TempFile;
 	std::error_code Ec;
 	SafeWriteFile(Path, Data, Ec);
 	if (Ec)
@@ -763,10 +679,45 @@ BasicFileWriter::~BasicFileWriter()
 }
 
 void
+BasicFileWriter::AddPadding(uint64_t Padding)
+{
+	while (Padding)
+	{
+		const uint64_t BufferOffset			   = m_BufferEnd - m_BufferStart;
+		const uint64_t RemainingBufferCapacity = m_BufferSize - BufferOffset;
+		const uint64_t BlockPadBytes		   = Min(RemainingBufferCapacity, Padding);
+
+		memset(m_Buffer + BufferOffset, 0, BlockPadBytes);
+		m_BufferEnd += BlockPadBytes;
+		Padding -= BlockPadBytes;
+
+		if ((BufferOffset + BlockPadBytes) == m_BufferSize)
+		{
+			Flush();
+		}
+	}
+}
+
+uint64_t
+BasicFileWriter::AlignTo(uint64_t Alignment)
+{
+	uint64_t AlignedPos = RoundUp(m_BufferEnd, Alignment);
+	uint64_t Padding	= AlignedPos - m_BufferEnd;
+	AddPadding(Padding);
+	return AlignedPos;
+}
+
+void
 BasicFileWriter::Write(const void* Data, uint64_t Size, uint64_t FileOffset)
 {
 	if (m_Buffer == nullptr || (Size >= m_BufferSize))
 	{
+		if (FileOffset == m_BufferEnd)
+		{
+			Flush();
+			m_BufferStart = m_BufferEnd = FileOffset + Size;
+		}
+
 		m_Base.Write(Data, Size, FileOffset);
 		return;
 	}
@@ -804,6 +755,17 @@ BasicFileWriter::Write(const void* Data, uint64_t Size, uint64_t FileOffset)
 }
 
 void
+BasicFileWriter::Write(const CompositeBuffer& Data, uint64_t FileOffset)
+{
+	for (const SharedBuffer& Segment : Data.GetSegments())
+	{
+		const uint64_t SegmentSize = Segment.GetSize();
+		Write(Segment.GetData(), SegmentSize, FileOffset);
+		FileOffset += SegmentSize;
+	}
+}
+
+void
 BasicFileWriter::Flush()
 {
 	const uint64_t BufferedBytes = m_BufferEnd - m_BufferStart;
@@ -817,6 +779,78 @@ BasicFileWriter::Flush()
 	m_Base.Write(m_Buffer, BufferedBytes, WriteOffset);
 }
 
+IoBuffer
+WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path)
+{
+	TemporaryFile	Temp;
+	std::error_code Ec;
+	Temp.CreateTemporary(Path.parent_path(), Ec);
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("Failed to create temp file for blob at '{}'", Path));
+	}
+
+	uint64_t BufferSize = Buffer.GetSize();
+	{
+		uint64_t			  Offset		= 0;
+		static const uint64_t BufferingSize = 256u * 1024u;
+		BasicFileWriter		  BufferedOutput(Temp, Min(BufferingSize, BufferSize));
+		for (const SharedBuffer& Segment : Buffer.GetSegments())
+		{
+			size_t SegmentSize = Segment.GetSize();
+
+			IoBufferFileReference FileRef;
+			if (SegmentSize >= (BufferingSize + BufferingSize / 2) && Segment.GetFileReference(FileRef))
+			{
+				ScanFile(FileRef.FileHandle,
+						 FileRef.FileChunkOffset,
+						 FileRef.FileChunkSize,
+						 BufferingSize,
+						 [&BufferedOutput, &Offset](const void* Data, size_t Size) {
+							 BufferedOutput.Write(Data, Size, Offset);
+							 Offset += Size;
+						 });
+			}
+			else
+			{
+				BufferedOutput.Write(Segment.GetData(), SegmentSize, Offset);
+				Offset += SegmentSize;
+			}
+		}
+	}
+
+	Temp.MoveTemporaryIntoPlace(Path, Ec);
+	if (Ec)
+	{
+		Ec.clear();
+		BasicFile OpenTemp(Path, BasicFile::Mode::kDelete, Ec);
+		if (Ec)
+		{
+			throw std::system_error(Ec, fmt::format("Failed to move temp file to '{}'", Path));
+		}
+		if (OpenTemp.FileSize() != BufferSize)
+		{
+			throw std::runtime_error(fmt::format("Failed to move temp file to '{}' - mismatching file size already exists", Path));
+		}
+		IoBuffer TmpBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BufferSize, true);
+
+		IoHash		 ExistingHash = IoHash::HashBuffer(TmpBuffer);
+		const IoHash ExpectedHash = IoHash::HashBuffer(Buffer);
+		if (ExistingHash != ExpectedHash)
+		{
+			throw std::runtime_error(fmt::format("Failed to move temp file to '{}' - mismatching file hash already exists", Path));
+		}
+		Buffer = CompositeBuffer{};
+		TmpBuffer.SetDeleteOnClose(true);
+		return TmpBuffer;
+	}
+	Buffer = CompositeBuffer{};
+	BasicFile OpenTemp(Path, BasicFile::Mode::kDelete);
+	IoBuffer  TmpBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BufferSize, true);
+	TmpBuffer.SetDeleteOnClose(true);
+	return TmpBuffer;
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 /*
@@ -866,9 +900,9 @@ TEST_CASE("TemporaryFile")
 			TmpFile.CreateTemporary(std::filesystem::current_path(), Ec);
 			CHECK(!Ec);
 			Path = TmpFile.GetPath();
-			CHECK(std::filesystem::exists(Path));
+			CHECK(IsFile(Path));
 		}
-		CHECK(std::filesystem::exists(Path) == false);
+		CHECK(IsFile(Path) == false);
 	}
 
 	SUBCASE("MoveIntoPlace")
@@ -879,11 +913,11 @@ TEST_CASE("TemporaryFile")
 		CHECK(!Ec);
 		std::filesystem::path TempPath	= TmpFile.GetPath();
 		std::filesystem::path FinalPath = std::filesystem::current_path() / "final";
-		CHECK(std::filesystem::exists(TempPath));
+		CHECK(IsFile(TempPath));
 		TmpFile.MoveTemporaryIntoPlace(FinalPath, Ec);
 		CHECK(!Ec);
-		CHECK(std::filesystem::exists(TempPath) == false);
-		CHECK(std::filesystem::exists(FinalPath));
+		CHECK(IsFile(TempPath) == false);
+		CHECK(IsFile(FinalPath));
 	}
 }