use direct file access for large file hash (#63)

- Improvement: Refactor `IoHash::HashBuffer` and `BLAKE3::HashBuffer` to not use memory mapped files. Performs better and saves ~10% of oplog export time on CI

1 files changed, 41 insertions, 1 deletions

diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index ca2b3101f..e41149d64 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -1015,6 +1015,46 @@ ReadFile(std::filesystem::path Path)
 	return Contents;
 }
 
+ZENCORE_API void
+ScanFile(void*												  NativeHandle,
+		 uint64_t											  Offset,
+		 uint64_t											  Size,
+		 uint64_t											  ChunkSize,
+		 std::function<void(const void* Data, size_t Size)>&& ProcessFunc)
+{
+	ZEN_ASSERT(NativeHandle != nullptr);
+	uint64_t			 BufferSize = Min(ChunkSize, Size);
+	std::vector<uint8_t> ReadBuffer(BufferSize);
+	uint64_t			 ReadOffset = 0;
+	while (ReadOffset < Size)
+	{
+		const uint64_t NumberOfBytesToRead = Min(Size - ReadOffset, BufferSize);
+		uint64_t	   FileOffset		   = Offset + ReadOffset;
+
+#if ZEN_PLATFORM_WINDOWS
+		OVERLAPPED Ovl{};
+
+		Ovl.Offset	   = DWORD(FileOffset & 0xffff'ffffu);
+		Ovl.OffsetHigh = DWORD(FileOffset >> 32);
+
+		DWORD BytesRead = 0;
+		BOOL  Success	= ::ReadFile(NativeHandle, ReadBuffer.data(), DWORD(NumberOfBytesToRead), &BytesRead, &Ovl);
+		if (!Success)
+		{
+			throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "file scan failed");
+		}
+#else
+		   int BytesRead = pread(int(intptr_t(NativeHandle)), ReadBuffer.data(), size_t(NumberOfBytesToRead), off_t(FileOffset));
+		   if (BytesRead < 0)
+		   {
+			   throw std::system_error(std::error_code(errno, std::system_category()), "file scan failed");
+		   }
+#endif
+		ProcessFunc(ReadBuffer.data(), (size_t)BytesRead);
+		ReadOffset += (uint64_t)BytesRead;
+	}
+}
+
 bool
 ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc)
 {
@@ -1050,7 +1090,7 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<voi
 		ProcessFunc(ReadBuffer.data(), dwBytesRead);
 	}
 #else
-	int Fd		  = open(Path.c_str(), O_RDONLY | O_CLOEXEC);
+	int Fd = open(Path.c_str(), O_RDONLY | O_CLOEXEC);
 	if (Fd < 0)
 	{
 		return false;