optimized index snapshot reading/writing (#561)

the previous implementation of in-memory index snapshots serialise data to memory before writing to disk and vice versa when reading. This leads to some memory spikes which end up pushing useful data out of system cache and also cause stalls on I/O operations.

this change moves more code to a streaming serialisation approach which scales better from a memory usage perspective and also performs much better

1 files changed, 23 insertions, 2 deletions

diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 85f7690bd..e9b147be5 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -334,11 +334,17 @@ SupportsBlockRefCounting(std::filesystem::path Path)
 #endif	// ZEN_PLATFORM_WINDOWS
 }
 
-bool
+static bool
 CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 {
 #if ZEN_PLATFORM_WINDOWS
-	windows::Handle FromFile(CreateFileW(FromPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
+	windows::Handle FromFile(CreateFileW(FromPath.c_str(),
+										 GENERIC_READ,
+										 FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+										 nullptr,
+										 OPEN_EXISTING,
+										 0,
+										 nullptr));
 	if (FromFile == INVALID_HANDLE_VALUE)
 	{
 		FromFile.Detach();
@@ -402,8 +408,10 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 	FILE_DISPOSITION_INFO FileDisposition = {TRUE};
 	if (!SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition))
 	{
+		const DWORD ErrorCode = ::GetLastError();
 		TargetFile.Close();
 		DeleteFileW(ToPath.c_str());
+		SetLastError(ErrorCode);
 		return false;
 	}
 
@@ -532,6 +540,19 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 #endif	// ZEN_PLATFORM_WINDOWS
 }
 
+void
+CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options, std::error_code& OutErrorCode)
+{
+	OutErrorCode.clear();
+
+	bool Success = CopyFile(FromPath, ToPath, Options);
+
+	if (!Success)
+	{
+		OutErrorCode = MakeErrorCodeFromLastError();
+	}
+}
+
 bool
 CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options)
 {