From 4d95b578350ebfbbf6d54407c9403547b01cac4c Mon Sep 17 00:00:00 2001
From: Stefan Boberg <stefan@boberg.org>
Date: Mon, 27 Nov 2023 14:32:19 +0100
Subject: optimized index snapshot reading/writing (#561)

the previous implementation of in-memory index snapshots serialise data to memory before writing to disk and vice versa when reading. This leads to some memory spikes which end up pushing useful data out of system cache and also cause stalls on I/O operations.

this change moves more code to a streaming serialisation approach which scales better from a memory usage perspective and also performs much better
---
 src/zencore/filesystem.cpp | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

(limited to 'src/zencore/filesystem.cpp')

diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 85f7690bd..e9b147be5 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -334,11 +334,17 @@ SupportsBlockRefCounting(std::filesystem::path Path)
 #endif	// ZEN_PLATFORM_WINDOWS
 }
 
-bool
+static bool
 CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 {
 #if ZEN_PLATFORM_WINDOWS
-	windows::Handle FromFile(CreateFileW(FromPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
+	windows::Handle FromFile(CreateFileW(FromPath.c_str(),
+										 GENERIC_READ,
+										 FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+										 nullptr,
+										 OPEN_EXISTING,
+										 0,
+										 nullptr));
 	if (FromFile == INVALID_HANDLE_VALUE)
 	{
 		FromFile.Detach();
@@ -402,8 +408,10 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 	FILE_DISPOSITION_INFO FileDisposition = {TRUE};
 	if (!SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition))
 	{
+		const DWORD ErrorCode = ::GetLastError();
 		TargetFile.Close();
 		DeleteFileW(ToPath.c_str());
+		SetLastError(ErrorCode);
 		return false;
 	}
 
@@ -532,6 +540,19 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 #endif	// ZEN_PLATFORM_WINDOWS
 }
 
+void
+CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options, std::error_code& OutErrorCode)
+{
+	OutErrorCode.clear();
+
+	bool Success = CopyFile(FromPath, ToPath, Options);
+
+	if (!Success)
+	{
+		OutErrorCode = MakeErrorCodeFromLastError();
+	}
+}
+
 bool
 CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options)
 {
-- 
cgit v1.2.3