optimized index snapshot reading/writing (#561)

the previous implementation of in-memory index snapshots serialise data to memory before writing to disk and vice versa when reading. This leads to some memory spikes which end up pushing useful data out of system cache and also cause stalls on I/O operations.

this change moves more code to a streaming serialisation approach which scales better from a memory usage perspective and also performs much better

3 files changed, 28 insertions, 2 deletions

diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 85f7690bd..e9b147be5 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -334,11 +334,17 @@ SupportsBlockRefCounting(std::filesystem::path Path)
 #endif	// ZEN_PLATFORM_WINDOWS
 }
 
-bool
+static bool
 CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 {
 #if ZEN_PLATFORM_WINDOWS
-	windows::Handle FromFile(CreateFileW(FromPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
+	windows::Handle FromFile(CreateFileW(FromPath.c_str(),
+										 GENERIC_READ,
+										 FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+										 nullptr,
+										 OPEN_EXISTING,
+										 0,
+										 nullptr));
 	if (FromFile == INVALID_HANDLE_VALUE)
 	{
 		FromFile.Detach();
@@ -402,8 +408,10 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 	FILE_DISPOSITION_INFO FileDisposition = {TRUE};
 	if (!SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition))
 	{
+		const DWORD ErrorCode = ::GetLastError();
 		TargetFile.Close();
 		DeleteFileW(ToPath.c_str());
+		SetLastError(ErrorCode);
 		return false;
 	}
 
@@ -532,6 +540,19 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 #endif	// ZEN_PLATFORM_WINDOWS
 }
 
+void
+CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options, std::error_code& OutErrorCode)
+{
+	OutErrorCode.clear();
+
+	bool Success = CopyFile(FromPath, ToPath, Options);
+
+	if (!Success)
+	{
+		OutErrorCode = MakeErrorCodeFromLastError();
+	}
+}
+
 bool
 CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options)
 {
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index 07eb72879..233941479 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -86,6 +86,10 @@ struct CopyFileOptions
 };
 
 ZENCORE_API bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options);
+ZENCORE_API void CopyFile(std::filesystem::path	 FromPath,
+						  std::filesystem::path	 ToPath,
+						  const CopyFileOptions& Options,
+						  std::error_code&		 OutError);
 ZENCORE_API void CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options);
 ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path);
 
diff --git a/src/zencore/include/zencore/windows.h b/src/zencore/include/zencore/windows.h
index 0943a85ea..14026fef1 100644
--- a/src/zencore/include/zencore/windows.h
+++ b/src/zencore/include/zencore/windows.h
@@ -24,6 +24,7 @@ struct IUnknown;  // Workaround for "combaseapi.h(229): error C2187: syntax erro
 #	include <windows.h>
 #	undef GetObject
 #	undef SendMessage
+#	undef CopyFile
 
 ZEN_THIRD_PARTY_INCLUDES_END