optimized index snapshot reading/writing (#561)

the previous implementation of in-memory index snapshots serialise data to memory before writing to disk and vice versa when reading. This leads to some memory spikes which end up pushing useful data out of system cache and also cause stalls on I/O operations.

this change moves more code to a streaming serialisation approach which scales better from a memory usage perspective and also performs much better

2 files changed, 36 insertions, 1 deletions

diff --git a/src/zenutil/include/zenutil/basicfile.h b/src/zenutil/include/zenutil/basicfile.h
index 7797258e8..f25d9f23c 100644
--- a/src/zenutil/include/zenutil/basicfile.h
+++ b/src/zenutil/include/zenutil/basicfile.h
@@ -44,7 +44,6 @@ public:
 		kModeMask	   = 0x0007,
 		kPreventDelete = 0x1000'0000,  // Do not open with delete sharing mode (prevent other processes from deleting file while open)
 		kPreventWrite  = 0x2000'0000,  // Do not open with write sharing mode (prevent other processes from writing to file while open)
-		kDeleteOnClose = 0x4000'0000,  // File should be deleted when the last handle is closed
 	};
 
 	void	 Open(const std::filesystem::path& FileName, Mode Mode);
@@ -138,6 +137,13 @@ public:
 	void	   Read(void* Data, uint64_t Size, uint64_t FileOffset);
 	MemoryView MakeView(uint64_t Size, uint64_t FileOffset);
 
+	template<typename T>
+	const T* MakeView(uint64_t FileOffset)
+	{
+		MemoryView View = MakeView(sizeof(T), FileOffset);
+		return reinterpret_cast<const T*>(View.GetData());
+	}
+
 private:
 	BasicFile&	   m_Base;
 	uint8_t*	   m_Buffer;
@@ -147,6 +153,29 @@ private:
 	uint64_t	   m_BufferEnd;
 };
 
+/** Adds a layer of buffered writing to a BasicFile
+
+This class is not intended for concurrent access, it is not thread safe.
+
+*/
+
+class BasicFileWriter
+{
+public:
+	BasicFileWriter(BasicFile& Base, uint64_t BufferSize);
+	~BasicFileWriter();
+
+	void Write(void* Data, uint64_t Size, uint64_t FileOffset);
+	void Flush();
+
+private:
+	BasicFile&	   m_Base;
+	uint8_t*	   m_Buffer;
+	const uint64_t m_BufferSize;
+	uint64_t	   m_BufferStart;
+	uint64_t	   m_BufferEnd;
+};
+
 ZENCORE_API void basicfile_forcelink();
 
 }  // namespace zen
diff --git a/src/zenutil/include/zenutil/zenutil.h b/src/zenutil/include/zenutil/zenutil.h
index 14d21ea0d..662743de8 100644
--- a/src/zenutil/include/zenutil/zenutil.h
+++ b/src/zenutil/include/zenutil/zenutil.h
@@ -1,3 +1,9 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
 #pragma once
+
+namespace zen {
+
+void zenutil_forcelinktests();
+
+}