sparse win file write (#398)

* Added `--use-sparse-files` option to `zen builds` command improving write performance of large files. Enabled by default.

4 files changed, 51 insertions, 1 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index c4a1344d4..117d0b291 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -95,6 +95,7 @@ namespace {
 
 	const bool SingleThreaded	  = false;
 	bool	   BoostWorkerThreads = false;
+	bool	   UseSparseFiles	  = false;
 
 	WorkerThreadPool& GetIOWorkerPool()
 	{
@@ -4261,8 +4262,19 @@ namespace {
 					ZEN_TRACE_CPU("WriteFileCache_WriteToFile_CacheWrite");
 					ZEN_ASSERT_SLOW(std::find(SeenTargetIndexes.begin(), SeenTargetIndexes.end(), TargetIndex) == SeenTargetIndexes.end());
 
-					OutputFile	   = std::move(NewOutputFile);
+					OutputFile = std::move(NewOutputFile);
+					if (UseSparseFiles)
+					{
+						void* Handle = OutputFile->Handle();
+						if (!PrepareFileForScatteredWrite(Handle, TargetFinalSize))
+						{
+							ZEN_DEBUG("Unable to to prepare file '{}' with size {} for random write",
+									  GetTargetPath(TargetIndex),
+									  TargetFinalSize);
+						}
+					}
 					OpenFileWriter = std::make_unique<BasicFileWriter>(*OutputFile, Min(TargetFinalSize, 256u * 1024u));
+
 					OpenFileWriter->Write(Buffer, FileOffset);
 					m_DiskStats.WriteCount++;
 					m_DiskStats.WriteByteCount += Buffer.GetSize();
@@ -8858,6 +8870,12 @@ BuildsCommand::BuildsCommand()
 
 	auto AddSystemOptions = [this](cxxopts::Options& Ops) {
 		Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>");
+		Ops.add_option("",
+					   "",
+					   "use-sparse-files",
+					   "Enable use of sparse files when writing large files. Defaults to true.",
+					   cxxopts::value(m_UseSparseFiles),
+					   "<usesparsefiles>");
 	};
 
 	auto AddAuthOptions = [this](cxxopts::Options& Ops) {
@@ -9872,6 +9890,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	};
 
 	BoostWorkerThreads = m_BoostWorkerThreads;
+	UseSparseFiles	   = m_UseSparseFiles;
 
 	try
 	{
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index de2d6fffc..41ed65105 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -31,6 +31,7 @@ private:
 	bool m_LogProgress		  = false;
 	bool m_Verbose			  = false;
 	bool m_BoostWorkerThreads = false;
+	bool m_UseSparseFiles	  = true;
 
 	std::filesystem::path m_ZenFolderPath;
 
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 018330d9b..c1df6d53e 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -2158,6 +2158,34 @@ MaximizeOpenFileCount()
 #endif
 }
 
+bool
+PrepareFileForScatteredWrite(void* FileHandle, uint64_t FinalSize)
+{
+	bool Result = true;
+#if ZEN_PLATFORM_WINDOWS
+	DWORD _	 = 0;
+	BOOL  Ok = DeviceIoControl(FileHandle, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &_, nullptr);
+	if (!Ok)
+	{
+		std::error_code DummyEc;
+		ZEN_DEBUG("Unable to set sparse mode for file '{}'", PathFromHandle(FileHandle, DummyEc));
+		Result = false;
+	}
+
+	FILE_ALLOCATION_INFO AllocationInfo	   = {};
+	AllocationInfo.AllocationSize.QuadPart = FinalSize;
+	if (!SetFileInformationByHandle(FileHandle, FileAllocationInfo, &AllocationInfo, DWORD(sizeof(AllocationInfo))))
+	{
+		std::error_code DummyEc;
+		ZEN_DEBUG("Unable to set file allocation size to {} for file '{}'", FinalSize, PathFromHandle(FileHandle, DummyEc));
+		Result = false;
+	}
+#else	// ZEN_PLATFORM_WINDOWS
+	ZEN_UNUSED(FileHandle, FinalSize);
+#endif	// ZEN_PLATFORM_WINDOWS
+	return Result;
+}
+
 void
 GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags Flags, DirectoryContent& OutContent)
 {
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index 1bc3943df..e62170eba 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -137,6 +137,8 @@ ZENCORE_API std::filesystem::path GetRunningExecutablePath();
  */
 ZENCORE_API void MaximizeOpenFileCount();
 
+ZENCORE_API bool PrepareFileForScatteredWrite(void* FileHandle, uint64_t FinalSize);
+
 struct FileContents
 {
 	std::vector<IoBuffer> Data;