add zenserver state snapshot support (#543)

this introduces a --snapshot-dir command line option to zenserver which specifies a directory which will be propagated to the persistence root directory on start-up.

This is most powerful with file systems which support block cloning, such as ReFS on Windows. This allows even very large state snapshots to be used repeatedly without having to worry about mutating the original dataset on disk. When using ReFS the state copy for even large state directories can be very fast since the duration is primarily proportional to the number of files in the tree rather than the size of the files being cloned. The storage requirements are also minimal as all data will be handled in a copy-on-write manner.

6 files changed, 146 insertions, 2 deletions

diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp
index e5ddbfa85..d68a99616 100644
--- a/src/zen/cmds/copy_cmd.cpp
+++ b/src/zen/cmds/copy_cmd.cpp
@@ -45,6 +45,22 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	FromPath = m_CopySource;
 	ToPath	 = m_CopyTarget;
 
+	std::error_code		  Ec;
+	std::filesystem::path FromCanonical = std::filesystem::canonical(FromPath, Ec);
+
+	if (!Ec)
+	{
+		std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec);
+
+		if (!Ec)
+		{
+			if (FromCanonical == ToCanonical)
+			{
+				throw std::runtime_error("Target and source must be distinct files or directories");
+			}
+		}
+	}
+
 	const bool IsFileCopy = std::filesystem::is_regular_file(m_CopySource);
 	const bool IsDirCopy  = std::filesystem::is_directory(m_CopySource);
 
@@ -76,6 +92,17 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			std::filesystem::create_directories(ToPath);
 		}
 
+		std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec);
+
+		if (!Ec)
+		{
+			if (!std::filesystem::relative(ToCanonical, FromCanonical).empty() ||
+				!std::filesystem::relative(FromCanonical, ToCanonical).empty())
+			{
+				throw std::runtime_error("Invalid parent/child relationship for source/target directories");
+			}
+		}
+
 		// Multi file copy
 
 		ZEN_CONSOLE("copying {} -> {}", FromPath, ToPath);
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 06cda7382..4a26c64e7 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -613,6 +613,103 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 }
 
 void
+CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options)
+{
+	// Validate arguments
+
+	if (FromPath.empty() || !std::filesystem::is_directory(FromPath))
+		throw std::runtime_error("invalid CopyTree source directory specified");
+
+	if (ToPath.empty())
+		throw std::runtime_error("no CopyTree target specified");
+
+	if (std::filesystem::exists(ToPath))
+	{
+		if (!std::filesystem::is_directory(ToPath))
+		{
+			throw std::runtime_error(fmt::format("specified CopyTree target '{}' is not a directory", ToPath));
+		}
+	}
+	else
+	{
+		std::filesystem::create_directories(ToPath);
+	}
+
+	struct CopyVisitor : public FileSystemTraversal::TreeVisitor
+	{
+		CopyVisitor(std::filesystem::path InBasePath, zen::CopyFileOptions InCopyOptions) : BasePath(InBasePath), CopyOptions(InCopyOptions)
+		{
+		}
+
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+		{
+			std::error_code				Ec;
+			const std::filesystem::path Relative = std::filesystem::relative(Parent, BasePath, Ec);
+
+			if (Ec)
+			{
+				FailedFileCount++;
+			}
+			else
+			{
+				const std::filesystem::path FromPath = Parent / File;
+				std::filesystem::path		ToPath;
+
+				if (Relative.compare("."))
+				{
+					zen::CreateDirectories(TargetPath / Relative);
+
+					ToPath = TargetPath / Relative / File;
+				}
+				else
+				{
+					ToPath = TargetPath / File;
+				}
+
+				try
+				{
+					if (zen::CopyFile(FromPath, ToPath, CopyOptions))
+					{
+						++FileCount;
+						ByteCount += FileSize;
+					}
+					else
+					{
+						throw std::runtime_error("CopyFile failed in an unexpected way");
+					}
+				}
+				catch (std::exception& Ex)
+				{
+					++FailedFileCount;
+
+					throw std::runtime_error(fmt::format("failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what()));
+				}
+			}
+		}
+
+		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
+
+		std::filesystem::path BasePath;
+		std::filesystem::path TargetPath;
+		zen::CopyFileOptions  CopyOptions;
+		int					  FileCount		  = 0;
+		uint64_t			  ByteCount		  = 0;
+		int					  FailedFileCount = 0;
+	};
+
+	CopyVisitor Visitor{FromPath, Options};
+	Visitor.TargetPath = ToPath;
+
+	FileSystemTraversal Traversal;
+	Traversal.TraverseFileSystem(FromPath, Visitor);
+
+	if (Visitor.FailedFileCount)
+	{
+		throw std::runtime_error(fmt::format("{} file copy operations FAILED", Visitor.FailedFileCount));
+	}
+}
+
+void
 WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount)
 {
 #if ZEN_PLATFORM_WINDOWS
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index 22eb40e45..963890fb9 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -1,5 +1,4 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
-// Copyright Epic Games, Inc. All Rights Reserved.
 
 #pragma once
 
@@ -87,6 +86,7 @@ struct CopyFileOptions
 };
 
 ZENCORE_API bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options);
+ZENCORE_API void CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options);
 ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path);
 
 ZENCORE_API void PathToUtf8(const std::filesystem::path& Path, StringBuilderBase& Out);
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index e7edd5745..e33c6ff70 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -938,6 +938,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 	std::string AbsLogFile;
 	std::string ConfigFile;
 	std::string OutputConfigFile;
+	std::string BaseSnapshotDir;
 
 	cxxopts::Options options("zenserver", "Zen Server");
 	options.add_options()("dedicated",
@@ -951,6 +952,9 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 	options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(ServerOptions.IsTest)->default_value("false"));
 	options.add_options()("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId));
 	options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::string>(DataDir));
+	options.add_options()("snapshot-dir",
+						  "Specify a snapshot of server state to mirror into the persistence root at startup",
+						  cxxopts::value<std::string>(BaseSnapshotDir));
 	options.add_options()("content-dir", "Frontend content directory", cxxopts::value<std::string>(ContentDir));
 	options.add_options()("abslog", "Path to log file", cxxopts::value<std::string>(AbsLogFile));
 	options.add_options()("config", "Path to Lua config file", cxxopts::value<std::string>(ConfigFile));
@@ -1362,11 +1366,21 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 		}
 
 		ServerOptions.DataDir						  = MakeSafePath(DataDir);
+		ServerOptions.BaseSnapshotDir				  = MakeSafePath(BaseSnapshotDir);
 		ServerOptions.ContentDir					  = MakeSafePath(ContentDir);
 		ServerOptions.AbsLogFile					  = MakeSafePath(AbsLogFile);
 		ServerOptions.ConfigFile					  = MakeSafePath(ConfigFile);
 		ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions);
 
+		if (!BaseSnapshotDir.empty())
+		{
+			if (DataDir.empty())
+				throw zen::OptionParseException("You must explicitly specify a data directory when specifying a base snapshot");
+
+			if (!std::filesystem::is_directory(ServerOptions.BaseSnapshotDir))
+				throw OptionParseException(fmt::format("Snapshot directory must be a directory: '{}", BaseSnapshotDir));
+		}
+
 		if (OpenIdProviderUrl.empty() == false)
 		{
 			if (OpenIdClientId.empty())
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index d55f0d5a1..406fb0b70 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -129,6 +129,7 @@ struct ZenServerOptions
 	std::filesystem::path	 ContentDir;				  // Root directory for serving frontend content (experimental)
 	std::filesystem::path	 AbsLogFile;				  // Absolute path to main log file
 	std::filesystem::path	 ConfigFile;				  // Path to Lua config file
+	std::filesystem::path	 BaseSnapshotDir;			  // Path to server state snapshot (will be copied into data dir on start)
 	std::string				 ChildId;					  // Id assigned by parent process (used for lifetime management)
 	std::string				 LogId;						  // Id for tagging log output
 	std::string				 EncryptionKey;				  // 256 bit AES encryption key
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index 44330d4c8..1aa0aa4df 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -336,7 +336,7 @@ main(int argc, char* argv[])
 		ZenServerOptions ServerOptions;
 		ParseCliOptions(argc, argv, ServerOptions);
 
-		if (ServerOptions.IsCleanStart)
+		if (ServerOptions.IsCleanStart || !ServerOptions.BaseSnapshotDir.empty())
 		{
 			DeleteDirectories(ServerOptions.DataDir);
 		}
@@ -347,6 +347,11 @@ main(int argc, char* argv[])
 			std::filesystem::create_directories(ServerOptions.DataDir);
 		}
 
+		if (!ServerOptions.BaseSnapshotDir.empty())
+		{
+			CopyTree(ServerOptions.BaseSnapshotDir, ServerOptions.DataDir, {.EnableClone = true});
+		}
+
 #if ZEN_WITH_TRACE
 		if (ServerOptions.TraceHost.size())
 		{