file share support (#328)

this change adds a serve command to the zen CLI. This can be used to establish links to a set of files which may be served to clients via the project store interface:

```cmd> zen serve Lyra/WindowsClient d:\temp_share\StagedBuilds\WindowsClient```

with the appropriate changes in UE you may then start an instance of the runtime and have it load all files via the remote file connection:

```
Lyra\Binaries\LyraClient.exe ../../../Lyra/Lyra.uproject -pak -basedir=D:\temp_share\StagedBuilds\WindowsClient/Lyra/Binaries/Win64 -Mount=Lyra/WindowsClient
```

1 files changed, 241 insertions, 0 deletions

diff --git a/src/zen/cmds/serve.cpp b/src/zen/cmds/serve.cpp
new file mode 100644
index 000000000..3a50adb69
--- /dev/null
+++ b/src/zen/cmds/serve.cpp
@@ -0,0 +1,241 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "serve.h"
+
+#include <zencore/blake3.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinarypackage.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zenhttp/httpclient.h>
+#include <zenutil/zenserverprocess.h>
+
+#if ZEN_PLATFORM_WINDOWS
+#	include <conio.h>	// TEMPORARY HACK
+#endif
+
+namespace zen {
+
+using namespace std::literals;
+
+ServeCommand::ServeCommand()
+{
+	m_Options.add_options()("h,help", "Print help");
+	m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+	m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), "<projectid>");
+	m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), "<oplogid>");
+	m_Options.add_option("", "", "path", "Root path to directory", cxxopts::value(m_RootPath), "<rootpath>");
+
+	m_Options.parse_positional({"project", "path"});
+	m_Options.positional_help("[<projectid> <rootpath>]");
+}
+
+ServeCommand::~ServeCommand()
+{
+}
+
+int
+ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+	ZEN_UNUSED(GlobalOptions);
+
+	if (!ParseOptions(argc, argv))
+	{
+		return 0;
+	}
+
+	if (m_ProjectName.empty())
+	{
+		throw zen::OptionParseException("command requires a project");
+	}
+
+	if (m_OplogName.empty())
+	{
+		if (auto pos = m_ProjectName.find_first_of('/'); pos != std::string::npos)
+		{
+			m_OplogName	  = m_ProjectName.substr(pos + 1);
+			m_ProjectName = m_ProjectName.substr(0, pos);
+		}
+		else
+		{
+			throw zen::OptionParseException("command requires an oplog");
+		}
+	}
+
+	if (m_RootPath.empty())
+	{
+		throw zen::OptionParseException("command requires a root path");
+	}
+
+	if (!std::filesystem::exists(m_RootPath) || !std::filesystem::is_directory(m_RootPath))
+	{
+		throw zen::OptionParseException(fmt::format("path must exist and must be a directory: '{}'", m_RootPath));
+	}
+
+	uint16_t ServerPort = 0;
+	m_HostName			= ResolveTargetHostSpec(m_HostName, ServerPort);
+
+	ZenServerEnvironment			 ServerEnvironment;
+	std::optional<ZenServerInstance> ServerInstance;
+
+	if (m_HostName.empty())
+	{
+		// Spawn a server
+
+		try
+		{
+			std::filesystem::path ExePath = zen::GetRunningExecutablePath();
+
+			ServerEnvironment.Initialize(ExePath.parent_path());
+			ServerInstance.emplace(ServerEnvironment);
+			ServerInstance->SetOwnerPid(zen::GetCurrentProcessId());
+			ServerInstance->SpawnServerAndWait(ServerPort);
+		}
+		catch (std::exception& Ex)
+		{
+			ZEN_CONSOLE("failed to spawn server on port {}: '{}'", ServerPort, Ex.what());
+
+			throw zen::OptionParseException("unable to resolve server specification (even after spawning server)");
+		}
+	}
+	else
+	{
+		std::filesystem::path ExePath = zen::GetRunningExecutablePath();
+
+		ServerEnvironment.Initialize(ExePath.parent_path());
+		ServerInstance.emplace(ServerEnvironment);
+		ServerInstance->DisableShutdownOnDestroy();
+		ServerInstance->AttachToRunningServer();
+	}
+
+	if (ServerInstance)
+	{
+		ZEN_CONSOLE("base uri: {}", ServerInstance->GetBaseUri());
+	}
+
+	// Generate manifest for tree
+
+	FileSystemTraversal Traversal;
+
+	struct FsVisitor : public FileSystemTraversal::TreeVisitor
+	{
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+		{
+			std::filesystem::path ServerPath	   = std::filesystem::relative(Parent / File, RootPath);
+			std::string			  ServerPathString = reinterpret_cast<const char*>(ServerPath.generic_u8string().c_str());
+
+			if (ServerPathString.starts_with("./"))
+			{
+				ServerPathString = ServerPathString.substr(2);
+			}
+
+			Files.emplace_back(FileEntry{ServerPathString, ServerPathString, FileSize});
+		}
+
+		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
+
+		struct FileEntry
+		{
+			std::string FilePath;
+			std::string ClientFilePath;
+			uint64_t	FileSize;
+		};
+
+		std::filesystem::path  RootPath;
+		std::vector<FileEntry> Files;
+	};
+
+	FsVisitor Visitor;
+	Visitor.RootPath = m_RootPath;
+	Traversal.TraverseFileSystem(m_RootPath, Visitor);
+
+	CbObjectWriter Cbo;
+
+	Cbo << "key"
+		<< "file_manifest";
+
+	Cbo.BeginArray("files");
+
+	for (const FsVisitor::FileEntry& Entry : Visitor.Files)
+	{
+		ZEN_CONSOLE("file: {}", Entry.FilePath);
+
+		Cbo.BeginObject();
+
+		BLAKE3 Hash		= BLAKE3::HashMemory(Entry.ClientFilePath.data(), Entry.ClientFilePath.size());
+		Hash.Hash[11]	= 7;  // FIoChunkType::ExternalFile
+		Oid FileChunkId = Oid::FromMemory(Hash.Hash);
+
+		Cbo << "id"sv << FileChunkId;
+		Cbo << "serverpath"sv << Entry.FilePath;
+		Cbo << "clientpath"sv << Entry.ClientFilePath;
+
+		Cbo.EndObject();
+	}
+
+	Cbo.EndArray();
+
+	CbObject Manifest = Cbo.Save();
+
+	// Persist manifest
+
+	const std::string ProjectUri	  = fmt::format("/prj/{}", m_ProjectName);
+	const std::string ProjectOplogUri = fmt::format("/prj/{}/oplog/{}", m_ProjectName, m_OplogName);
+
+	HttpClient Client(m_HostName);
+
+	// Ensure project exists
+
+	if (HttpClient::Response ProjectResponse = Client.Get(ProjectUri); !ProjectResponse)
+	{
+		// Create project
+
+		CbObjectWriter Project;
+
+		Project << "root" << m_RootPath;
+
+		if (auto NewProjectResponse = Client.Post(ProjectUri, Project.Save()); !NewProjectResponse)
+		{
+			// TODO: include details
+			throw std::runtime_error("failed to create project");
+		}
+	}
+
+	// Ensure oplog exists
+
+	if (HttpClient::Response OplogResponse = Client.Get(ProjectOplogUri); !OplogResponse)
+	{
+		// Create oplog
+
+		CbObjectWriter Oplog;
+
+		if (auto NewOplogResponse = Client.Post(ProjectOplogUri, Oplog.Save()); !NewOplogResponse)
+		{
+			// TODO: include details
+			throw std::runtime_error("failed to create oplog");
+		}
+	}
+
+	// Append manifest
+
+	const std::string Uri = fmt::format("/prj/{}/oplog/{}/new", m_ProjectName, m_OplogName);
+
+	HttpClient::Response HttpResponse = Client.Post(Uri, Manifest);
+
+	if (!HttpResponse)
+	{
+		ZEN_CONSOLE("error: failed to append manifest!");
+
+		return 1;
+	}
+
+	ZEN_CONSOLE("ok serving files now");
+
+#if ZEN_PLATFORM_WINDOWS
+	_getch();  // TEMPORARY HACK
+#endif
+
+	return 0;
+}
+
+}  // namespace zen