workspace shares (#84)

Feature: New 'workspaces' service which allows a user to share a local folder via zenserver. A workspace can have mulitple workspace shares and they provie an HTTP API that is compatible with the project oplog HTTP API. Workspaces and shares are preserved between runs. Workspaces feature is disabled by default - enable with --workspaces-enabled option when launching zenserver.

1 files changed, 523 insertions, 0 deletions

diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp
new file mode 100644
index 000000000..503bc24cf
--- /dev/null
+++ b/src/zen/cmds/workspaces_cmd.cpp
@@ -0,0 +1,523 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "workspaces_cmd.h"
+
+#include <zencore/except.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/string.h>
+#include <zencore/uid.h>
+#include <zenhttp/formatters.h>
+#include <zenhttp/httpclient.h>
+#include <zenhttp/httpcommon.h>
+#include <zenutil/chunkrequests.h>
+#include <zenutil/zenserverprocess.h>
+
+#include <memory>
+
+namespace zen {
+
+WorkspaceCommand::WorkspaceCommand()
+{
+	m_Options.add_options()("h,help", "Print help");
+	m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+	m_Options.add_option("", "v", "verb", "Verb for workspace - create, remove, info", cxxopts::value(m_Verb), "<verb>");
+	m_Options.parse_positional({"verb"});
+	m_Options.positional_help("verb");
+
+	m_CreateOptions.add_options()("h,help", "Print help");
+	m_CreateOptions.add_option("", "w", "workspace", "Workspace identity(id)", cxxopts::value(m_Id), "<workspaceid>");
+	m_CreateOptions.add_option("", "r", "folder", "Root file system folder for workspace", cxxopts::value(m_Path), "<folder>");
+	m_CreateOptions.parse_positional({"folder", "workspace"});
+	m_CreateOptions.positional_help("folder workspace");
+
+	m_InfoOptions.add_options()("h,help", "Print help");
+	m_InfoOptions.add_option("", "w", "workspace", "Workspace identity(id)", cxxopts::value(m_Id), "<workspaceid>");
+	m_InfoOptions.parse_positional({"workspace"});
+	m_InfoOptions.positional_help("workspace");
+
+	m_RemoveOptions.add_options()("h,help", "Print help");
+	m_RemoveOptions.add_option("", "w", "workspace", "Workspace identity(id)", cxxopts::value(m_Id), "<workspaceid>");
+	m_RemoveOptions.parse_positional({"workspace"});
+	m_InfoOptions.positional_help("workspace");
+}
+
+WorkspaceCommand::~WorkspaceCommand() = default;
+
+int
+WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+	ZEN_UNUSED(GlobalOptions);
+
+	using namespace std::literals;
+
+	std::vector<char*> SubCommandArguments;
+	cxxopts::Options*  SubOption			 = nullptr;
+	int				   ParentCommandArgCount = GetSubCommand(m_Options, argc, argv, m_SubCommands, SubOption, SubCommandArguments);
+	if (!ParseOptions(ParentCommandArgCount, argv))
+	{
+		return 0;
+	}
+
+	if (SubOption == nullptr)
+	{
+		throw zen::OptionParseException("command verb is missing");
+	}
+
+	m_HostName = ResolveTargetHostSpec(m_HostName);
+
+	if (m_HostName.empty())
+	{
+		throw zen::OptionParseException("unable to resolve server specification");
+	}
+
+	if (!ParseOptions(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data()))
+	{
+		return 0;
+	}
+
+	HttpClient Http(m_HostName);
+
+	if (SubOption == &m_CreateOptions)
+	{
+		if (m_Path.empty())
+		{
+			throw zen::OptionParseException(fmt::format("path is required\n{}", m_CreateOptions.help()));
+		}
+		if (m_Id.empty())
+		{
+			m_Id = Oid::Zero.ToString();
+			ZEN_CONSOLE("Using generated workspace id from path '{}'", m_Path);
+		}
+
+		HttpClient::KeyValueMap Params{{"root_path", m_Path}};
+		if (HttpClient::Response Result = Http.Put(fmt::format("/ws/{}", m_Id), Params))
+		{
+			ZEN_CONSOLE("{}. Id: {}", Result, Result.AsText());
+			return 0;
+		}
+		else
+		{
+			Result.ThrowError(fmt::format("failed to create workspace {}", m_Id));
+			return 1;
+		}
+	}
+
+	if (SubOption == &m_InfoOptions)
+	{
+		if (m_Id.empty())
+		{
+			throw zen::OptionParseException(fmt::format("id is required", m_InfoOptions.help()));
+		}
+		if (HttpClient::Response Result = Http.Get(fmt::format("/ws/{}", m_Id)))
+		{
+			ZEN_CONSOLE("{}", Result.ToText());
+			return 0;
+		}
+		else
+		{
+			Result.ThrowError(fmt::format("failed to get info for workspace {}", m_Id));
+			return 1;
+		}
+	}
+
+	if (SubOption == &m_RemoveOptions)
+	{
+		if (m_Id.empty())
+		{
+			throw zen::OptionParseException(fmt::format("id is required", m_RemoveOptions.help()));
+		}
+		if (HttpClient::Response Result = Http.Delete(fmt::format("/ws/{}", m_Id)))
+		{
+			ZEN_CONSOLE("{}", Result);
+			return 0;
+		}
+		else
+		{
+			Result.ThrowError(fmt::format("failed to remove workspace {}", m_Id));
+			return 1;
+		}
+	}
+
+	ZEN_ASSERT(false);
+}
+
+/////////////////////////////////////////////////////////////////////////
+
+WorkspaceShareCommand::WorkspaceShareCommand()
+{
+	m_Options.add_options()("h,help", "Print help");
+	m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+	m_Options.add_option("", "v", "verb", "Verb for workspace - create, remove, info", cxxopts::value(m_Verb), "<verb>");
+	m_Options.parse_positional({"verb"});
+	m_Options.positional_help("verb");
+
+	m_CreateOptions.add_options()("h,help", "Print help");
+	m_CreateOptions.add_option("", "w", "workspace", "Workspace identity (id)", cxxopts::value(m_WorkspaceId), "<workspaceid>");
+	m_CreateOptions.add_option("", "s", "share", "Workspace share identity(id)", cxxopts::value(m_ShareId), "<shareid>");
+	m_CreateOptions.add_option("", "r", "folder", "Folder path inside the workspace to share", cxxopts::value(m_SharePath), "<folder>");
+	m_CreateOptions.parse_positional({"workspace", "folder", "share"});
+	m_CreateOptions.positional_help("workspace folder share");
+
+	m_InfoOptions.add_options()("h,help", "Print help");
+	m_InfoOptions.add_option("", "w", "workspace", "Workspace identity (id)", cxxopts::value(m_WorkspaceId), "<workspaceid>");
+	m_InfoOptions.add_option("", "s", "share", "Workspace share identity(id)", cxxopts::value(m_ShareId), "<shareid>");
+	m_InfoOptions.add_option("", "r", "refresh", "Refresh workspace share", cxxopts::value(m_Refresh), "<refresh>");
+	m_InfoOptions.parse_positional({"workspace", "share"});
+	m_InfoOptions.positional_help("workspace share");
+
+	m_RemoveOptions.add_options()("h,help", "Print help");
+	m_RemoveOptions.add_option("", "w", "workspace", "Workspace identity (id)", cxxopts::value(m_WorkspaceId), "<workspaceid>");
+	m_RemoveOptions.add_option("", "s", "share", "Workspace share identity(id)", cxxopts::value(m_ShareId), "<shareid>");
+	m_RemoveOptions.parse_positional({"workspace", "share"});
+	m_RemoveOptions.positional_help("workspace share");
+
+	m_FilesOptions.add_options()("h,help", "Print help");
+	m_FilesOptions.add_option("", "w", "workspace", "Workspace identity (id)", cxxopts::value(m_WorkspaceId), "<workspaceid>");
+	m_FilesOptions.add_option("", "s", "share", "Workspace share identity(id)", cxxopts::value(m_ShareId), "<shareid>");
+	m_FilesOptions.add_option("",
+							  "",
+							  "filter",
+							  "A list of comma separated fields to include in the response - empty means all",
+							  cxxopts::value(m_FieldFilter),
+							  "<fields>");
+	m_FilesOptions.add_option("", "r", "refresh", "Refresh workspace share", cxxopts::value(m_Refresh), "<refresh>");
+	m_FilesOptions.parse_positional({"workspace", "share"});
+	m_FilesOptions.positional_help("workspace share");
+
+	m_EntriesOptions.add_options()("h,help", "Print help");
+	m_EntriesOptions.add_option("", "w", "workspace", "Workspace identity (id)", cxxopts::value(m_WorkspaceId), "<workspaceid>");
+	m_EntriesOptions.add_option("", "s", "share", "Workspace share identity(id)", cxxopts::value(m_ShareId), "<shareid>");
+	m_EntriesOptions.add_option("",
+								"",
+								"filter",
+								"A list of comma separated fields to include in the response - empty means all",
+								cxxopts::value(m_FieldFilter),
+								"<fields>");
+	m_EntriesOptions.add_option("", "", "opkey", "Filter the query to a particular key (id)", cxxopts::value(m_ChunkId), "<oid>");
+	m_EntriesOptions.add_option("", "r", "refresh", "Refresh workspace share", cxxopts::value(m_Refresh), "<refresh>");
+	m_EntriesOptions.parse_positional({"workspace", "share", "opkey"});
+	m_EntriesOptions.positional_help("workspace share opkey");
+
+	m_GetChunkOptions.add_options()("h,help", "Print help");
+	m_GetChunkOptions.add_option("", "w", "workspace", "Workspace identity (id)", cxxopts::value(m_WorkspaceId), "<workspaceid>");
+	m_GetChunkOptions.add_option("", "s", "share", "Workspace share identity(id)", cxxopts::value(m_ShareId), "<shareid>");
+	m_GetChunkOptions.add_option("", "c", "chunk", "Chunk identity (id)", cxxopts::value(m_ChunkId), "<chunkid>");
+	m_GetChunkOptions.add_option("", "", "offset", "Offset in chunk", cxxopts::value(m_Offset), "<offset>");
+	m_GetChunkOptions.add_option("", "", "size", "Size of chunk", cxxopts::value(m_Size), "<size>");
+	m_GetChunkOptions.parse_positional({"workspace", "share", "chunk"});
+	m_GetChunkOptions.positional_help("workspace share chunk");
+
+	m_GetChunkBatchOptions.add_options()("h,help", "Print help");
+	m_GetChunkBatchOptions.add_option("", "s", "share", "Workspace share identity(id)", cxxopts::value(m_ShareId), "<shareid>");
+	m_GetChunkBatchOptions.add_option("", "w", "workspace", "Workspace identity (id)", cxxopts::value(m_WorkspaceId), "<workspaceid>");
+	m_GetChunkBatchOptions.add_option("", "", "chunks", "A list of identities (id)", cxxopts::value(m_ChunkIds), "<chunkids>");
+	m_GetChunkBatchOptions.parse_positional({"workspace", "share", "chunks"});
+	m_GetChunkBatchOptions.positional_help("workspace share chunks");
+}
+
+WorkspaceShareCommand::~WorkspaceShareCommand() = default;
+
+int
+WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+	ZEN_UNUSED(GlobalOptions);
+
+	using namespace std::literals;
+
+	std::vector<char*> SubCommandArguments;
+	cxxopts::Options*  SubOption			 = nullptr;
+	int				   ParentCommandArgCount = GetSubCommand(m_Options, argc, argv, m_SubCommands, SubOption, SubCommandArguments);
+	if (!ParseOptions(ParentCommandArgCount, argv))
+	{
+		return 0;
+	}
+
+	if (SubOption == nullptr)
+	{
+		throw zen::OptionParseException("command verb is missing");
+	}
+
+	m_HostName = ResolveTargetHostSpec(m_HostName);
+
+	if (m_HostName.empty())
+	{
+		throw zen::OptionParseException("unable to resolve server specification");
+	}
+
+	if (!ParseOptions(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data()))
+	{
+		return 0;
+	}
+
+	if (m_WorkspaceId.empty())
+	{
+		throw zen::OptionParseException("workspace id is required");
+	}
+
+	HttpClient Http(m_HostName);
+
+	if (SubOption == &m_CreateOptions)
+	{
+		if (m_ShareId.empty())
+		{
+			if (m_SharePath.ends_with(std::filesystem::path::preferred_separator))
+			{
+				m_SharePath.pop_back();
+			}
+
+			m_ShareId = Oid::Zero.ToString();
+			ZEN_CONSOLE("Using generated share id for path '{}'", m_SharePath);
+		}
+
+		HttpClient::KeyValueMap Params{{"share_path", m_SharePath}};
+
+		if (HttpClient::Response Result = Http.Put(fmt::format("/ws/{}/{}", m_WorkspaceId, m_ShareId), Params))
+		{
+			ZEN_CONSOLE("{}. Id: {}", Result, Result.AsText());
+			return 0;
+		}
+		else
+		{
+			Result.ThrowError("failed to create workspace share"sv);
+			return 1;
+		}
+	}
+
+	if (SubOption == &m_InfoOptions)
+	{
+		if (m_ShareId.empty())
+		{
+			throw zen::OptionParseException(fmt::format("share id is required", m_InfoOptions.help()));
+		}
+
+		if (HttpClient::Response Result = Http.Get(fmt::format("/ws/{}/{}", m_WorkspaceId, m_ShareId)))
+		{
+			ZEN_CONSOLE("{}", Result.ToText());
+			return 0;
+		}
+		else
+		{
+			Result.ThrowError(fmt::format("failed to get info for share {} in workspace {}", m_ShareId, m_WorkspaceId));
+			return 1;
+		}
+	}
+
+	if (SubOption == &m_RemoveOptions)
+	{
+		if (m_ShareId.empty())
+		{
+			throw zen::OptionParseException(fmt::format("share id is required", m_InfoOptions.help()));
+		}
+		if (HttpClient::Response Result = Http.Delete(fmt::format("/ws/{}/{}", m_WorkspaceId, m_ShareId)))
+		{
+			ZEN_CONSOLE("{}", Result);
+			return 0;
+		}
+		else
+		{
+			Result.ThrowError(fmt::format("failed to remove share {} in workspace {}", m_WorkspaceId, m_ShareId));
+			return 1;
+		}
+	}
+
+	if (SubOption == &m_FilesOptions)
+	{
+		if (m_ShareId.empty())
+		{
+			throw zen::OptionParseException(fmt::format("share id is required", m_InfoOptions.help()));
+		}
+
+		HttpClient::KeyValueMap Params;
+		if (!m_FieldFilter.empty())
+		{
+			Params.Entries.insert_or_assign("fieldnames", m_FieldFilter);
+		}
+		if (m_Refresh)
+		{
+			Params.Entries.insert_or_assign("refresh", ToString(m_Refresh));
+		}
+
+		if (HttpClient::Response Result = Http.Get(fmt::format("/ws/{}/{}/files", m_WorkspaceId, m_ShareId), {}, Params))
+		{
+			ZEN_CONSOLE("{}: {}", Result, Result.ToText());
+			return 0;
+		}
+		else
+		{
+			Result.ThrowError("failed to get workspace share files"sv);
+			return 1;
+		}
+	}
+
+	if (SubOption == &m_EntriesOptions)
+	{
+		if (m_ShareId.empty())
+		{
+			throw zen::OptionParseException(fmt::format("share id is required", m_InfoOptions.help()));
+		}
+
+		HttpClient::KeyValueMap Params;
+		if (!m_ChunkId.empty())
+		{
+			Params.Entries.insert_or_assign("opkey", m_ChunkId);
+		}
+		if (!m_FieldFilter.empty())
+		{
+			Params.Entries.insert_or_assign("fieldfilter", m_FieldFilter);
+		}
+		if (m_Refresh)
+		{
+			Params.Entries.insert_or_assign("refresh", ToString(m_Refresh));
+		}
+
+		if (HttpClient::Response Result = Http.Get(fmt::format("/ws/{}/{}/entries", m_WorkspaceId, m_ShareId), {}, Params))
+		{
+			ZEN_CONSOLE("{}: {}", Result, Result.ToText());
+			return 0;
+		}
+		else
+		{
+			Result.ThrowError("failed to get workspace share entries"sv);
+			return 1;
+		}
+	}
+
+	auto ChunksToOidStrings =
+		[&Http, WorkspaceId = m_WorkspaceId, ShareId = m_ShareId](std::span<const std::string> ChunkIds) -> std::vector<std::string> {
+		std::vector<std::string> Oids;
+		Oids.reserve(ChunkIds.size());
+		std::vector<size_t> NeedsConvertIndexes;
+		for (const std::string& StringChunkId : ChunkIds)
+		{
+			Oid ChunkId = Oid::TryFromHexString(StringChunkId);
+			if (ChunkId == Oid::Zero)
+			{
+				NeedsConvertIndexes.push_back(Oids.size());
+			}
+			Oids.push_back(ChunkId.ToString());
+		}
+		if (!NeedsConvertIndexes.empty())
+		{
+			if (HttpClient::Response Result = Http.Get(fmt::format("/ws/{}/{}/files", WorkspaceId, ShareId),
+													   {},
+													   HttpClient::KeyValueMap{{"fieldnames", "id,clientpath"}}))
+			{
+				std::unordered_map<std::string, Oid> PathToOid;
+				for (CbFieldView EntryView : Result.AsObject()["files"sv])
+				{
+					CbObjectView Entry										 = EntryView.AsObjectView();
+					PathToOid[std::string(Entry["clientpath"sv].AsString())] = Entry["id"sv].AsObjectId();
+				}
+				for (size_t PathIndex : NeedsConvertIndexes)
+				{
+					if (auto It = PathToOid.find(ChunkIds[PathIndex]); It != PathToOid.end())
+					{
+						Oids[PathIndex] = It->second.ToString();
+						ZEN_CONSOLE("Converted path '{}' to id '{}'", ChunkIds[PathIndex], Oids[PathIndex]);
+					}
+					else
+					{
+						Result.ThrowError(
+							fmt::format("unable to resolve path {} workspace  {}, share {}"sv, ChunkIds[PathIndex], WorkspaceId, ShareId));
+					}
+				}
+			}
+			else
+			{
+				Result.ThrowError("failed to get workspace share file list to resolve paths"sv);
+			}
+		}
+		return Oids;
+	};
+
+	if (SubOption == &m_GetChunkOptions)
+	{
+		if (m_ShareId.empty())
+		{
+			throw zen::OptionParseException(fmt::format("share id is required", m_InfoOptions.help()));
+		}
+
+		if (m_ChunkId.empty())
+		{
+			throw zen::OptionParseException("chunk id is required");
+		}
+
+		m_ChunkId = ChunksToOidStrings(std::vector<std::string>{m_ChunkId})[0];
+
+		HttpClient::KeyValueMap Params;
+		if (m_Offset != 0)
+		{
+			Params.Entries.insert_or_assign("offset", fmt::format("{}", m_Offset));
+		}
+		if (m_Size != ~uint64_t(0))
+		{
+			Params.Entries.insert_or_assign("size", fmt::format("{}", m_Size));
+		}
+
+		if (HttpClient::Response Result = Http.Get(fmt::format("/ws/{}/{}/{}", m_WorkspaceId, m_ShareId, m_ChunkId), {}, Params))
+		{
+			ZEN_CONSOLE("{}: Bytes: {}", Result, NiceBytes(Result.ResponsePayload.GetSize()));
+			return 0;
+		}
+		else
+		{
+			Result.ThrowError("failed to get workspace share chunk"sv);
+			return 1;
+		}
+	}
+
+	if (SubOption == &m_GetChunkBatchOptions)
+	{
+		if (m_ShareId.empty())
+		{
+			throw zen::OptionParseException(fmt::format("share id is required", m_InfoOptions.help()));
+		}
+
+		if (m_ChunkIds.empty())
+		{
+			throw zen::OptionParseException("share is is required");
+		}
+
+		m_ChunkIds = ChunksToOidStrings(m_ChunkIds);
+
+		std::vector<RequestChunkEntry> ChunkRequests;
+		ChunkRequests.resize(m_ChunkIds.size());
+		for (size_t Index = 0; Index < m_ChunkIds.size(); Index++)
+		{
+			ChunkRequests[Index] = RequestChunkEntry{.ChunkId		= Oid::FromHexString(m_ChunkIds[Index]),
+													 .CorrelationId = gsl::narrow<uint32_t>(Index),
+													 .Offset		= 0,
+													 .RequestBytes	= uint64_t(-1)};
+		}
+		IoBuffer Payload = BuildChunkBatchRequest(ChunkRequests);
+
+		if (HttpClient::Response Result = Http.Post(fmt::format("/ws/{}/{}/batch", m_WorkspaceId, m_ShareId), Payload))
+		{
+			ZEN_CONSOLE("{}: Bytes: {}", Result, NiceBytes(Result.ResponsePayload.GetSize()));
+			std::vector<IoBuffer> Results = ParseChunkBatchResponse(Result.ResponsePayload);
+			if (Results.size() != m_ChunkIds.size())
+			{
+				throw std::runtime_error(
+					fmt::format("failed to get workspace share batch - invalid result count recevied (expected: {}, received: {}",
+								m_ChunkIds.size(),
+								Results.size()));
+			}
+			for (size_t Index = 0; Index < m_ChunkIds.size(); Index++)
+			{
+				ZEN_CONSOLE("{}: Bytes: {}", m_ChunkIds[Index], NiceBytes(Results[Index].GetSize()));
+			}
+			return 0;
+		}
+		else
+		{
+			Result.ThrowError("failed to get workspace share batch"sv);
+			return 1;
+		}
+	}
+
+	ZEN_ASSERT(false);
+}
+
+}  // namespace zen