path: root/src/zen/zen.cpp

// Copyright Epic Games, Inc. All Rights Reserved.

// Zen command line client utility
//

#include "zen.h"

#include "cmds/admin_cmd.h"
#include "cmds/bench_cmd.h"
#include "cmds/cache_cmd.h"
#include "cmds/copy_cmd.h"
#include "cmds/dedup_cmd.h"
#include "cmds/info_cmd.h"
#include "cmds/print_cmd.h"
#include "cmds/projectstore_cmd.h"
#include "cmds/rpcreplay_cmd.h"
#include "cmds/run_cmd.h"
#include "cmds/serve_cmd.h"
#include "cmds/status_cmd.h"
#include "cmds/top_cmd.h"
#include "cmds/trace_cmd.h"
#include "cmds/up_cmd.h"
#include "cmds/version_cmd.h"
#include "cmds/vfs_cmd.h"
#include "cmds/workspaces_cmd.h"

#include <zencore/filesystem.h>
#include <zencore/logging.h>
#include <zencore/scopeguard.h>
#include <zencore/string.h>
#include <zenhttp/httpcommon.h>
#include <zenutil/zenserverprocess.h>

#if ZEN_WITH_TESTS
#	define ZEN_TEST_WITH_RUNNER 1
#	include <zencore/testing.h>
#endif

ZEN_THIRD_PARTY_INCLUDES_START
#include <cpr/cpr.h>
#include <spdlog/sinks/ansicolor_sink.h>
#include <spdlog/spdlog.h>
#include <gsl/gsl-lite.hpp>
ZEN_THIRD_PARTY_INCLUDES_END

#if ZEN_USE_MIMALLOC
#	include <mimalloc-new-delete.h>
#endif

//////////////////////////////////////////////////////////////////////////

namespace zen {

ZenCmdCategory DefaultCategory{.Name = "general commands"};
ZenCmdCategory g_UtilitiesCategory{.Name = "utility commands"};
ZenCmdCategory g_ProjectStoreCategory{.Name = "project store commands"};
ZenCmdCategory g_CacheStoreCategory{.Name = "cache store commands"};
ZenCmdCategory g_StorageCategory{.Name = "storage management commands"};

ZenCmdCategory&
ZenCmdBase::CommandCategory() const
{
	return DefaultCategory;
}

bool
ZenCmdBase::ParseOptions(int argc, char** argv)
{
	return ParseOptions(Options(), argc, argv);
}

bool
ZenCmdBase::ParseOptions(cxxopts::Options& CmdOptions, int argc, char** argv)
{
	cxxopts::ParseResult Result;

	try
	{
		Result = CmdOptions.parse(argc, argv);
	}
	catch (const std::exception& Ex)
	{
		throw zen::OptionParseException(Ex.what());
	}

	CmdOptions.show_positional_help();

	if (Result.count("help"))
	{
		printf("%s\n", CmdOptions.help().c_str());
		return false;
	}

	if (!Result.unmatched().empty())
	{
		zen::ExtendableStringBuilder<64> StringBuilder;
		for (bool First = true; const auto& Param : Result.unmatched())
		{
			if (!First)
			{
				StringBuilder.Append(", ");
			}
			StringBuilder.Append('"');
			StringBuilder.Append(Param);
			StringBuilder.Append('"');
			First = false;
		}

		throw zen::OptionParseException(fmt::format("Invalid arguments: {}", StringBuilder.ToView()));
	}

	return true;
}

// Get the number of args including the sub command
// Build an array for sub command to parse
int
ZenCmdBase::GetSubCommand(cxxopts::Options&,
						  int						   argc,
						  char**					   argv,
						  std::span<cxxopts::Options*> SubOptions,
						  cxxopts::Options*&		   OutSubOption,
						  std::vector<char*>&		   OutSubCommandArguments)
{
	for (int i = 1; i < argc; ++i)
	{
		if (auto It = std::find_if(SubOptions.begin(),
								   SubOptions.end(),
								   [&](cxxopts::Options* SubOption) { return SubOption->program() == argv[i]; });
			It != SubOptions.end())
		{
			OutSubOption = (*It);
			OutSubCommandArguments.push_back(argv[0]);
			std::copy(&argv[i + 1], &argv[argc], std::back_inserter(OutSubCommandArguments));
			return i + 1;
		}
	}
	// No Sub command found
	OutSubOption = nullptr;
	return argc;
}

std::string
ZenCmdBase::FormatHttpResponse(const cpr::Response& Response)
{
	if (Response.error.code != cpr::ErrorCode::OK)
	{
		if (Response.error.message.empty())
		{
			return fmt::format("Request '{}' failed, error code {}", Response.url.str(), static_cast<int>(Response.error.code));
		}
		return fmt::format("Request '{}' failed. Reason: '{}' ({})",
						   Response.url.str(),
						   Response.error.message,
						   static_cast<int>(Response.error.code));
	}

	std::string Content;
	if (auto It = Response.header.find("Content-Type"); It != Response.header.end())
	{
		zen::HttpContentType ContentType = zen::ParseContentType(It->second);
		if (ContentType == zen::HttpContentType::kText)
		{
			Content = Response.text;
		}
		else if (ContentType == zen::HttpContentType::kJSON)
		{
			Content = fmt::format("\n{}", Response.text);
		}
		else if (!Response.text.empty())
		{
			Content = fmt::format("[{}]", MapContentTypeToString(ContentType));
		}
	}

	std::string_view ResponseString = zen::ReasonStringForHttpResultCode(
		Response.status_code == static_cast<long>(zen::HttpResponseCode::NoContent) ? static_cast<long>(zen::HttpResponseCode::OK)
																					: Response.status_code);
	if (Content.empty())
	{
		return std::string(ResponseString);
	}

	return fmt::format("{}: {}", ResponseString, Content);
}

int
ZenCmdBase::MapHttpToCommandReturnCode(const cpr::Response& Response)
{
	if (zen::IsHttpSuccessCode(Response.status_code))
	{
		return 0;
	}
	if (Response.error.code != cpr::ErrorCode::OK)
	{
		return static_cast<int>(Response.error.code);
	}
	return 1;
}

std::string
ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec, uint16_t& OutEffectivePort)
{
	if (InHostSpec.empty())
	{
		// If no host is specified then look to see if we have an instance
		// running on this host and use that as the default to interact with

		zen::ZenServerState Servers;

		if (Servers.InitializeReadOnly())
		{
			std::string ResolvedSpec;

			Servers.Snapshot([&](const zen::ZenServerState::ZenServerEntry& Entry) {
				if (ResolvedSpec.empty())
				{
					ResolvedSpec	 = fmt::format("http://localhost:{}", Entry.EffectiveListenPort.load());
					OutEffectivePort = Entry.EffectiveListenPort;
				}
			});

			return ResolvedSpec;
		}
	}

	// Parse out port from the specification provided, to be consistent with
	// the auto-discovery logic above.

	std::string_view PortSpec(InHostSpec);
	if (size_t PrefixIndex = PortSpec.find_last_of(":"); PrefixIndex != std::string_view::npos)
	{
		PortSpec.remove_prefix(PrefixIndex + 1);

		std::optional<uint16_t> EffectivePort = zen::ParseInt<uint16_t>(PortSpec);

		if (EffectivePort)
		{
			OutEffectivePort = EffectivePort.value();
		}
	}

	// note: We should consider adding validation/normalization of the provided spec here

	return InHostSpec;
}

std::string
ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec)
{
	uint16_t Dummy = 0;
	return ResolveTargetHostSpec(InHostSpec, /* out */ Dummy);
}

}  // namespace zen

//////////////////////////////////////////////////////////////////////////
// TODO: should make this Unicode-aware so we can pass anything in on the
// command line.

int
main(int argc, char** argv)
{
	using namespace zen;
	using namespace std::literals;

#if ZEN_USE_MIMALLOC
	mi_version();
#endif

	zen::logging::InitializeLogging();

	// Set output mode to handle virtual terminal sequences
	zen::logging::EnableVTMode();
	std::set_terminate([]() { ZEN_CRITICAL("Program exited abnormally via std::terminate()"); });

	LoggerRef DefaultLogger = zen::logging::Default();
	auto&	  Sinks			= DefaultLogger.SpdLogger->sinks();

	Sinks.clear();
	auto ConsoleSink = std::make_shared<spdlog::sinks::ansicolor_stdout_sink_mt>();
	Sinks.push_back(ConsoleSink);

	zen::MaximizeOpenFileCount();

	//////////////////////////////////////////////////////////////////////////

	auto _ = zen::MakeGuard([] { spdlog::shutdown(); });

	AttachCommand			 AttachCmd;
	BenchCommand			 BenchCmd;
	CacheDetailsCommand		 CacheDetailsCmd;
	CacheGenerateCommand	 CacheGenerateCmd;
	CacheInfoCommand		 CacheInfoCmd;
	CacheStatsCommand		 CacheStatsCmd;
	CopyCommand				 CopyCmd;
	CopyStateCommand		 CopyStateCmd;
	CreateOplogCommand		 CreateOplogCmd;
	CreateProjectCommand	 CreateProjectCmd;
	DedupCommand			 DedupCmd;
	DownCommand				 DownCmd;
	DropCommand				 DropCmd;
	DropProjectCommand		 ProjectDropCmd;
	ExportOplogCommand		 ExportOplogCmd;
	FlushCommand			 FlushCmd;
	GcCommand				 GcCmd;
	GcStatusCommand			 GcStatusCmd;
	GcStopCommand			 GcStopCmd;
	ImportOplogCommand		 ImportOplogCmd;
	InfoCommand				 InfoCmd;
	JobCommand				 JobCmd;
	OplogMirrorCommand		 OplogMirrorCmd;
	PrintCommand			 PrintCmd;
	PrintPackageCommand		 PrintPkgCmd;
	ProjectDetailsCommand	 ProjectDetailsCmd;
	ProjectInfoCommand		 ProjectInfoCmd;
	ProjectStatsCommand		 ProjectStatsCmd;
	PsCommand				 PsCmd;
	RpcReplayCommand		 RpcReplayCmd;
	RpcStartRecordingCommand RpcStartRecordingCmd;
	RpcStopRecordingCommand	 RpcStopRecordingCmd;
	RunCommand				 RunCmd;
	ScrubCommand			 ScrubCmd;
	ServeCommand			 ServeCmd;
	SnapshotOplogCommand	 SnapshotOplogCmd;
	StatusCommand			 StatusCmd;
	LoggingCommand			 LoggingCmd;
	TopCommand				 TopCmd;
	TraceCommand			 TraceCmd;
	UpCommand				 UpCmd;
	VersionCommand			 VersionCmd;
	VfsCommand				 VfsCmd;
	WorkspaceCommand		 WorkspaceCmd;
	WorkspaceShareCommand	 WorkspaceShareCmd;

	const struct CommandInfo
	{
		const char* CmdName;
		ZenCmdBase* Cmd;
		const char* CmdSummary;
	} Commands[] = {
		// clang-format off
		{AttachCommand::Name,					&AttachCmd,					AttachCommand::Description},
		{"bench",								&BenchCmd,					"Utility command for benchmarking"},
		{"cache-details",						&CacheDetailsCmd,			"Details on cache"},
		{"cache-info",							&CacheInfoCmd,				"Info on cache, namespace or bucket"},
		{CacheGenerateCommand::Name,			&CacheGenerateCmd,			CacheGenerateCommand::Description},
		{"cache-stats",							&CacheStatsCmd,				"Stats on cache"},
		{"copy",								&CopyCmd,					"Copy file(s)"},
		{"copy-state",							&CopyStateCmd,				"Copy zen server disk state"},
		{"dedup",								&DedupCmd,					"Dedup files"},
		{DownCommand::Name,						&DownCmd,					DownCommand::Description},
		{"drop",								&DropCmd,					"Drop cache namespace or bucket"},
		{"gc-status",							&GcStatusCmd,				"Garbage collect zen storage status check"},
		{"gc-stop",								&GcStopCmd,					"Request cancel of running garbage collection in zen storage"},
		{"gc",									&GcCmd,						"Garbage collect zen storage"},
		{"info",								&InfoCmd,					"Show high level Zen server information"},
		{"jobs",								&JobCmd,					"Show/cancel zen background jobs"},
		{"logs",								&LoggingCmd,				"Show/control zen logging"},
		{"oplog-create",						&CreateOplogCmd,			"Create a project oplog"},
		{"oplog-export",						&ExportOplogCmd,			"Export project store oplog"},
		{"oplog-import",						&ImportOplogCmd,			"Import project store oplog"},
		{"oplog-mirror",						&OplogMirrorCmd,			"Mirror project store oplog to file system"},
		{"oplog-snapshot",						&SnapshotOplogCmd,			"Snapshot project store oplog"},
		{"print",								&PrintCmd,					"Print compact binary object"},
		{"printpackage",						&PrintPkgCmd,				"Print compact binary package"},
		{"project-create",						&CreateProjectCmd,			"Create a project"},
		{"project-details",						&ProjectDetailsCmd,			"Details on project store"},
		{"project-drop",						&ProjectDropCmd,			"Drop project or project oplog"},
		{"project-info",						&ProjectInfoCmd,			"Info on project or project oplog"},
		{"project-stats",						&ProjectStatsCmd,			"Stats on project store"},
		{"ps",									&PsCmd,						"Enumerate running zen server instances"},
		{"rpc-record-replay",					&RpcReplayCmd,				"Replays a previously recorded session of rpc requests"},
		{"rpc-record-start",					&RpcStartRecordingCmd,		"Starts recording of cache rpc requests on a host"},
		{"rpc-record-stop",						&RpcStopRecordingCmd,		"Stops recording of cache rpc requests on a host"},
		{"run",									&RunCmd,					"Run command with special options"},
		{"scrub",								&ScrubCmd,					"Scrub zen storage (verify data integrity)"},
		{"serve",								&ServeCmd,					"Serve files from a directory"},
		{StatusCommand::Name,					&StatusCmd,					StatusCommand::Description},
		{"top",									&TopCmd,					"Monitor zen server activity"},
		{"trace",								&TraceCmd,					"Control zen realtime tracing"},
		{UpCommand::Name,						&UpCmd,						UpCommand::Description},
		{"version",								&VersionCmd,				"Get zen server version"},
		{"vfs",									&VfsCmd,					"Manage virtual file system"},
		{"flush",								&FlushCmd,					"Flush storage"},
		{WorkspaceCommand::Name,				&WorkspaceCmd,				WorkspaceCommand::Description},
		{WorkspaceShareCommand::Name,			&WorkspaceShareCmd,			WorkspaceShareCommand::Description},
		// clang-format on
	};

	// Build set containing available commands

	std::unordered_set<std::string> CommandSet;

	for (const auto& Cmd : Commands)
		CommandSet.insert(Cmd.CmdName);

	// Split command line into options, commands and any pass-through arguments

	std::string				 Passthrough;
	std::string				 PassthroughArgs;
	std::vector<std::string> PassthroughArgV;

	for (int i = 1; i < argc; ++i)
	{
		if ("--"sv == argv[i])
		{
			bool							  IsFirst = true;
			zen::ExtendableStringBuilder<256> Line;
			zen::ExtendableStringBuilder<256> Arguments;

			for (int j = i + 1; j < argc; ++j)
			{
				auto AppendAscii = [&](auto X) {
					Line.Append(X);
					if (!IsFirst)
					{
						Arguments.Append(X);
					}
				};

				if (!IsFirst)
				{
					AppendAscii(" ");
				}

				std::string_view ThisArg(argv[j]);
				PassthroughArgV.push_back(std::string(ThisArg));

				const bool NeedsQuotes = (ThisArg.find(' ') != std::string_view::npos);

				if (NeedsQuotes)
				{
					AppendAscii("\"");
				}

				AppendAscii(ThisArg);

				if (NeedsQuotes)
				{
					AppendAscii("\"");
				}

				IsFirst = false;
			}

			Passthrough		= Line.c_str();
			PassthroughArgs = Arguments.c_str();

			// This will "truncate" the arg vector and terminate the loop
			argc = i;
		}
	}

	// Split command line into global vs command options. We do this by simply
	// scanning argv for a string we recognise as a command and split it there

	std::vector<char*> CommandArgVec;
	CommandArgVec.push_back(argv[0]);

	for (int i = 1; i < argc; ++i)
	{
		if (CommandSet.find(argv[i]) != CommandSet.end())
		{
			int commandArgCount = /* exec name */ 1 + argc - (i + 1);
			CommandArgVec.resize(commandArgCount);
			std::copy(argv + i + 1, argv + argc, CommandArgVec.begin() + 1);

			argc = i + 1;

			break;
		}
	}

	// Parse global CLI arguments

	ZenCliOptions GlobalOptions;

	GlobalOptions.PassthroughCommandLine = Passthrough;
	GlobalOptions.PassthroughArgs		 = PassthroughArgs;
	GlobalOptions.PassthroughArgV		 = PassthroughArgV;

	std::string SubCommand = "<None>";

	cxxopts::Options Options("zen", "Zen management tool");

	Options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(GlobalOptions.IsDebug));
	Options.add_options()("v, verbose", "Enable verbose logging", cxxopts::value<bool>(GlobalOptions.IsVerbose));
	Options.add_options()("help", "Show command line help");
	Options.add_options()("c, command", "Sub command", cxxopts::value<std::string>(SubCommand));

	Options.parse_positional({"command"});

	const bool IsNullInvoke = (argc == 1);	// If no arguments are passed we want to print usage information

	try
	{
		cxxopts::ParseResult ParseResult = Options.parse(argc, argv);

		if (ParseResult.count("help") || IsNullInvoke == 1)
		{
			std::string Help = Options.help();

			printf("%s\n", Help.c_str());

			printf("available commands:\n");

			std::map<std::string, ZenCmdCategory*> Categories;

			for (const CommandInfo& CmdInfo : Commands)
			{
				ZenCmdCategory& Category = CmdInfo.Cmd->CommandCategory();

				Categories[Category.Name]			 = &Category;
				Category.SortedCmds[CmdInfo.CmdName] = CmdInfo.CmdSummary;
			}

			for (const auto& CategoryKv : Categories)
			{
				fmt::print("  {}\n\n", CategoryKv.first);

				for (const auto& Kv : CategoryKv.second->SortedCmds)
				{
					printf("    %-20s %s\n", Kv.first.c_str(), Kv.second.c_str());
				}

				printf("\n");
			}

			exit(0);
		}

		if (GlobalOptions.IsDebug)
		{
			logging::SetLogLevel(logging::level::Debug);
		}
		if (GlobalOptions.IsVerbose)
		{
			logging::SetLogLevel(logging::level::Trace);
		}

		for (const CommandInfo& CmdInfo : Commands)
		{
			if (StrCaseCompare(SubCommand.c_str(), CmdInfo.CmdName) == 0)
			{
				cxxopts::Options& VerbOptions = CmdInfo.Cmd->Options();

				try
				{
					return CmdInfo.Cmd->Run(GlobalOptions, (int)CommandArgVec.size(), CommandArgVec.data());
				}
				catch (const OptionParseException& Ex)
				{
					std::string help = VerbOptions.help();

					printf("Error parsing arguments for command '%s': %s\n\n%s", SubCommand.c_str(), Ex.what(), help.c_str());

					exit(11);
				}
			}
		}

		printf("Unknown command specified: '%s', exiting\n", SubCommand.c_str());
	}
	catch (const OptionParseException& Ex)
	{
		std::string HelpMessage = Options.help();

		printf("Error parsing program arguments: %s\n\n%s", Ex.what(), HelpMessage.c_str());

		return 9;
	}
	catch (const std::system_error& Ex)
	{
		printf("System Error: %s\n", Ex.what());

		return Ex.code() ? Ex.code().value() : 10;
	}
	catch (const std::exception& Ex)
	{
		printf("Error: %s\n", Ex.what());

		return 11;
	}

	return 0;
}