From c7d4dc6a4d13881028d566f5ce501335e47e48bf Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 22 Sep 2023 08:22:06 -0400 Subject: Collect all zen admin-related commands into admin.h/.cpp (#418) * move commands in scrub.h/cpp to admin_cmd.h/cpp * move job command into admin_cmd.h/.cpp * admin -> admin_cmd * bench -> bench_cmd * cache -> cache_cmd * copy -> copy_cmd * dedup -> dedup_cmd * hash -> hash_cmd * print -> print_cmd * projectstore -> projectstore_cmd * rpcreplay -> rpcreplay_cmd * serve -> serve_cmd * status -> status_cmd * top -> top_cmd * trace -> trace_cmd * up -> up_cmd * version -> version_cmd --- src/zen/cmds/admin_cmd.cpp | 270 +++++++ src/zen/cmds/admin_cmd.h | 76 ++ src/zen/cmds/bench.cpp | 217 ------ src/zen/cmds/bench.h | 20 - src/zen/cmds/bench_cmd.cpp | 217 ++++++ src/zen/cmds/bench_cmd.h | 20 + src/zen/cmds/cache.cpp | 304 -------- src/zen/cmds/cache.h | 68 -- src/zen/cmds/cache_cmd.cpp | 304 ++++++++ src/zen/cmds/cache_cmd.h | 68 ++ src/zen/cmds/copy.cpp | 194 ----- src/zen/cmds/copy.h | 27 - src/zen/cmds/copy_cmd.cpp | 194 +++++ src/zen/cmds/copy_cmd.h | 27 + src/zen/cmds/dedup.cpp | 302 ------- src/zen/cmds/dedup.h | 28 - src/zen/cmds/dedup_cmd.cpp | 302 +++++++ src/zen/cmds/dedup_cmd.h | 28 + src/zen/cmds/hash.cpp | 171 ---- src/zen/cmds/hash.h | 27 - src/zen/cmds/hash_cmd.cpp | 171 ++++ src/zen/cmds/hash_cmd.h | 27 + src/zen/cmds/jobs.cpp | 82 -- src/zen/cmds/jobs.h | 27 - src/zen/cmds/print.cpp | 193 ----- src/zen/cmds/print.h | 41 - src/zen/cmds/print_cmd.cpp | 193 +++++ src/zen/cmds/print_cmd.h | 41 + src/zen/cmds/projectstore.cpp | 1558 ------------------------------------- src/zen/cmds/projectstore.h | 256 ------ src/zen/cmds/projectstore_cmd.cpp | 1558 +++++++++++++++++++++++++++++++++++++ src/zen/cmds/projectstore_cmd.h | 256 ++++++ src/zen/cmds/rpcreplay.cpp | 438 ----------- src/zen/cmds/rpcreplay.h | 65 -- src/zen/cmds/rpcreplay_cmd.cpp | 438 +++++++++++ src/zen/cmds/rpcreplay_cmd.h | 65 ++ src/zen/cmds/scrub.cpp | 201 ----- src/zen/cmds/scrub.h | 58 -- src/zen/cmds/serve.cpp | 242 ------ src/zen/cmds/serve.h | 28 - src/zen/cmds/serve_cmd.cpp | 242 ++++++ src/zen/cmds/serve_cmd.h | 28 + src/zen/cmds/status.cpp | 42 - src/zen/cmds/status.h | 22 - src/zen/cmds/status_cmd.cpp | 42 + src/zen/cmds/status_cmd.h | 22 + src/zen/cmds/top.cpp | 90 --- src/zen/cmds/top.h | 35 - src/zen/cmds/top_cmd.cpp | 90 +++ src/zen/cmds/top_cmd.h | 35 + src/zen/cmds/trace.cpp | 93 --- src/zen/cmds/trace.h | 28 - src/zen/cmds/trace_cmd.cpp | 93 +++ src/zen/cmds/trace_cmd.h | 28 + src/zen/cmds/up.cpp | 176 ----- src/zen/cmds/up.h | 54 -- src/zen/cmds/up_cmd.cpp | 176 +++++ src/zen/cmds/up_cmd.h | 54 ++ src/zen/cmds/version.cpp | 79 -- src/zen/cmds/version.h | 24 - src/zen/cmds/version_cmd.cpp | 79 ++ src/zen/cmds/version_cmd.h | 24 + src/zen/zen.cpp | 31 +- 63 files changed, 5183 insertions(+), 5206 deletions(-) create mode 100644 src/zen/cmds/admin_cmd.cpp create mode 100644 src/zen/cmds/admin_cmd.h delete mode 100644 src/zen/cmds/bench.cpp delete mode 100644 src/zen/cmds/bench.h create mode 100644 src/zen/cmds/bench_cmd.cpp create mode 100644 src/zen/cmds/bench_cmd.h delete mode 100644 src/zen/cmds/cache.cpp delete mode 100644 src/zen/cmds/cache.h create mode 100644 src/zen/cmds/cache_cmd.cpp create mode 100644 src/zen/cmds/cache_cmd.h delete mode 100644 src/zen/cmds/copy.cpp delete mode 100644 src/zen/cmds/copy.h create mode 100644 src/zen/cmds/copy_cmd.cpp create mode 100644 src/zen/cmds/copy_cmd.h delete mode 100644 src/zen/cmds/dedup.cpp delete mode 100644 src/zen/cmds/dedup.h create mode 100644 src/zen/cmds/dedup_cmd.cpp create mode 100644 src/zen/cmds/dedup_cmd.h delete mode 100644 src/zen/cmds/hash.cpp delete mode 100644 src/zen/cmds/hash.h create mode 100644 src/zen/cmds/hash_cmd.cpp create mode 100644 src/zen/cmds/hash_cmd.h delete mode 100644 src/zen/cmds/jobs.cpp delete mode 100644 src/zen/cmds/jobs.h delete mode 100644 src/zen/cmds/print.cpp delete mode 100644 src/zen/cmds/print.h create mode 100644 src/zen/cmds/print_cmd.cpp create mode 100644 src/zen/cmds/print_cmd.h delete mode 100644 src/zen/cmds/projectstore.cpp delete mode 100644 src/zen/cmds/projectstore.h create mode 100644 src/zen/cmds/projectstore_cmd.cpp create mode 100644 src/zen/cmds/projectstore_cmd.h delete mode 100644 src/zen/cmds/rpcreplay.cpp delete mode 100644 src/zen/cmds/rpcreplay.h create mode 100644 src/zen/cmds/rpcreplay_cmd.cpp create mode 100644 src/zen/cmds/rpcreplay_cmd.h delete mode 100644 src/zen/cmds/scrub.cpp delete mode 100644 src/zen/cmds/scrub.h delete mode 100644 src/zen/cmds/serve.cpp delete mode 100644 src/zen/cmds/serve.h create mode 100644 src/zen/cmds/serve_cmd.cpp create mode 100644 src/zen/cmds/serve_cmd.h delete mode 100644 src/zen/cmds/status.cpp delete mode 100644 src/zen/cmds/status.h create mode 100644 src/zen/cmds/status_cmd.cpp create mode 100644 src/zen/cmds/status_cmd.h delete mode 100644 src/zen/cmds/top.cpp delete mode 100644 src/zen/cmds/top.h create mode 100644 src/zen/cmds/top_cmd.cpp create mode 100644 src/zen/cmds/top_cmd.h delete mode 100644 src/zen/cmds/trace.cpp delete mode 100644 src/zen/cmds/trace.h create mode 100644 src/zen/cmds/trace_cmd.cpp create mode 100644 src/zen/cmds/trace_cmd.h delete mode 100644 src/zen/cmds/up.cpp delete mode 100644 src/zen/cmds/up.h create mode 100644 src/zen/cmds/up_cmd.cpp create mode 100644 src/zen/cmds/up_cmd.h delete mode 100644 src/zen/cmds/version.cpp delete mode 100644 src/zen/cmds/version.h create mode 100644 src/zen/cmds/version_cmd.cpp create mode 100644 src/zen/cmds/version_cmd.h (limited to 'src') diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp new file mode 100644 index 000000000..0aef968a9 --- /dev/null +++ b/src/zen/cmds/admin_cmd.cpp @@ -0,0 +1,270 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "admin_cmd.h" +#include +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +using namespace std::literals; + +namespace zen { + +ScrubCommand::ScrubCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); +} + +ScrubCommand::~ScrubCommand() = default; + +int +ScrubCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + zen::HttpClient Http(m_HostName); + + if (zen::HttpClient::Response Response = Http.Post("/admin/scrub"sv)) + { + ZEN_CONSOLE("OK: {}", Response.ToText()); + + return 0; + } + else if (int StatusCode = (int)Response.StatusCode) + { + ZEN_ERROR("scrub start failed: {}: {} ({})", + (int)Response.StatusCode, + ReasonStringForHttpResultCode((int)Response.StatusCode), + Response.AsText()); + } + else + { + ZEN_ERROR("scrub start failed: {}", Response.AsText()); + } + + return 1; +} + +////////////////////////////////////////////////////////////////////////// + +GcCommand::GcCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", + "s", + "smallobjects", + "Collect small objects", + cxxopts::value(m_SmallObjects)->default_value("false"), + ""); + m_Options.add_option("", + "m", + "maxcacheduration", + "Max cache lifetime (in seconds)", + cxxopts::value(m_MaxCacheDuration)->default_value("0"), + ""); + m_Options.add_option("", + "d", + "disksizesoftlimit", + "Max disk usage size (in bytes)", + cxxopts::value(m_DiskSizeSoftLimit)->default_value("0"), + ""); +} + +GcCommand::~GcCommand() +{ +} + +int +GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + cpr::Parameters Params; + if (m_SmallObjects) + { + Params.Add({"smallobjects", "true"}); + } + if (m_MaxCacheDuration != 0) + { + Params.Add({"maxcacheduration", fmt::format("{}", m_MaxCacheDuration)}); + } + if (m_DiskSizeSoftLimit != 0) + { + Params.Add({"disksizesoftlimit", fmt::format("{}", m_DiskSizeSoftLimit)}); + } + + cpr::Session Session; + Session.SetHeader(cpr::Header{{"Accept", "application/json"}}); + Session.SetUrl({fmt::format("{}/admin/gc", m_HostName)}); + Session.SetParameters(Params); + + cpr::Response Result = Session.Post(); + + if (zen::IsHttpSuccessCode(Result.status_code)) + { + ZEN_CONSOLE("OK: {}", Result.text); + return 0; + } + + if (Result.status_code) + { + ZEN_ERROR("GC start failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); + } + else + { + ZEN_ERROR("GC start failed: {}", Result.error.message); + } + + return 1; +} + +GcStatusCommand::GcStatusCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); +} + +GcStatusCommand::~GcStatusCommand() +{ +} + +int +GcStatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + cpr::Session Session; + Session.SetHeader(cpr::Header{{"Accept", "application/json"}}); + Session.SetUrl({fmt::format("{}/admin/gc", m_HostName)}); + + cpr::Response Result = Session.Get(); + + if (zen::IsHttpSuccessCode(Result.status_code)) + { + ZEN_CONSOLE("OK: {}", Result.text); + return 0; + } + + if (Result.status_code) + { + ZEN_ERROR("GC status failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); + } + else + { + ZEN_ERROR("GC status failed: {}", Result.error.message); + } + + return 1; +} + +//////////////////////////////////////////// + +JobCommand::JobCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "j", "jobid", "Job id", cxxopts::value(m_JobId), ""); + m_Options.add_option("", "c", "cancel", "Cancel job id", cxxopts::value(m_Cancel), ""); +} + +JobCommand::~JobCommand() = default; + +int +JobCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + using namespace std::literals; + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + HttpClient Http(m_HostName); + + if (m_Cancel) + { + if (m_JobId == 0) + { + ZEN_ERROR("Job id must be given"); + return 1; + } + } + std::string Url = m_JobId != 0 ? fmt::format("/admin/jobs/{}", m_JobId) : "/admin/jobs"; + + if (m_Cancel) + { + if (HttpClient::Response Result = Http.Delete(Url, HttpClient::Accept(ZenContentType::kJSON))) + { + ZEN_CONSOLE("{}", Result); + } + else + { + Result.ThrowError("failed cancelling job"sv); + return 1; + } + } + else if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) + { + ZEN_CONSOLE("{}", Result.AsText()); + } + else + { + Result.ThrowError("failed fetching job info"sv); + return 1; + } + + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h new file mode 100644 index 000000000..6caab7138 --- /dev/null +++ b/src/zen/cmds/admin_cmd.h @@ -0,0 +1,76 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +/** Scrub storage + */ +class ScrubCommand : public ZenCmdBase +{ +public: + ScrubCommand(); + ~ScrubCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"scrub", "Scrub zen storage"}; + std::string m_HostName; +}; + +/** Garbage collect storage + */ +class GcCommand : public ZenCmdBase +{ +public: + GcCommand(); + ~GcCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"gc", "Garbage collect zen storage"}; + std::string m_HostName; + bool m_SmallObjects{false}; + uint64_t m_MaxCacheDuration{0}; + uint64_t m_DiskSizeSoftLimit{0}; +}; + +class GcStatusCommand : public ZenCmdBase +{ +public: + GcStatusCommand(); + ~GcStatusCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"gc-status", "Garbage collect zen storage status check"}; + std::string m_HostName; +}; + +//////////////////////////////////////////// + +class JobCommand : public ZenCmdBase +{ +public: + JobCommand(); + ~JobCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"jobs", "Show/cancel zen background jobs"}; + std::string m_HostName; + std::uint64_t m_JobId = 0; + bool m_Cancel = 0; +}; + +} // namespace zen diff --git a/src/zen/cmds/bench.cpp b/src/zen/cmds/bench.cpp deleted file mode 100644 index a2986ce16..000000000 --- a/src/zen/cmds/bench.cpp +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "bench.h" - -#include -#include -#include -#include -#include -#include -#include - -#if ZEN_PLATFORM_WINDOWS -# include -# include -# include -# include - -namespace zen::bench::util { - -// See https://www.geoffchappell.com/studies/windows/km/ntoskrnl/api/ex/sysinfo/set.htm - -typedef DWORD NTSTATUS; - -# define NT_SUCCESS(Status) (((NTSTATUS)(Status)) >= 0) -# define STATUS_PRIVILEGE_NOT_HELD ((NTSTATUS)0xC0000061L) - -typedef enum _SYSTEM_INFORMATION_CLASS -{ - SystemMemoryListInformation = - 80, // 80, q: SYSTEM_MEMORY_LIST_INFORMATION; s: SYSTEM_MEMORY_LIST_COMMAND (requires SeProfileSingleProcessPrivilege) -} SYSTEM_INFORMATION_CLASS; - -// private -typedef enum _SYSTEM_MEMORY_LIST_COMMAND -{ - MemoryCaptureAccessedBits, - MemoryCaptureAndResetAccessedBits, - MemoryEmptyWorkingSets, - MemoryFlushModifiedList, - MemoryPurgeStandbyList, - MemoryPurgeLowPriorityStandbyList, - MemoryCommandMax -} SYSTEM_MEMORY_LIST_COMMAND; - -BOOL -ObtainPrivilege(HANDLE TokenHandle, LPCSTR lpName, int flags) -{ - LUID Luid; - TOKEN_PRIVILEGES CurrentPriv; - TOKEN_PRIVILEGES NewPriv; - - DWORD dwBufferLength = 16; - if (LookupPrivilegeValueA(0, lpName, &Luid)) - { - NewPriv.PrivilegeCount = 1; - NewPriv.Privileges[0].Luid = Luid; - NewPriv.Privileges[0].Attributes = 0; - - if (AdjustTokenPrivileges(TokenHandle, - 0, - &NewPriv, - DWORD((LPBYTE) & (NewPriv.Privileges[1]) - (LPBYTE)&NewPriv), - &CurrentPriv, - &dwBufferLength)) - { - CurrentPriv.PrivilegeCount = 1; - CurrentPriv.Privileges[0].Luid = Luid; - CurrentPriv.Privileges[0].Attributes = flags != 0 ? 2 : 0; - - return AdjustTokenPrivileges(TokenHandle, 0, &CurrentPriv, dwBufferLength, 0, 0); - } - } - return FALSE; -} - -typedef NTSTATUS(WINAPI* NtSetSystemInformationFn)(INT, PVOID, ULONG); -typedef NTSTATUS(WINAPI* NtQuerySystemInformationFn)(INT, PVOID, ULONG, PULONG); - -struct elevation_required_exception : public std::runtime_error -{ - explicit elevation_required_exception(const std::string& What) : std::runtime_error{What} {} -}; - -void -EmptyStandByList() -{ - HMODULE NtDll = LoadLibrary(L"ntdll.dll"); - if (!NtDll) - { - zen::ThrowLastError("Could not LoadLibrary ntdll"); - } - - HANDLE hToken; - - if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, &hToken)) - { - zen::ThrowLastError("Could not open current process token"); - } - - if (!ObtainPrivilege(hToken, "SeProfileSingleProcessPrivilege", 1)) - { - zen::ThrowLastError("Unable to obtain SeProfileSingleProcessPrivilege"); - } - - CloseHandle(hToken); - - NtSetSystemInformationFn NtSetSystemInformation = (NtSetSystemInformationFn)GetProcAddress(NtDll, "NtSetSystemInformation"); - NtQuerySystemInformationFn NtQuerySystemInformation = (NtQuerySystemInformationFn)GetProcAddress(NtDll, "NtQuerySystemInformation"); - - if (!NtSetSystemInformation || !NtQuerySystemInformation) - { - throw std::runtime_error("Failed to look up required ntdll functions"); - } - - SYSTEM_MEMORY_LIST_COMMAND MemoryListCommand = MemoryPurgeStandbyList; - NTSTATUS NtStatus = NtSetSystemInformation(SystemMemoryListInformation, &MemoryListCommand, sizeof(MemoryListCommand)); - - if (NtStatus == STATUS_PRIVILEGE_NOT_HELD) - { - throw elevation_required_exception("Insufficient privileges to execute the memory list command"); - } - else if (!NT_SUCCESS(NtStatus)) - { - throw std::runtime_error(fmt::format("Unable to execute the memory list command (status={})", NtStatus)); - } -} - -} // namespace zen::bench::util - -#endif - -BenchCommand::BenchCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_options()("purge", - "Purge standby memory (system cache)", - cxxopts::value(m_PurgeStandbyLists)->default_value("false")); - m_Options.add_options()("single", "Do not spawn child processes", cxxopts::value(m_SingleProcess)->default_value("false")); -} - -BenchCommand::~BenchCommand() = default; - -int -BenchCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - -#if ZEN_PLATFORM_WINDOWS - if (m_PurgeStandbyLists) - { - bool Ok = false; - - zen::Stopwatch Timer; - - try - { - zen::bench::util::EmptyStandByList(); - - Ok = true; - } - catch (zen::bench::util::elevation_required_exception&) - { - ZEN_CONSOLE("purging standby lists requires elevation. Will try launch as elevated process"); - } - catch (std::exception& Ex) - { - ZEN_CONSOLE("ERROR: {}", Ex.what()); - } - - if (!Ok && !m_SingleProcess) - { - try - { - zen::CreateProcOptions Cpo; - Cpo.Flags = zen::CreateProcOptions::Flag_Elevated | zen::CreateProcOptions::Flag_NewConsole; - - std::filesystem::path CurExe{zen::GetRunningExecutablePath()}; - - if (zen::CreateProcResult Cpr = zen::CreateProc(CurExe, fmt::format("bench --purge --single"), Cpo)) - { - zen::ProcessHandle ProcHandle; - ProcHandle.Initialize(Cpr); - - int ExitCode = ProcHandle.WaitExitCode(); - - if (ExitCode == 0) - { - Ok = true; - } - else - { - ZEN_CONSOLE("ERROR: Elevated child process failed with return code {}", ExitCode); - } - } - } - catch (std::exception& Ex) - { - ZEN_CONSOLE("ERROR: {}", Ex.what()); - } - } - - if (Ok) - { - // TODO: could also add reporting on just how much memory was purged - ZEN_CONSOLE("purged standby lists! (took {})", zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - } - } -#endif - - return 0; -} diff --git a/src/zen/cmds/bench.h b/src/zen/cmds/bench.h deleted file mode 100644 index 8a8bd4a7c..000000000 --- a/src/zen/cmds/bench.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -class BenchCommand : public ZenCmdBase -{ -public: - BenchCommand(); - ~BenchCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"bench", "Benchmarking utility command"}; - bool m_PurgeStandbyLists = false; - bool m_SingleProcess = false; -}; diff --git a/src/zen/cmds/bench_cmd.cpp b/src/zen/cmds/bench_cmd.cpp new file mode 100644 index 000000000..06b8967a3 --- /dev/null +++ b/src/zen/cmds/bench_cmd.cpp @@ -0,0 +1,217 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "bench_cmd.h" + +#include +#include +#include +#include +#include +#include +#include + +#if ZEN_PLATFORM_WINDOWS +# include +# include +# include +# include + +namespace zen::bench::util { + +// See https://www.geoffchappell.com/studies/windows/km/ntoskrnl/api/ex/sysinfo/set.htm + +typedef DWORD NTSTATUS; + +# define NT_SUCCESS(Status) (((NTSTATUS)(Status)) >= 0) +# define STATUS_PRIVILEGE_NOT_HELD ((NTSTATUS)0xC0000061L) + +typedef enum _SYSTEM_INFORMATION_CLASS +{ + SystemMemoryListInformation = + 80, // 80, q: SYSTEM_MEMORY_LIST_INFORMATION; s: SYSTEM_MEMORY_LIST_COMMAND (requires SeProfileSingleProcessPrivilege) +} SYSTEM_INFORMATION_CLASS; + +// private +typedef enum _SYSTEM_MEMORY_LIST_COMMAND +{ + MemoryCaptureAccessedBits, + MemoryCaptureAndResetAccessedBits, + MemoryEmptyWorkingSets, + MemoryFlushModifiedList, + MemoryPurgeStandbyList, + MemoryPurgeLowPriorityStandbyList, + MemoryCommandMax +} SYSTEM_MEMORY_LIST_COMMAND; + +BOOL +ObtainPrivilege(HANDLE TokenHandle, LPCSTR lpName, int flags) +{ + LUID Luid; + TOKEN_PRIVILEGES CurrentPriv; + TOKEN_PRIVILEGES NewPriv; + + DWORD dwBufferLength = 16; + if (LookupPrivilegeValueA(0, lpName, &Luid)) + { + NewPriv.PrivilegeCount = 1; + NewPriv.Privileges[0].Luid = Luid; + NewPriv.Privileges[0].Attributes = 0; + + if (AdjustTokenPrivileges(TokenHandle, + 0, + &NewPriv, + DWORD((LPBYTE) & (NewPriv.Privileges[1]) - (LPBYTE)&NewPriv), + &CurrentPriv, + &dwBufferLength)) + { + CurrentPriv.PrivilegeCount = 1; + CurrentPriv.Privileges[0].Luid = Luid; + CurrentPriv.Privileges[0].Attributes = flags != 0 ? 2 : 0; + + return AdjustTokenPrivileges(TokenHandle, 0, &CurrentPriv, dwBufferLength, 0, 0); + } + } + return FALSE; +} + +typedef NTSTATUS(WINAPI* NtSetSystemInformationFn)(INT, PVOID, ULONG); +typedef NTSTATUS(WINAPI* NtQuerySystemInformationFn)(INT, PVOID, ULONG, PULONG); + +struct elevation_required_exception : public std::runtime_error +{ + explicit elevation_required_exception(const std::string& What) : std::runtime_error{What} {} +}; + +void +EmptyStandByList() +{ + HMODULE NtDll = LoadLibrary(L"ntdll.dll"); + if (!NtDll) + { + zen::ThrowLastError("Could not LoadLibrary ntdll"); + } + + HANDLE hToken; + + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, &hToken)) + { + zen::ThrowLastError("Could not open current process token"); + } + + if (!ObtainPrivilege(hToken, "SeProfileSingleProcessPrivilege", 1)) + { + zen::ThrowLastError("Unable to obtain SeProfileSingleProcessPrivilege"); + } + + CloseHandle(hToken); + + NtSetSystemInformationFn NtSetSystemInformation = (NtSetSystemInformationFn)GetProcAddress(NtDll, "NtSetSystemInformation"); + NtQuerySystemInformationFn NtQuerySystemInformation = (NtQuerySystemInformationFn)GetProcAddress(NtDll, "NtQuerySystemInformation"); + + if (!NtSetSystemInformation || !NtQuerySystemInformation) + { + throw std::runtime_error("Failed to look up required ntdll functions"); + } + + SYSTEM_MEMORY_LIST_COMMAND MemoryListCommand = MemoryPurgeStandbyList; + NTSTATUS NtStatus = NtSetSystemInformation(SystemMemoryListInformation, &MemoryListCommand, sizeof(MemoryListCommand)); + + if (NtStatus == STATUS_PRIVILEGE_NOT_HELD) + { + throw elevation_required_exception("Insufficient privileges to execute the memory list command"); + } + else if (!NT_SUCCESS(NtStatus)) + { + throw std::runtime_error(fmt::format("Unable to execute the memory list command (status={})", NtStatus)); + } +} + +} // namespace zen::bench::util + +#endif + +BenchCommand::BenchCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_options()("purge", + "Purge standby memory (system cache)", + cxxopts::value(m_PurgeStandbyLists)->default_value("false")); + m_Options.add_options()("single", "Do not spawn child processes", cxxopts::value(m_SingleProcess)->default_value("false")); +} + +BenchCommand::~BenchCommand() = default; + +int +BenchCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + +#if ZEN_PLATFORM_WINDOWS + if (m_PurgeStandbyLists) + { + bool Ok = false; + + zen::Stopwatch Timer; + + try + { + zen::bench::util::EmptyStandByList(); + + Ok = true; + } + catch (zen::bench::util::elevation_required_exception&) + { + ZEN_CONSOLE("purging standby lists requires elevation. Will try launch as elevated process"); + } + catch (std::exception& Ex) + { + ZEN_CONSOLE("ERROR: {}", Ex.what()); + } + + if (!Ok && !m_SingleProcess) + { + try + { + zen::CreateProcOptions Cpo; + Cpo.Flags = zen::CreateProcOptions::Flag_Elevated | zen::CreateProcOptions::Flag_NewConsole; + + std::filesystem::path CurExe{zen::GetRunningExecutablePath()}; + + if (zen::CreateProcResult Cpr = zen::CreateProc(CurExe, fmt::format("bench --purge --single"), Cpo)) + { + zen::ProcessHandle ProcHandle; + ProcHandle.Initialize(Cpr); + + int ExitCode = ProcHandle.WaitExitCode(); + + if (ExitCode == 0) + { + Ok = true; + } + else + { + ZEN_CONSOLE("ERROR: Elevated child process failed with return code {}", ExitCode); + } + } + } + catch (std::exception& Ex) + { + ZEN_CONSOLE("ERROR: {}", Ex.what()); + } + } + + if (Ok) + { + // TODO: could also add reporting on just how much memory was purged + ZEN_CONSOLE("purged standby lists! (took {})", zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } + } +#endif + + return 0; +} diff --git a/src/zen/cmds/bench_cmd.h b/src/zen/cmds/bench_cmd.h new file mode 100644 index 000000000..8a8bd4a7c --- /dev/null +++ b/src/zen/cmds/bench_cmd.h @@ -0,0 +1,20 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +class BenchCommand : public ZenCmdBase +{ +public: + BenchCommand(); + ~BenchCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"bench", "Benchmarking utility command"}; + bool m_PurgeStandbyLists = false; + bool m_SingleProcess = false; +}; diff --git a/src/zen/cmds/cache.cpp b/src/zen/cmds/cache.cpp deleted file mode 100644 index 15c24f9ee..000000000 --- a/src/zen/cmds/cache.cpp +++ /dev/null @@ -1,304 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "cache.h" - -#include -#include -#include -#include -#include - -#include - -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END - -DropCommand::DropCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), ""); - m_Options.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), ""); - m_Options.parse_positional({"namespace", "bucket"}); -} - -DropCommand::~DropCommand() = default; - -int -DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw zen::OptionParseException("unable to resolve server specification"); - } - - if (m_NamespaceName.empty()) - { - throw zen::OptionParseException("Drop command requires a namespace"); - } - - cpr::Session Session; - if (m_BucketName.empty()) - { - ZEN_CONSOLE("Dropping cache namespace '{}' from '{}'", m_NamespaceName, m_HostName); - Session.SetUrl({fmt::format("{}/z$/{}", m_HostName, m_NamespaceName)}); - } - else - { - ZEN_CONSOLE("Dropping cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, m_HostName); - Session.SetUrl({fmt::format("{}/z$/{}/{}", m_HostName, m_NamespaceName, m_BucketName)}); - } - - cpr::Response Result = Session.Delete(); - - if (zen::IsHttpSuccessCode(Result.status_code)) - { - ZEN_CONSOLE("OK: drop succeeded"); - - return 0; - } - - if (Result.status_code) - { - ZEN_ERROR("Drop failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); - } - else - { - ZEN_ERROR("Drop failed: {}", Result.error.message); - } - - return 1; -} - -CacheInfoCommand::CacheInfoCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), ""); - m_Options.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), ""); - m_Options.parse_positional({"namespace", "bucket"}); -} - -CacheInfoCommand::~CacheInfoCommand() = default; - -int -CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw zen::OptionParseException("unable to resolve server specification"); - } - - cpr::Session Session; - Session.SetHeader(cpr::Header{{"Accept", "application/json"}}); - if (m_HostName.empty()) - { - ZEN_CONSOLE("Info on cache from '{}'", m_HostName); - Session.SetUrl({fmt::format("{}/z$", m_HostName)}); - } - else if (m_BucketName.empty()) - { - ZEN_CONSOLE("Info on cache namespace '{}' from '{}'", m_NamespaceName, m_HostName); - Session.SetUrl({fmt::format("{}/z$/{}", m_HostName, m_NamespaceName)}); - } - else - { - ZEN_CONSOLE("Info on cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, m_HostName); - Session.SetUrl({fmt::format("{}/z$/{}/{}", m_HostName, m_NamespaceName, m_BucketName)}); - } - - cpr::Response Result = Session.Get(); - - if (zen::IsHttpSuccessCode(Result.status_code)) - { - ZEN_CONSOLE("{}", Result.text); - - return 0; - } - - if (Result.status_code) - { - ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); - } - else - { - ZEN_ERROR("Info failed: {}", Result.error.message); - } - - return 1; -} - -CacheStatsCommand::CacheStatsCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); -} - -CacheStatsCommand::~CacheStatsCommand() = default; - -int -CacheStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw zen::OptionParseException("unable to resolve server specification"); - } - - cpr::Session Session; - Session.SetUrl({fmt::format("{}/stats/z$", m_HostName)}); - Session.SetHeader(cpr::Header{{"Accept", "application/json"}}); - - cpr::Response Result = Session.Get(); - - if (zen::IsHttpSuccessCode(Result.status_code)) - { - ZEN_CONSOLE("{}", Result.text); - - return 0; - } - - if (Result.status_code) - { - ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); - } - else - { - ZEN_ERROR("Info failed: {}", Result.error.message); - } - - return 1; -} - -CacheDetailsCommand::CacheDetailsCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "c", "csv", "Info on csv format", cxxopts::value(m_CSV), ""); - m_Options.add_option("", "d", "details", "Get detailed information about records", cxxopts::value(m_Details), "
"); - m_Options.add_option("", - "a", - "attachmentdetails", - "Get detailed information about attachments", - cxxopts::value(m_AttachmentDetails), - ""); - m_Options.add_option("", "n", "namespace", "Namespace name to get info for", cxxopts::value(m_Namespace), ""); - m_Options.add_option("", "b", "bucket", "Filter on bucket name", cxxopts::value(m_Bucket), ""); - m_Options.add_option("", "v", "valuekey", "Filter on value key hash string", cxxopts::value(m_ValueKey), ""); -} - -CacheDetailsCommand::~CacheDetailsCommand() = default; - -int -CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw zen::OptionParseException("unable to resolve server specification"); - } - - cpr::Session Session; - cpr::Parameters Parameters; - if (m_Details) - { - Parameters.Add({"details", "true"}); - } - if (m_AttachmentDetails) - { - Parameters.Add({"attachmentdetails", "true"}); - } - if (m_CSV) - { - Parameters.Add({"csv", "true"}); - } - else - { - Session.SetHeader(cpr::Header{{"Accept", "application/json"}}); - } - - if (!m_ValueKey.empty()) - { - if (m_Namespace.empty() || m_Bucket.empty()) - { - ZEN_ERROR("Provide namespace and bucket name"); - ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); - return 1; - } - Session.SetUrl({fmt::format("{}/z$/details$/{}/{}/{}", m_HostName, m_Namespace, m_Bucket, m_ValueKey)}); - } - else if (!m_Bucket.empty()) - { - if (m_Namespace.empty()) - { - ZEN_ERROR("Provide namespace name"); - ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); - return 1; - } - Session.SetUrl({fmt::format("{}/z$/details$/{}/{}", m_HostName, m_Namespace, m_Bucket)}); - } - else if (!m_Namespace.empty()) - { - Session.SetUrl({fmt::format("{}/z$/details$/{}", m_HostName, m_Namespace)}); - } - else - { - Session.SetUrl({fmt::format("{}/z$/details$", m_HostName)}); - } - Session.SetParameters(Parameters); - - cpr::Response Result = Session.Get(); - - if (zen::IsHttpSuccessCode(Result.status_code)) - { - ZEN_CONSOLE("{}", Result.text); - - return 0; - } - - if (Result.status_code) - { - ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); - } - else - { - ZEN_ERROR("Info failed: {}", Result.error.message); - } - - return 1; -} diff --git a/src/zen/cmds/cache.h b/src/zen/cmds/cache.h deleted file mode 100644 index 1f368bdec..000000000 --- a/src/zen/cmds/cache.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -class DropCommand : public ZenCmdBase -{ -public: - DropCommand(); - ~DropCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"drop", "Drop cache namespace or bucket"}; - std::string m_HostName; - std::string m_NamespaceName; - std::string m_BucketName; -}; - -class CacheInfoCommand : public ZenCmdBase -{ -public: - CacheInfoCommand(); - ~CacheInfoCommand(); - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"cache-info", "Info on cache, namespace or bucket"}; - std::string m_HostName; - std::string m_NamespaceName; - std::string m_BucketName; -}; - -class CacheStatsCommand : public ZenCmdBase -{ -public: - CacheStatsCommand(); - ~CacheStatsCommand(); - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"cache-stats", "Stats info on cache"}; - std::string m_HostName; -}; - -class CacheDetailsCommand : public ZenCmdBase -{ -public: - CacheDetailsCommand(); - ~CacheDetailsCommand(); - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"cache-details", "Detailed info on cache"}; - std::string m_HostName; - bool m_CSV; - bool m_Details; - bool m_AttachmentDetails; - std::string m_Namespace; - std::string m_Bucket; - std::string m_ValueKey; -}; diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp new file mode 100644 index 000000000..1bf6ee60e --- /dev/null +++ b/src/zen/cmds/cache_cmd.cpp @@ -0,0 +1,304 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "cache_cmd.h" + +#include +#include +#include +#include +#include + +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +DropCommand::DropCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), ""); + m_Options.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), ""); + m_Options.parse_positional({"namespace", "bucket"}); +} + +DropCommand::~DropCommand() = default; + +int +DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw zen::OptionParseException("unable to resolve server specification"); + } + + if (m_NamespaceName.empty()) + { + throw zen::OptionParseException("Drop command requires a namespace"); + } + + cpr::Session Session; + if (m_BucketName.empty()) + { + ZEN_CONSOLE("Dropping cache namespace '{}' from '{}'", m_NamespaceName, m_HostName); + Session.SetUrl({fmt::format("{}/z$/{}", m_HostName, m_NamespaceName)}); + } + else + { + ZEN_CONSOLE("Dropping cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, m_HostName); + Session.SetUrl({fmt::format("{}/z$/{}/{}", m_HostName, m_NamespaceName, m_BucketName)}); + } + + cpr::Response Result = Session.Delete(); + + if (zen::IsHttpSuccessCode(Result.status_code)) + { + ZEN_CONSOLE("OK: drop succeeded"); + + return 0; + } + + if (Result.status_code) + { + ZEN_ERROR("Drop failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); + } + else + { + ZEN_ERROR("Drop failed: {}", Result.error.message); + } + + return 1; +} + +CacheInfoCommand::CacheInfoCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "n", "namespace", "Namespace name", cxxopts::value(m_NamespaceName), ""); + m_Options.add_option("", "b", "bucket", "Bucket name", cxxopts::value(m_BucketName), ""); + m_Options.parse_positional({"namespace", "bucket"}); +} + +CacheInfoCommand::~CacheInfoCommand() = default; + +int +CacheInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw zen::OptionParseException("unable to resolve server specification"); + } + + cpr::Session Session; + Session.SetHeader(cpr::Header{{"Accept", "application/json"}}); + if (m_HostName.empty()) + { + ZEN_CONSOLE("Info on cache from '{}'", m_HostName); + Session.SetUrl({fmt::format("{}/z$", m_HostName)}); + } + else if (m_BucketName.empty()) + { + ZEN_CONSOLE("Info on cache namespace '{}' from '{}'", m_NamespaceName, m_HostName); + Session.SetUrl({fmt::format("{}/z$/{}", m_HostName, m_NamespaceName)}); + } + else + { + ZEN_CONSOLE("Info on cache bucket '{}/{}' from '{}'", m_NamespaceName, m_BucketName, m_HostName); + Session.SetUrl({fmt::format("{}/z$/{}/{}", m_HostName, m_NamespaceName, m_BucketName)}); + } + + cpr::Response Result = Session.Get(); + + if (zen::IsHttpSuccessCode(Result.status_code)) + { + ZEN_CONSOLE("{}", Result.text); + + return 0; + } + + if (Result.status_code) + { + ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); + } + else + { + ZEN_ERROR("Info failed: {}", Result.error.message); + } + + return 1; +} + +CacheStatsCommand::CacheStatsCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); +} + +CacheStatsCommand::~CacheStatsCommand() = default; + +int +CacheStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw zen::OptionParseException("unable to resolve server specification"); + } + + cpr::Session Session; + Session.SetUrl({fmt::format("{}/stats/z$", m_HostName)}); + Session.SetHeader(cpr::Header{{"Accept", "application/json"}}); + + cpr::Response Result = Session.Get(); + + if (zen::IsHttpSuccessCode(Result.status_code)) + { + ZEN_CONSOLE("{}", Result.text); + + return 0; + } + + if (Result.status_code) + { + ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); + } + else + { + ZEN_ERROR("Info failed: {}", Result.error.message); + } + + return 1; +} + +CacheDetailsCommand::CacheDetailsCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "c", "csv", "Info on csv format", cxxopts::value(m_CSV), ""); + m_Options.add_option("", "d", "details", "Get detailed information about records", cxxopts::value(m_Details), "
"); + m_Options.add_option("", + "a", + "attachmentdetails", + "Get detailed information about attachments", + cxxopts::value(m_AttachmentDetails), + ""); + m_Options.add_option("", "n", "namespace", "Namespace name to get info for", cxxopts::value(m_Namespace), ""); + m_Options.add_option("", "b", "bucket", "Filter on bucket name", cxxopts::value(m_Bucket), ""); + m_Options.add_option("", "v", "valuekey", "Filter on value key hash string", cxxopts::value(m_ValueKey), ""); +} + +CacheDetailsCommand::~CacheDetailsCommand() = default; + +int +CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw zen::OptionParseException("unable to resolve server specification"); + } + + cpr::Session Session; + cpr::Parameters Parameters; + if (m_Details) + { + Parameters.Add({"details", "true"}); + } + if (m_AttachmentDetails) + { + Parameters.Add({"attachmentdetails", "true"}); + } + if (m_CSV) + { + Parameters.Add({"csv", "true"}); + } + else + { + Session.SetHeader(cpr::Header{{"Accept", "application/json"}}); + } + + if (!m_ValueKey.empty()) + { + if (m_Namespace.empty() || m_Bucket.empty()) + { + ZEN_ERROR("Provide namespace and bucket name"); + ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); + return 1; + } + Session.SetUrl({fmt::format("{}/z$/details$/{}/{}/{}", m_HostName, m_Namespace, m_Bucket, m_ValueKey)}); + } + else if (!m_Bucket.empty()) + { + if (m_Namespace.empty()) + { + ZEN_ERROR("Provide namespace name"); + ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); + return 1; + } + Session.SetUrl({fmt::format("{}/z$/details$/{}/{}", m_HostName, m_Namespace, m_Bucket)}); + } + else if (!m_Namespace.empty()) + { + Session.SetUrl({fmt::format("{}/z$/details$/{}", m_HostName, m_Namespace)}); + } + else + { + Session.SetUrl({fmt::format("{}/z$/details$", m_HostName)}); + } + Session.SetParameters(Parameters); + + cpr::Response Result = Session.Get(); + + if (zen::IsHttpSuccessCode(Result.status_code)) + { + ZEN_CONSOLE("{}", Result.text); + + return 0; + } + + if (Result.status_code) + { + ZEN_ERROR("Info failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); + } + else + { + ZEN_ERROR("Info failed: {}", Result.error.message); + } + + return 1; +} diff --git a/src/zen/cmds/cache_cmd.h b/src/zen/cmds/cache_cmd.h new file mode 100644 index 000000000..1f368bdec --- /dev/null +++ b/src/zen/cmds/cache_cmd.h @@ -0,0 +1,68 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +class DropCommand : public ZenCmdBase +{ +public: + DropCommand(); + ~DropCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"drop", "Drop cache namespace or bucket"}; + std::string m_HostName; + std::string m_NamespaceName; + std::string m_BucketName; +}; + +class CacheInfoCommand : public ZenCmdBase +{ +public: + CacheInfoCommand(); + ~CacheInfoCommand(); + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"cache-info", "Info on cache, namespace or bucket"}; + std::string m_HostName; + std::string m_NamespaceName; + std::string m_BucketName; +}; + +class CacheStatsCommand : public ZenCmdBase +{ +public: + CacheStatsCommand(); + ~CacheStatsCommand(); + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"cache-stats", "Stats info on cache"}; + std::string m_HostName; +}; + +class CacheDetailsCommand : public ZenCmdBase +{ +public: + CacheDetailsCommand(); + ~CacheDetailsCommand(); + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"cache-details", "Detailed info on cache"}; + std::string m_HostName; + bool m_CSV; + bool m_Details; + bool m_AttachmentDetails; + std::string m_Namespace; + std::string m_Bucket; + std::string m_ValueKey; +}; diff --git a/src/zen/cmds/copy.cpp b/src/zen/cmds/copy.cpp deleted file mode 100644 index 6fff973ba..000000000 --- a/src/zen/cmds/copy.cpp +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "copy.h" - -#include -#include -#include -#include -#include - -namespace zen { - -CopyCommand::CopyCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_options()("no-clone", "Do not perform block clone", cxxopts::value(m_NoClone)->default_value("false")); - m_Options.add_option("", "s", "source", "Copy source", cxxopts::value(m_CopySource), ""); - m_Options.add_option("", "t", "target", "Copy target", cxxopts::value(m_CopyTarget), ""); - m_Options.parse_positional({"source", "target"}); -} - -CopyCommand::~CopyCommand() = default; - -int -CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ZenCmdBase::ParseOptions(argc, argv)) - { - return 0; - } - - // Validate arguments - - if (m_CopySource.empty()) - throw std::runtime_error("No source specified"); - - if (m_CopyTarget.empty()) - throw std::runtime_error("No target specified"); - - std::filesystem::path FromPath; - std::filesystem::path ToPath; - - FromPath = m_CopySource; - ToPath = m_CopyTarget; - - const bool IsFileCopy = std::filesystem::is_regular_file(m_CopySource); - const bool IsDirCopy = std::filesystem::is_directory(m_CopySource); - - if (!IsFileCopy && !IsDirCopy) - { - throw std::runtime_error("Invalid source specification (neither directory nor file)"); - } - - if (IsFileCopy && IsDirCopy) - { - throw std::runtime_error("Invalid source specification (both directory AND file!?)"); - } - - if (IsDirCopy) - { - if (std::filesystem::exists(ToPath)) - { - const bool IsTargetDir = std::filesystem::is_directory(ToPath); - if (!IsTargetDir) - { - if (std::filesystem::is_regular_file(ToPath)) - { - throw std::runtime_error("Attempted copy of directory into file"); - } - } - } - else - { - std::filesystem::create_directories(ToPath); - } - - // Multi file copy - - ZEN_CONSOLE("copying {} -> {}", FromPath, ToPath); - - zen::Stopwatch Timer; - - struct CopyVisitor : public FileSystemTraversal::TreeVisitor - { - CopyVisitor(std::filesystem::path InBasePath, zen::CopyFileOptions InCopyOptions) - : BasePath(InBasePath) - , CopyOptions(InCopyOptions) - { - } - - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override - { - ZEN_UNUSED(FileSize); - std::error_code Ec; - const std::filesystem::path Relative = std::filesystem::relative(Parent, BasePath, Ec); - - if (Ec) - { - FailedFileCount++; - } - else - { - const std::filesystem::path FromPath = Parent / File; - const std::filesystem::path ToPath = TargetPath / Relative / File; - - try - { - zen::CreateDirectories(TargetPath / Relative); - if (zen::CopyFile(FromPath, ToPath, CopyOptions)) - { - ++FileCount; - ByteCount += FileSize; - } - else - { - throw std::logic_error("CopyFile failed in an unexpected way"); - } - } - catch (std::exception& Ex) - { - ++FailedFileCount; - - ZEN_CONSOLE("Error: failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what()); - } - } - } - - virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; } - - std::filesystem::path BasePath; - std::filesystem::path TargetPath; - zen::CopyFileOptions CopyOptions; - int FileCount = 0; - uint64_t ByteCount = 0; - int FailedFileCount = 0; - }; - - zen::CopyFileOptions CopyOptions; - CopyOptions.EnableClone = !m_NoClone; - - CopyVisitor Visitor{FromPath, CopyOptions}; - Visitor.TargetPath = ToPath; - - FileSystemTraversal Traversal; - Traversal.TraverseFileSystem(FromPath, Visitor); - - ZEN_CONSOLE("Copy of {} files ({}) completed in {} ({})", - Visitor.FileCount, - NiceBytes(Visitor.ByteCount), - zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - zen::NiceRate(Visitor.ByteCount, (uint32_t)Timer.GetElapsedTimeMs())); - - if (Visitor.FailedFileCount) - { - ZEN_CONSOLE("{} file copy operations FAILED"); - - return 1; - } - } - else - { - // Single file copy - - zen::Stopwatch Timer; - - zen::CopyFileOptions CopyOptions; - CopyOptions.EnableClone = !m_NoClone; - - try - { - zen::CreateDirectories(ToPath.parent_path()); - if (zen::CopyFile(FromPath, ToPath, CopyOptions)) - { - ZEN_CONSOLE("Copy completed in {}", zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - } - else - { - throw std::logic_error("CopyFile failed in an unexpected way"); - } - } - catch (std::exception& Ex) - { - ZEN_CONSOLE("Error: failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what()); - - return 1; - } - } - - return 0; -} - -} // namespace zen diff --git a/src/zen/cmds/copy.h b/src/zen/cmds/copy.h deleted file mode 100644 index 549114160..000000000 --- a/src/zen/cmds/copy.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -/** Copy files, possibly using block cloning - */ -class CopyCommand : public ZenCmdBase -{ -public: - CopyCommand(); - ~CopyCommand(); - - virtual cxxopts::Options& Options() override { return m_Options; } - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - -private: - cxxopts::Options m_Options{"copy", "Copy files"}; - std::string m_CopySource; - std::string m_CopyTarget; - bool m_NoClone = false; -}; - -} // namespace zen diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp new file mode 100644 index 000000000..9f689e5bb --- /dev/null +++ b/src/zen/cmds/copy_cmd.cpp @@ -0,0 +1,194 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "copy_cmd.h" + +#include +#include +#include +#include +#include + +namespace zen { + +CopyCommand::CopyCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_options()("no-clone", "Do not perform block clone", cxxopts::value(m_NoClone)->default_value("false")); + m_Options.add_option("", "s", "source", "Copy source", cxxopts::value(m_CopySource), ""); + m_Options.add_option("", "t", "target", "Copy target", cxxopts::value(m_CopyTarget), ""); + m_Options.parse_positional({"source", "target"}); +} + +CopyCommand::~CopyCommand() = default; + +int +CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ZenCmdBase::ParseOptions(argc, argv)) + { + return 0; + } + + // Validate arguments + + if (m_CopySource.empty()) + throw std::runtime_error("No source specified"); + + if (m_CopyTarget.empty()) + throw std::runtime_error("No target specified"); + + std::filesystem::path FromPath; + std::filesystem::path ToPath; + + FromPath = m_CopySource; + ToPath = m_CopyTarget; + + const bool IsFileCopy = std::filesystem::is_regular_file(m_CopySource); + const bool IsDirCopy = std::filesystem::is_directory(m_CopySource); + + if (!IsFileCopy && !IsDirCopy) + { + throw std::runtime_error("Invalid source specification (neither directory nor file)"); + } + + if (IsFileCopy && IsDirCopy) + { + throw std::runtime_error("Invalid source specification (both directory AND file!?)"); + } + + if (IsDirCopy) + { + if (std::filesystem::exists(ToPath)) + { + const bool IsTargetDir = std::filesystem::is_directory(ToPath); + if (!IsTargetDir) + { + if (std::filesystem::is_regular_file(ToPath)) + { + throw std::runtime_error("Attempted copy of directory into file"); + } + } + } + else + { + std::filesystem::create_directories(ToPath); + } + + // Multi file copy + + ZEN_CONSOLE("copying {} -> {}", FromPath, ToPath); + + zen::Stopwatch Timer; + + struct CopyVisitor : public FileSystemTraversal::TreeVisitor + { + CopyVisitor(std::filesystem::path InBasePath, zen::CopyFileOptions InCopyOptions) + : BasePath(InBasePath) + , CopyOptions(InCopyOptions) + { + } + + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override + { + ZEN_UNUSED(FileSize); + std::error_code Ec; + const std::filesystem::path Relative = std::filesystem::relative(Parent, BasePath, Ec); + + if (Ec) + { + FailedFileCount++; + } + else + { + const std::filesystem::path FromPath = Parent / File; + const std::filesystem::path ToPath = TargetPath / Relative / File; + + try + { + zen::CreateDirectories(TargetPath / Relative); + if (zen::CopyFile(FromPath, ToPath, CopyOptions)) + { + ++FileCount; + ByteCount += FileSize; + } + else + { + throw std::logic_error("CopyFile failed in an unexpected way"); + } + } + catch (std::exception& Ex) + { + ++FailedFileCount; + + ZEN_CONSOLE("Error: failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what()); + } + } + } + + virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; } + + std::filesystem::path BasePath; + std::filesystem::path TargetPath; + zen::CopyFileOptions CopyOptions; + int FileCount = 0; + uint64_t ByteCount = 0; + int FailedFileCount = 0; + }; + + zen::CopyFileOptions CopyOptions; + CopyOptions.EnableClone = !m_NoClone; + + CopyVisitor Visitor{FromPath, CopyOptions}; + Visitor.TargetPath = ToPath; + + FileSystemTraversal Traversal; + Traversal.TraverseFileSystem(FromPath, Visitor); + + ZEN_CONSOLE("Copy of {} files ({}) completed in {} ({})", + Visitor.FileCount, + NiceBytes(Visitor.ByteCount), + zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs()), + zen::NiceRate(Visitor.ByteCount, (uint32_t)Timer.GetElapsedTimeMs())); + + if (Visitor.FailedFileCount) + { + ZEN_CONSOLE("{} file copy operations FAILED"); + + return 1; + } + } + else + { + // Single file copy + + zen::Stopwatch Timer; + + zen::CopyFileOptions CopyOptions; + CopyOptions.EnableClone = !m_NoClone; + + try + { + zen::CreateDirectories(ToPath.parent_path()); + if (zen::CopyFile(FromPath, ToPath, CopyOptions)) + { + ZEN_CONSOLE("Copy completed in {}", zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } + else + { + throw std::logic_error("CopyFile failed in an unexpected way"); + } + } + catch (std::exception& Ex) + { + ZEN_CONSOLE("Error: failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what()); + + return 1; + } + } + + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/copy_cmd.h b/src/zen/cmds/copy_cmd.h new file mode 100644 index 000000000..549114160 --- /dev/null +++ b/src/zen/cmds/copy_cmd.h @@ -0,0 +1,27 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +/** Copy files, possibly using block cloning + */ +class CopyCommand : public ZenCmdBase +{ +public: + CopyCommand(); + ~CopyCommand(); + + virtual cxxopts::Options& Options() override { return m_Options; } + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + +private: + cxxopts::Options m_Options{"copy", "Copy files"}; + std::string m_CopySource; + std::string m_CopyTarget; + bool m_NoClone = false; +}; + +} // namespace zen diff --git a/src/zen/cmds/dedup.cpp b/src/zen/cmds/dedup.cpp deleted file mode 100644 index b48fb8c2d..000000000 --- a/src/zen/cmds/dedup.cpp +++ /dev/null @@ -1,302 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "dedup.h" - -#include -#include -#include -#include -#include -#include -#include - -#if ZEN_PLATFORM_WINDOWS -# include -#endif - -#include - -namespace zen { - -//////////////////////////////////////////////////////////////////////////////// - -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - -namespace Concurrency { - - template - inline void parallel_invoke(T0 const& t0, T1 const& t1) - { - t0(); - t1(); - } - -} // namespace Concurrency - -#endif // ZEN_PLATFORM_LINUX/MAC - -//////////////////////////////////////////////////////////////////////////////// - -DedupCommand::DedupCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_options()("size", "Configure size threshold for dedup", cxxopts::value(m_SizeThreshold)->default_value("131072")); - m_Options.add_option("", "s", "source", "Copy source", cxxopts::value(m_DedupSource), ""); - m_Options.add_option("", "t", "target", "Copy target", cxxopts::value(m_DedupTarget), ""); - m_Options.add_option("", "", "positional", "Positional arguments", cxxopts::value(m_Positional), ""); - m_Options.parse_positional({"source", "target", "positional"}); -} - -DedupCommand::~DedupCommand() = default; - -int -DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - // Validate arguments - - const bool SourceGood = zen::SupportsBlockRefCounting(m_DedupSource); - const bool TargetGood = zen::SupportsBlockRefCounting(m_DedupTarget); - - if (!SourceGood) - { - ZEN_ERROR("Source directory '{}' does not support deduplication", m_DedupSource); - - return 0; - } - - if (!TargetGood) - { - ZEN_ERROR("Target directory '{}' does not support deduplication", m_DedupTarget); - - return 0; - } - - ZEN_CONSOLE("Performing dedup operation between {} and {}, size threshold {}", - m_DedupSource, - m_DedupTarget, - zen::NiceBytes(m_SizeThreshold)); - - using DirEntryList_t = std::list; - - zen::RwLock MapLock; - std::unordered_map FileSizeMap; - size_t CandidateCount = 0; - - auto AddToList = [&](const std::filesystem::directory_entry& Entry) { - if (Entry.is_regular_file()) - { - uintmax_t FileSize = Entry.file_size(); - if (FileSize > m_SizeThreshold) - { - zen::RwLock::ExclusiveLockScope _(MapLock); - FileSizeMap[FileSize].push_back(Entry); - ++CandidateCount; - } - } - }; - - std::filesystem::recursive_directory_iterator DirEnd; - - ZEN_CONSOLE("Gathering file info from source: '{}'", m_DedupSource); - ZEN_CONSOLE("Gathering file info from target: '{}'", m_DedupTarget); - - { - zen::Stopwatch Timer; - - Concurrency::parallel_invoke( - [&] { - for (std::filesystem::recursive_directory_iterator DirIt1(m_DedupSource); DirIt1 != DirEnd; ++DirIt1) - { - AddToList(*DirIt1); - } - }, - [&] { - for (std::filesystem::recursive_directory_iterator DirIt2(m_DedupTarget); DirIt2 != DirEnd; ++DirIt2) - { - AddToList(*DirIt2); - } - }); - - ZEN_CONSOLE("Gathered {} candidates across {} size buckets. Elapsed: {}", - CandidateCount, - FileSizeMap.size(), - zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - } - - ZEN_CONSOLE("Sorting buckets by size"); - - zen::Stopwatch Timer; - - uint64_t DupeBytes = 0; - - struct SizeList - { - size_t Size; - DirEntryList_t* DirEntries; - }; - - std::vector SizeLists{FileSizeMap.size()}; - - { - int i = 0; - - for (auto& kv : FileSizeMap) - { - ZEN_ASSERT(kv.first >= m_SizeThreshold); - SizeLists[i].Size = kv.first; - SizeLists[i].DirEntries = &kv.second; - ++i; - } - } - - std::sort(begin(SizeLists), end(SizeLists), [](const SizeList& Lhs, const SizeList& Rhs) { return Lhs.Size > Rhs.Size; }); - - ZEN_CONSOLE("Bucket summary:"); - - std::vector BucketId; - std::vector BucketOffsets; - std::vector BucketSizes; - std::vector BucketFileCounts; - - size_t TotalFileSizes = 0; - size_t TotalFileCount = 0; - - { - size_t CurrentPow2 = 0; - size_t BucketSize = 0; - size_t BucketFileCount = 0; - bool FirstBucket = true; - - for (size_t i = 0; i < SizeLists.size(); ++i) - { - const size_t ThisSize = SizeLists[i].Size; - const size_t Pow2 = zen::NextPow2(ThisSize); - - if (CurrentPow2 != Pow2) - { - CurrentPow2 = Pow2; - - if (!FirstBucket) - { - BucketSizes.push_back(BucketSize); - BucketFileCounts.push_back(BucketFileCount); - } - - BucketId.push_back(Pow2); - BucketOffsets.push_back(i); - - FirstBucket = false; - BucketSize = 0; - BucketFileCount = 0; - } - - BucketSize += ThisSize; - TotalFileSizes += ThisSize; - BucketFileCount += SizeLists[i].DirEntries->size(); - TotalFileCount += SizeLists[i].DirEntries->size(); - } - - if (!FirstBucket) - { - BucketSizes.push_back(BucketSize); - BucketFileCounts.push_back(BucketFileCount); - } - - ZEN_ASSERT(BucketOffsets.size() == BucketSizes.size()); - ZEN_ASSERT(BucketOffsets.size() == BucketFileCounts.size()); - } - - for (size_t i = 0; i < BucketOffsets.size(); ++i) - { - ZEN_CONSOLE(" Bucket {} : {}, {} candidates", zen::NiceBytes(BucketId[i]), zen::NiceBytes(BucketSizes[i]), BucketFileCounts[i]); - } - - ZEN_CONSOLE("Total : {}, {} candidates", zen::NiceBytes(TotalFileSizes), TotalFileCount); - - std::string CurrentNice; - - for (SizeList& Size : SizeLists) - { - std::string CurNice{zen::NiceBytes(zen::NextPow2(Size.Size))}; - - if (CurNice != CurrentNice) - { - CurrentNice = CurNice; - ZEN_CONSOLE("Now scanning bucket: {}", CurrentNice); - } - - std::unordered_map DedupMap; - - for (const auto& Entry : *Size.DirEntries) - { - zen::BLAKE3 Hash; - - if constexpr (true) - { - zen::BLAKE3Stream b3s; - - zen::ScanFile(Entry.path(), 64 * 1024, [&](const void* Data, size_t Size) { b3s.Append(Data, Size); }); - - Hash = b3s.GetHash(); - } - else - { - zen::FileContents Contents = zen::ReadFile(Entry.path()); - - zen::BLAKE3Stream b3s; - - for (zen::IoBuffer& Buffer : Contents.Data) - { - b3s.Append(Buffer.Data(), Buffer.Size()); - } - Hash = b3s.GetHash(); - } - - if (const std::filesystem::directory_entry* Dupe = DedupMap[Hash]) - { - std::string FileA = PathToUtf8(Dupe->path()); - std::string FileB = PathToUtf8(Entry.path()); - - size_t MinLen = std::min(FileA.size(), FileB.size()); - auto Its = std::mismatch(FileB.rbegin(), FileB.rbegin() + MinLen, FileA.rbegin()); - - if (Its.first != FileB.rbegin()) - { - if (Its.first[-1] == '\\' || Its.first[-1] == '/') - --Its.first; - - FileB = std::string(FileB.begin(), Its.first.base()) + "..."; - } - - ZEN_INFO("{} {} <-> {}", zen::NiceBytes(Entry.file_size()).c_str(), FileA.c_str(), FileB.c_str()); - - zen::CopyFileOptions Options; - Options.EnableClone = true; - Options.MustClone = true; - - zen::CopyFile(Dupe->path(), Entry.path(), Options); - - DupeBytes += Entry.file_size(); - } - else - { - DedupMap[Hash] = &Entry; - } - } - - Size.DirEntries->clear(); - } - - ZEN_CONSOLE("Elapsed: {} Deduped: {}", zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs()), zen::NiceBytes(DupeBytes)); - - return 0; -} - -} // namespace zen diff --git a/src/zen/cmds/dedup.h b/src/zen/cmds/dedup.h deleted file mode 100644 index 6318704f5..000000000 --- a/src/zen/cmds/dedup.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -/** Deduplicate files in a tree using block cloning - */ -class DedupCommand : public ZenCmdBase -{ -public: - DedupCommand(); - ~DedupCommand(); - - virtual cxxopts::Options& Options() override { return m_Options; } - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - -private: - cxxopts::Options m_Options{"dedup", "Deduplicate files"}; - std::vector m_Positional; - std::string m_DedupSource; - std::string m_DedupTarget; - size_t m_SizeThreshold = 1024 * 1024; -}; - -} // namespace zen diff --git a/src/zen/cmds/dedup_cmd.cpp b/src/zen/cmds/dedup_cmd.cpp new file mode 100644 index 000000000..d496cf404 --- /dev/null +++ b/src/zen/cmds/dedup_cmd.cpp @@ -0,0 +1,302 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "dedup_cmd.h" + +#include +#include +#include +#include +#include +#include +#include + +#if ZEN_PLATFORM_WINDOWS +# include +#endif + +#include + +namespace zen { + +//////////////////////////////////////////////////////////////////////////////// + +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + +namespace Concurrency { + + template + inline void parallel_invoke(T0 const& t0, T1 const& t1) + { + t0(); + t1(); + } + +} // namespace Concurrency + +#endif // ZEN_PLATFORM_LINUX/MAC + +//////////////////////////////////////////////////////////////////////////////// + +DedupCommand::DedupCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_options()("size", "Configure size threshold for dedup", cxxopts::value(m_SizeThreshold)->default_value("131072")); + m_Options.add_option("", "s", "source", "Copy source", cxxopts::value(m_DedupSource), ""); + m_Options.add_option("", "t", "target", "Copy target", cxxopts::value(m_DedupTarget), ""); + m_Options.add_option("", "", "positional", "Positional arguments", cxxopts::value(m_Positional), ""); + m_Options.parse_positional({"source", "target", "positional"}); +} + +DedupCommand::~DedupCommand() = default; + +int +DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + // Validate arguments + + const bool SourceGood = zen::SupportsBlockRefCounting(m_DedupSource); + const bool TargetGood = zen::SupportsBlockRefCounting(m_DedupTarget); + + if (!SourceGood) + { + ZEN_ERROR("Source directory '{}' does not support deduplication", m_DedupSource); + + return 0; + } + + if (!TargetGood) + { + ZEN_ERROR("Target directory '{}' does not support deduplication", m_DedupTarget); + + return 0; + } + + ZEN_CONSOLE("Performing dedup operation between {} and {}, size threshold {}", + m_DedupSource, + m_DedupTarget, + zen::NiceBytes(m_SizeThreshold)); + + using DirEntryList_t = std::list; + + zen::RwLock MapLock; + std::unordered_map FileSizeMap; + size_t CandidateCount = 0; + + auto AddToList = [&](const std::filesystem::directory_entry& Entry) { + if (Entry.is_regular_file()) + { + uintmax_t FileSize = Entry.file_size(); + if (FileSize > m_SizeThreshold) + { + zen::RwLock::ExclusiveLockScope _(MapLock); + FileSizeMap[FileSize].push_back(Entry); + ++CandidateCount; + } + } + }; + + std::filesystem::recursive_directory_iterator DirEnd; + + ZEN_CONSOLE("Gathering file info from source: '{}'", m_DedupSource); + ZEN_CONSOLE("Gathering file info from target: '{}'", m_DedupTarget); + + { + zen::Stopwatch Timer; + + Concurrency::parallel_invoke( + [&] { + for (std::filesystem::recursive_directory_iterator DirIt1(m_DedupSource); DirIt1 != DirEnd; ++DirIt1) + { + AddToList(*DirIt1); + } + }, + [&] { + for (std::filesystem::recursive_directory_iterator DirIt2(m_DedupTarget); DirIt2 != DirEnd; ++DirIt2) + { + AddToList(*DirIt2); + } + }); + + ZEN_CONSOLE("Gathered {} candidates across {} size buckets. Elapsed: {}", + CandidateCount, + FileSizeMap.size(), + zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } + + ZEN_CONSOLE("Sorting buckets by size"); + + zen::Stopwatch Timer; + + uint64_t DupeBytes = 0; + + struct SizeList + { + size_t Size; + DirEntryList_t* DirEntries; + }; + + std::vector SizeLists{FileSizeMap.size()}; + + { + int i = 0; + + for (auto& kv : FileSizeMap) + { + ZEN_ASSERT(kv.first >= m_SizeThreshold); + SizeLists[i].Size = kv.first; + SizeLists[i].DirEntries = &kv.second; + ++i; + } + } + + std::sort(begin(SizeLists), end(SizeLists), [](const SizeList& Lhs, const SizeList& Rhs) { return Lhs.Size > Rhs.Size; }); + + ZEN_CONSOLE("Bucket summary:"); + + std::vector BucketId; + std::vector BucketOffsets; + std::vector BucketSizes; + std::vector BucketFileCounts; + + size_t TotalFileSizes = 0; + size_t TotalFileCount = 0; + + { + size_t CurrentPow2 = 0; + size_t BucketSize = 0; + size_t BucketFileCount = 0; + bool FirstBucket = true; + + for (size_t i = 0; i < SizeLists.size(); ++i) + { + const size_t ThisSize = SizeLists[i].Size; + const size_t Pow2 = zen::NextPow2(ThisSize); + + if (CurrentPow2 != Pow2) + { + CurrentPow2 = Pow2; + + if (!FirstBucket) + { + BucketSizes.push_back(BucketSize); + BucketFileCounts.push_back(BucketFileCount); + } + + BucketId.push_back(Pow2); + BucketOffsets.push_back(i); + + FirstBucket = false; + BucketSize = 0; + BucketFileCount = 0; + } + + BucketSize += ThisSize; + TotalFileSizes += ThisSize; + BucketFileCount += SizeLists[i].DirEntries->size(); + TotalFileCount += SizeLists[i].DirEntries->size(); + } + + if (!FirstBucket) + { + BucketSizes.push_back(BucketSize); + BucketFileCounts.push_back(BucketFileCount); + } + + ZEN_ASSERT(BucketOffsets.size() == BucketSizes.size()); + ZEN_ASSERT(BucketOffsets.size() == BucketFileCounts.size()); + } + + for (size_t i = 0; i < BucketOffsets.size(); ++i) + { + ZEN_CONSOLE(" Bucket {} : {}, {} candidates", zen::NiceBytes(BucketId[i]), zen::NiceBytes(BucketSizes[i]), BucketFileCounts[i]); + } + + ZEN_CONSOLE("Total : {}, {} candidates", zen::NiceBytes(TotalFileSizes), TotalFileCount); + + std::string CurrentNice; + + for (SizeList& Size : SizeLists) + { + std::string CurNice{zen::NiceBytes(zen::NextPow2(Size.Size))}; + + if (CurNice != CurrentNice) + { + CurrentNice = CurNice; + ZEN_CONSOLE("Now scanning bucket: {}", CurrentNice); + } + + std::unordered_map DedupMap; + + for (const auto& Entry : *Size.DirEntries) + { + zen::BLAKE3 Hash; + + if constexpr (true) + { + zen::BLAKE3Stream b3s; + + zen::ScanFile(Entry.path(), 64 * 1024, [&](const void* Data, size_t Size) { b3s.Append(Data, Size); }); + + Hash = b3s.GetHash(); + } + else + { + zen::FileContents Contents = zen::ReadFile(Entry.path()); + + zen::BLAKE3Stream b3s; + + for (zen::IoBuffer& Buffer : Contents.Data) + { + b3s.Append(Buffer.Data(), Buffer.Size()); + } + Hash = b3s.GetHash(); + } + + if (const std::filesystem::directory_entry* Dupe = DedupMap[Hash]) + { + std::string FileA = PathToUtf8(Dupe->path()); + std::string FileB = PathToUtf8(Entry.path()); + + size_t MinLen = std::min(FileA.size(), FileB.size()); + auto Its = std::mismatch(FileB.rbegin(), FileB.rbegin() + MinLen, FileA.rbegin()); + + if (Its.first != FileB.rbegin()) + { + if (Its.first[-1] == '\\' || Its.first[-1] == '/') + --Its.first; + + FileB = std::string(FileB.begin(), Its.first.base()) + "..."; + } + + ZEN_INFO("{} {} <-> {}", zen::NiceBytes(Entry.file_size()).c_str(), FileA.c_str(), FileB.c_str()); + + zen::CopyFileOptions Options; + Options.EnableClone = true; + Options.MustClone = true; + + zen::CopyFile(Dupe->path(), Entry.path(), Options); + + DupeBytes += Entry.file_size(); + } + else + { + DedupMap[Hash] = &Entry; + } + } + + Size.DirEntries->clear(); + } + + ZEN_CONSOLE("Elapsed: {} Deduped: {}", zen::NiceTimeSpanMs(Timer.GetElapsedTimeMs()), zen::NiceBytes(DupeBytes)); + + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/dedup_cmd.h b/src/zen/cmds/dedup_cmd.h new file mode 100644 index 000000000..6318704f5 --- /dev/null +++ b/src/zen/cmds/dedup_cmd.h @@ -0,0 +1,28 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +/** Deduplicate files in a tree using block cloning + */ +class DedupCommand : public ZenCmdBase +{ +public: + DedupCommand(); + ~DedupCommand(); + + virtual cxxopts::Options& Options() override { return m_Options; } + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + +private: + cxxopts::Options m_Options{"dedup", "Deduplicate files"}; + std::vector m_Positional; + std::string m_DedupSource; + std::string m_DedupTarget; + size_t m_SizeThreshold = 1024 * 1024; +}; + +} // namespace zen diff --git a/src/zen/cmds/hash.cpp b/src/zen/cmds/hash.cpp deleted file mode 100644 index cc59ed46e..000000000 --- a/src/zen/cmds/hash.cpp +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "hash.h" - -#include -#include -#include -#include - -#if ZEN_PLATFORM_WINDOWS -# include -#endif - -namespace zen { - -//////////////////////////////////////////////////////////////////////////////// - -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - -namespace Concurrency { - - template - void parallel_for_each(IterType Cursor, IterType End, const LambdaType& Lambda) - { - for (; Cursor < End; ++Cursor) - { - Lambda(*Cursor); - } - } - - template - struct combinable - { - combinable& local() { return *this; } - - void operator+=(T Rhs) { Value += Rhs; } - - template - void combine_each(const LambdaType& Lambda) - { - Lambda(Value); - } - - T Value = 0; - }; - -} // namespace Concurrency - -#endif // ZEN_PLATFORM_LINUX|MAC - -//////////////////////////////////////////////////////////////////////////////// - -HashCommand::HashCommand() -{ - m_Options.add_options()("d,dir", "Directory to scan", cxxopts::value(m_ScanDirectory))( - "o,output", - "Output file", - cxxopts::value(m_OutputFile)); -} - -HashCommand::~HashCommand() = default; - -int -HashCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - bool valid = m_ScanDirectory.length(); - - if (!valid) - throw zen::OptionParseException("Hash command requires a directory to scan"); - - // Gather list of files to process - - ZEN_CONSOLE("Gathering files from {}", m_ScanDirectory); - - struct FileEntry - { - std::filesystem::path FilePath; - zen::BLAKE3 FileHash; - }; - - std::vector FileList; - uint64_t FileBytes = 0; - - std::filesystem::path ScanDirectoryPath{m_ScanDirectory}; - - for (const std::filesystem::directory_entry& Entry : std::filesystem::recursive_directory_iterator(ScanDirectoryPath)) - { - if (Entry.is_regular_file()) - { - FileList.push_back({Entry.path()}); - FileBytes += Entry.file_size(); - } - } - - ZEN_CONSOLE("Gathered {} files, total size {}", FileList.size(), zen::NiceBytes(FileBytes)); - - Concurrency::combinable TotalBytes; - - auto hashFile = [&](FileEntry& File) { - InternalFile InputFile; - InputFile.OpenRead(File.FilePath); - const uint8_t* DataPointer = (const uint8_t*)InputFile.MemoryMapFile(); - const size_t DataSize = InputFile.GetFileSize(); - - File.FileHash = zen::BLAKE3::HashMemory(DataPointer, DataSize); - - TotalBytes.local() += DataSize; - }; - - // Process them as quickly as possible - - zen::Stopwatch Timer; - -#if 1 - Concurrency::parallel_for_each(begin(FileList), end(FileList), [&](auto& file) { hashFile(file); }); -#else - for (const auto& file : FileList) - { - hashFile(file); - } -#endif - - size_t TotalByteCount = 0; - - TotalBytes.combine_each([&](size_t Total) { TotalByteCount += Total; }); - - const uint64_t ElapsedMs = Timer.GetElapsedTimeMs(); - ZEN_CONSOLE("Scanned {} files in {}", FileList.size(), zen::NiceTimeSpanMs(ElapsedMs)); - ZEN_CONSOLE("Total bytes {} ({})", zen::NiceBytes(TotalByteCount), zen::NiceByteRate(TotalByteCount, ElapsedMs)); - - InternalFile Output; - - if (m_OutputFile.empty()) - { - // TEMPORARY -- should properly open stdout - Output.OpenWrite("CONOUT$", false); - } - else - { - Output.OpenWrite(m_OutputFile, true); - } - - zen::ExtendableStringBuilder<256> Line; - - uint64_t CurrentOffset = 0; - - for (const auto& File : FileList) - { - Line.Append(File.FilePath.generic_u8string().c_str()); - Line.Append(','); - File.FileHash.ToHexString(Line); - Line.Append('\n'); - - Output.Write(Line.Data(), Line.Size(), CurrentOffset); - CurrentOffset += Line.Size(); - - Line.Reset(); - } - - // TODO: implement snapshot enumeration and display - return 0; -} - -} // namespace zen diff --git a/src/zen/cmds/hash.h b/src/zen/cmds/hash.h deleted file mode 100644 index e5ee071e9..000000000 --- a/src/zen/cmds/hash.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../internalfile.h" -#include "../zen.h" - -namespace zen { - -/** Generate hash list file - */ -class HashCommand : public ZenCmdBase -{ -public: - HashCommand(); - ~HashCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"hash", "Hash files"}; - std::string m_ScanDirectory; - std::string m_OutputFile; -}; - -} // namespace zen diff --git a/src/zen/cmds/hash_cmd.cpp b/src/zen/cmds/hash_cmd.cpp new file mode 100644 index 000000000..d1f7a1975 --- /dev/null +++ b/src/zen/cmds/hash_cmd.cpp @@ -0,0 +1,171 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "hash_cmd.h" + +#include +#include +#include +#include + +#if ZEN_PLATFORM_WINDOWS +# include +#endif + +namespace zen { + +//////////////////////////////////////////////////////////////////////////////// + +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + +namespace Concurrency { + + template + void parallel_for_each(IterType Cursor, IterType End, const LambdaType& Lambda) + { + for (; Cursor < End; ++Cursor) + { + Lambda(*Cursor); + } + } + + template + struct combinable + { + combinable& local() { return *this; } + + void operator+=(T Rhs) { Value += Rhs; } + + template + void combine_each(const LambdaType& Lambda) + { + Lambda(Value); + } + + T Value = 0; + }; + +} // namespace Concurrency + +#endif // ZEN_PLATFORM_LINUX|MAC + +//////////////////////////////////////////////////////////////////////////////// + +HashCommand::HashCommand() +{ + m_Options.add_options()("d,dir", "Directory to scan", cxxopts::value(m_ScanDirectory))( + "o,output", + "Output file", + cxxopts::value(m_OutputFile)); +} + +HashCommand::~HashCommand() = default; + +int +HashCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + bool valid = m_ScanDirectory.length(); + + if (!valid) + throw zen::OptionParseException("Hash command requires a directory to scan"); + + // Gather list of files to process + + ZEN_CONSOLE("Gathering files from {}", m_ScanDirectory); + + struct FileEntry + { + std::filesystem::path FilePath; + zen::BLAKE3 FileHash; + }; + + std::vector FileList; + uint64_t FileBytes = 0; + + std::filesystem::path ScanDirectoryPath{m_ScanDirectory}; + + for (const std::filesystem::directory_entry& Entry : std::filesystem::recursive_directory_iterator(ScanDirectoryPath)) + { + if (Entry.is_regular_file()) + { + FileList.push_back({Entry.path()}); + FileBytes += Entry.file_size(); + } + } + + ZEN_CONSOLE("Gathered {} files, total size {}", FileList.size(), zen::NiceBytes(FileBytes)); + + Concurrency::combinable TotalBytes; + + auto hashFile = [&](FileEntry& File) { + InternalFile InputFile; + InputFile.OpenRead(File.FilePath); + const uint8_t* DataPointer = (const uint8_t*)InputFile.MemoryMapFile(); + const size_t DataSize = InputFile.GetFileSize(); + + File.FileHash = zen::BLAKE3::HashMemory(DataPointer, DataSize); + + TotalBytes.local() += DataSize; + }; + + // Process them as quickly as possible + + zen::Stopwatch Timer; + +#if 1 + Concurrency::parallel_for_each(begin(FileList), end(FileList), [&](auto& file) { hashFile(file); }); +#else + for (const auto& file : FileList) + { + hashFile(file); + } +#endif + + size_t TotalByteCount = 0; + + TotalBytes.combine_each([&](size_t Total) { TotalByteCount += Total; }); + + const uint64_t ElapsedMs = Timer.GetElapsedTimeMs(); + ZEN_CONSOLE("Scanned {} files in {}", FileList.size(), zen::NiceTimeSpanMs(ElapsedMs)); + ZEN_CONSOLE("Total bytes {} ({})", zen::NiceBytes(TotalByteCount), zen::NiceByteRate(TotalByteCount, ElapsedMs)); + + InternalFile Output; + + if (m_OutputFile.empty()) + { + // TEMPORARY -- should properly open stdout + Output.OpenWrite("CONOUT$", false); + } + else + { + Output.OpenWrite(m_OutputFile, true); + } + + zen::ExtendableStringBuilder<256> Line; + + uint64_t CurrentOffset = 0; + + for (const auto& File : FileList) + { + Line.Append(File.FilePath.generic_u8string().c_str()); + Line.Append(','); + File.FileHash.ToHexString(Line); + Line.Append('\n'); + + Output.Write(Line.Data(), Line.Size(), CurrentOffset); + CurrentOffset += Line.Size(); + + Line.Reset(); + } + + // TODO: implement snapshot enumeration and display + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/hash_cmd.h b/src/zen/cmds/hash_cmd.h new file mode 100644 index 000000000..e5ee071e9 --- /dev/null +++ b/src/zen/cmds/hash_cmd.h @@ -0,0 +1,27 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../internalfile.h" +#include "../zen.h" + +namespace zen { + +/** Generate hash list file + */ +class HashCommand : public ZenCmdBase +{ +public: + HashCommand(); + ~HashCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"hash", "Hash files"}; + std::string m_ScanDirectory; + std::string m_OutputFile; +}; + +} // namespace zen diff --git a/src/zen/cmds/jobs.cpp b/src/zen/cmds/jobs.cpp deleted file mode 100644 index 137c321af..000000000 --- a/src/zen/cmds/jobs.cpp +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "jobs.h" - -#include -#include -#include -#include -#include -#include - -namespace zen { - -//////////////////////////////////////////// - -JobCommand::JobCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "j", "jobid", "Job id", cxxopts::value(m_JobId), ""); - m_Options.add_option("", "c", "cancel", "Cancel job id", cxxopts::value(m_Cancel), ""); -} - -JobCommand::~JobCommand() = default; - -int -JobCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - using namespace std::literals; - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - HttpClient Http(m_HostName); - - if (m_Cancel) - { - if (m_JobId == 0) - { - ZEN_ERROR("Job id must be given"); - return 1; - } - } - std::string Url = m_JobId != 0 ? fmt::format("/admin/jobs/{}", m_JobId) : "/admin/jobs"; - - if (m_Cancel) - { - if (HttpClient::Response Result = Http.Delete(Url, HttpClient::Accept(ZenContentType::kJSON))) - { - ZEN_CONSOLE("{}", Result); - } - else - { - Result.ThrowError("failed cancelling job"sv); - return 1; - } - } - else if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) - { - ZEN_CONSOLE("{}", Result.AsText()); - } - else - { - Result.ThrowError("failed fetching job info"sv); - return 1; - } - - return 0; -} - -} // namespace zen diff --git a/src/zen/cmds/jobs.h b/src/zen/cmds/jobs.h deleted file mode 100644 index 2c523f24a..000000000 --- a/src/zen/cmds/jobs.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -//////////////////////////////////////////// - -class JobCommand : public ZenCmdBase -{ -public: - JobCommand(); - ~JobCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"jobs", "Show/cancel zen background jobs"}; - std::string m_HostName; - std::uint64_t m_JobId = 0; - bool m_Cancel = 0; -}; - -} // namespace zen diff --git a/src/zen/cmds/print.cpp b/src/zen/cmds/print.cpp deleted file mode 100644 index a3a9bb3cc..000000000 --- a/src/zen/cmds/print.cpp +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "print.h" - -#include -#include -#include -#include -#include -#include -#include - -using namespace std::literals; - -namespace zen { - -static void -PrintCbObject(CbObject Object) -{ - zen::ExtendableStringBuilder<1024> ObjStr; - zen::CompactBinaryToJson(Object, ObjStr); - ZEN_CONSOLE("{}", ObjStr); -} - -static void -PrintCbObject(IoBuffer Data) -{ - zen::CbObject Object{SharedBuffer(Data)}; - - PrintCbObject(Object); -} - -PrintCommand::PrintCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "s", "source", "Object payload file (use '-' to read from STDIN)", cxxopts::value(m_Filename), ""); - m_Options.parse_positional({"source"}); -} - -PrintCommand::~PrintCommand() = default; - -int -PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - // Validate arguments - - if (m_Filename.empty()) - throw std::runtime_error("No file specified"); - - zen::FileContents Fc; - - if (m_Filename == "-") - { - Fc = zen::ReadStdIn(); - } - else - { - Fc = zen::ReadFile(m_Filename); - } - - if (Fc.ErrorCode) - { - ZEN_ERROR("Failed to read file '{}': {}", m_Filename, Fc.ErrorCode.message()); - - return 1; - } - - IoBuffer Data = Fc.Flatten(); - - IoHash RawHash; - uint64_t RawSize; - if (CompressedBuffer::ValidateCompressedHeader(Data, RawHash, RawSize)) - { - ZEN_CONSOLE("Compressed binary: size {}, raw size {}, hash: {}", Data.GetSize(), RawSize, RawHash); - } - else if (IsPackageMessage(Data)) - { - CbPackage Package = ParsePackageMessage(Data); - - CbObject Object = Package.GetObject(); - std::span Attachments = Package.GetAttachments(); - - ZEN_CONSOLE("Package - {} attachments, object hash {}", Package.GetAttachments().size(), Package.GetObjectHash()); - ZEN_CONSOLE(""); - - int AttachmentIndex = 1; - - for (const CbAttachment& Attachment : Attachments) - { - std::string AttachmentSize = "n/a"; - const char* AttachmentType = "unknown"; - - if (Attachment.IsCompressedBinary()) - { - AttachmentType = "Compressed"; - AttachmentSize = fmt::format("{} ({} uncompressed)", - Attachment.AsCompressedBinary().GetCompressedSize(), - Attachment.AsCompressedBinary().DecodeRawSize()); - } - else if (Attachment.IsBinary()) - { - AttachmentType = "Binary"; - AttachmentSize = fmt::format("{}", Attachment.AsBinary().GetSize()); - } - else if (Attachment.IsObject()) - { - AttachmentType = "Object"; - AttachmentSize = fmt::format("{}", Attachment.AsObject().GetSize()); - } - else if (Attachment.IsNull()) - { - AttachmentType = "null"; - } - - ZEN_CONSOLE("Attachment #{} : {}, {}, size {}", AttachmentIndex, Attachment.GetHash(), AttachmentType, AttachmentSize); - - ++AttachmentIndex; - } - - ZEN_CONSOLE("---8<---"); - - PrintCbObject(Object); - } - else if (CbValidateError Result = ValidateCompactBinary(Data, CbValidateMode::All); Result == CbValidateError::None) - { - PrintCbObject(Data); - } - else - { - ZEN_ERROR("Data in file '{}' does not appear to be compact binary (validation error {:#x})", m_Filename, uint32_t(Result)); - - return 1; - } - - return 0; -} - -////////////////////////////////////////////////////////////////////////// - -PrintPackageCommand::PrintPackageCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "s", "source", "Package payload file", cxxopts::value(m_Filename), ""); - m_Options.parse_positional({"source"}); -} - -PrintPackageCommand::~PrintPackageCommand() -{ -} - -int -PrintPackageCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - // Validate arguments - - if (m_Filename.empty()) - throw std::runtime_error("No file specified"); - - zen::FileContents Fc = zen::ReadFile(m_Filename); - IoBuffer Data = Fc.Flatten(); - zen::CbPackage Package; - - bool Ok = Package.TryLoad(Data) || zen::legacy::TryLoadCbPackage(Package, Data, &UniqueBuffer::Alloc); - - if (Ok) - { - zen::ExtendableStringBuilder<1024> ObjStr; - zen::CompactBinaryToJson(Package.GetObject(), ObjStr); - ZEN_CONSOLE("{}", ObjStr); - } - else - { - ZEN_ERROR("error: malformed package?"); - } - - return 0; -} - -} // namespace zen diff --git a/src/zen/cmds/print.h b/src/zen/cmds/print.h deleted file mode 100644 index 09d91830a..000000000 --- a/src/zen/cmds/print.h +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -/** Print Compact Binary - */ -class PrintCommand : public ZenCmdBase -{ -public: - PrintCommand(); - ~PrintCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"print", "Print compact binary object"}; - std::string m_Filename; -}; - -/** Print Compact Binary Package - */ -class PrintPackageCommand : public ZenCmdBase -{ -public: - PrintPackageCommand(); - ~PrintPackageCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"printpkg", "Print compact binary package"}; - std::string m_Filename; -}; - -} // namespace zen diff --git a/src/zen/cmds/print_cmd.cpp b/src/zen/cmds/print_cmd.cpp new file mode 100644 index 000000000..acffb2002 --- /dev/null +++ b/src/zen/cmds/print_cmd.cpp @@ -0,0 +1,193 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "print_cmd.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace std::literals; + +namespace zen { + +static void +PrintCbObject(CbObject Object) +{ + zen::ExtendableStringBuilder<1024> ObjStr; + zen::CompactBinaryToJson(Object, ObjStr); + ZEN_CONSOLE("{}", ObjStr); +} + +static void +PrintCbObject(IoBuffer Data) +{ + zen::CbObject Object{SharedBuffer(Data)}; + + PrintCbObject(Object); +} + +PrintCommand::PrintCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "s", "source", "Object payload file (use '-' to read from STDIN)", cxxopts::value(m_Filename), ""); + m_Options.parse_positional({"source"}); +} + +PrintCommand::~PrintCommand() = default; + +int +PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + // Validate arguments + + if (m_Filename.empty()) + throw std::runtime_error("No file specified"); + + zen::FileContents Fc; + + if (m_Filename == "-") + { + Fc = zen::ReadStdIn(); + } + else + { + Fc = zen::ReadFile(m_Filename); + } + + if (Fc.ErrorCode) + { + ZEN_ERROR("Failed to read file '{}': {}", m_Filename, Fc.ErrorCode.message()); + + return 1; + } + + IoBuffer Data = Fc.Flatten(); + + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer::ValidateCompressedHeader(Data, RawHash, RawSize)) + { + ZEN_CONSOLE("Compressed binary: size {}, raw size {}, hash: {}", Data.GetSize(), RawSize, RawHash); + } + else if (IsPackageMessage(Data)) + { + CbPackage Package = ParsePackageMessage(Data); + + CbObject Object = Package.GetObject(); + std::span Attachments = Package.GetAttachments(); + + ZEN_CONSOLE("Package - {} attachments, object hash {}", Package.GetAttachments().size(), Package.GetObjectHash()); + ZEN_CONSOLE(""); + + int AttachmentIndex = 1; + + for (const CbAttachment& Attachment : Attachments) + { + std::string AttachmentSize = "n/a"; + const char* AttachmentType = "unknown"; + + if (Attachment.IsCompressedBinary()) + { + AttachmentType = "Compressed"; + AttachmentSize = fmt::format("{} ({} uncompressed)", + Attachment.AsCompressedBinary().GetCompressedSize(), + Attachment.AsCompressedBinary().DecodeRawSize()); + } + else if (Attachment.IsBinary()) + { + AttachmentType = "Binary"; + AttachmentSize = fmt::format("{}", Attachment.AsBinary().GetSize()); + } + else if (Attachment.IsObject()) + { + AttachmentType = "Object"; + AttachmentSize = fmt::format("{}", Attachment.AsObject().GetSize()); + } + else if (Attachment.IsNull()) + { + AttachmentType = "null"; + } + + ZEN_CONSOLE("Attachment #{} : {}, {}, size {}", AttachmentIndex, Attachment.GetHash(), AttachmentType, AttachmentSize); + + ++AttachmentIndex; + } + + ZEN_CONSOLE("---8<---"); + + PrintCbObject(Object); + } + else if (CbValidateError Result = ValidateCompactBinary(Data, CbValidateMode::All); Result == CbValidateError::None) + { + PrintCbObject(Data); + } + else + { + ZEN_ERROR("Data in file '{}' does not appear to be compact binary (validation error {:#x})", m_Filename, uint32_t(Result)); + + return 1; + } + + return 0; +} + +////////////////////////////////////////////////////////////////////////// + +PrintPackageCommand::PrintPackageCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "s", "source", "Package payload file", cxxopts::value(m_Filename), ""); + m_Options.parse_positional({"source"}); +} + +PrintPackageCommand::~PrintPackageCommand() +{ +} + +int +PrintPackageCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + // Validate arguments + + if (m_Filename.empty()) + throw std::runtime_error("No file specified"); + + zen::FileContents Fc = zen::ReadFile(m_Filename); + IoBuffer Data = Fc.Flatten(); + zen::CbPackage Package; + + bool Ok = Package.TryLoad(Data) || zen::legacy::TryLoadCbPackage(Package, Data, &UniqueBuffer::Alloc); + + if (Ok) + { + zen::ExtendableStringBuilder<1024> ObjStr; + zen::CompactBinaryToJson(Package.GetObject(), ObjStr); + ZEN_CONSOLE("{}", ObjStr); + } + else + { + ZEN_ERROR("error: malformed package?"); + } + + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/print_cmd.h b/src/zen/cmds/print_cmd.h new file mode 100644 index 000000000..09d91830a --- /dev/null +++ b/src/zen/cmds/print_cmd.h @@ -0,0 +1,41 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +/** Print Compact Binary + */ +class PrintCommand : public ZenCmdBase +{ +public: + PrintCommand(); + ~PrintCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"print", "Print compact binary object"}; + std::string m_Filename; +}; + +/** Print Compact Binary Package + */ +class PrintPackageCommand : public ZenCmdBase +{ +public: + PrintPackageCommand(); + ~PrintPackageCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"printpkg", "Print compact binary package"}; + std::string m_Filename; +}; + +} // namespace zen diff --git a/src/zen/cmds/projectstore.cpp b/src/zen/cmds/projectstore.cpp deleted file mode 100644 index edeff7d85..000000000 --- a/src/zen/cmds/projectstore.cpp +++ /dev/null @@ -1,1558 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "projectstore.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END - -#include - -namespace zen { - -namespace { - - using namespace std::literals; - - const std::string DefaultCloudAccessTokenEnvVariableName( -#if ZEN_PLATFORM_WINDOWS - "UE-CloudDataCacheAccessToken"sv -#endif -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - "UE_CloudDataCacheAccessToken"sv -#endif - - ); - - IoBuffer MakeCbObjectPayload(std::function WriteCB) - { - CbObjectWriter Writer; - WriteCB(Writer); - IoBuffer Payload = Writer.Save().GetBuffer().AsIoBuffer(); - Payload.SetContentType(ZenContentType::kCbObject); - return Payload; - }; - - static std::atomic_uint32_t SignalCounter[NSIG] = {0}; - - static void SignalCallbackHandler(int SigNum) - { - if (SigNum >= 0 && SigNum < NSIG) - { - SignalCounter[SigNum].fetch_add(1); - } - } - - void AsyncPost(HttpClient& Http, std::string_view Url, IoBuffer&& Payload) - { - if (HttpClient::Response Result = Http.Post(Url, Payload)) - { - if (Result.StatusCode == HttpResponseCode::Accepted) - { - signal(SIGINT, SignalCallbackHandler); - bool Cancelled = false; - - std::string_view JobIdText = Result.AsText(); - std::optional JobIdMaybe = ParseInt(JobIdText); - if (!JobIdMaybe) - { - Result.ThrowError("invalid job id"sv); - } - - std::string LastCurrentOp; - uint32_t LastCurrentOpPercentComplete = 0; - - uint64_t JobId = JobIdMaybe.value(); - while (true) - { - HttpClient::Response StatusResult = - Http.Get(fmt::format("/admin/jobs/{}", JobId), HttpClient::Accept(ZenContentType::kCbObject)); - if (!StatusResult) - { - StatusResult.ThrowError("failed to create project"sv); - } - CbObject StatusObject = StatusResult.AsObject(); - std::string_view Status = StatusObject["Status"sv].AsString(); - CbArrayView Messages = StatusObject["Messages"sv].AsArrayView(); - for (auto M : Messages) - { - std::string_view Message = M.AsString(); - ZEN_CONSOLE("{}", Message); - } - if (Status == "Complete") - { - if (Cancelled) - { - ZEN_CONSOLE("Cancelled"); - } - else - { - double QueueTimeS = StatusObject["QueueTimeS"].AsDouble(); - double RuntimeS = StatusObject["RunTimeS"].AsDouble(); - ZEN_CONSOLE("Completed: QueueTime: {:.3} s, RunTime: {:.3} s", QueueTimeS, RuntimeS); - } - break; - } - if (Status == "Aborted") - { - Result.ThrowError("Aborted"); - break; - } - if (Status == "Queued") - { - double QueueTimeS = StatusObject["QueueTimeS"].AsDouble(); - ZEN_CONSOLE("Queued, waited {:.3} s...", QueueTimeS); - } - if (Status == "Running") - { - std::string_view CurrentOp = StatusObject["CurrentOp"sv].AsString(); - uint32_t CurrentOpPercentComplete = StatusObject["CurrentOpPercentComplete"sv].AsUInt32(); - if (CurrentOp != LastCurrentOp || CurrentOpPercentComplete != LastCurrentOpPercentComplete) - { - LastCurrentOp = CurrentOp; - LastCurrentOpPercentComplete = CurrentOpPercentComplete; - ZEN_CONSOLE("{} {}%", CurrentOp, CurrentOpPercentComplete); - } - } - uint32_t AbortCounter = SignalCounter[SIGINT].load(); - if (SignalCounter[SIGINT] > 0) - { - SignalCounter[SIGINT].fetch_sub(AbortCounter); - if (HttpClient::Response DeleteResult = Http.Delete(fmt::format("/admin/jobs/{}", JobId))) - { - ZEN_CONSOLE("Requested cancel..."); - Cancelled = true; - } - else - { - ZEN_CONSOLE("Failed cancelling job {}", DeleteResult); - } - continue; - } - Sleep(100); - } - } - else - { - ZEN_CONSOLE("{}", Result); - } - } - else - { - Result.ThrowError("failed to start operation"sv); - } - } - -} // namespace - -/////////////////////////////////////// - -DropProjectCommand::DropProjectCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), ""); - m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), ""); - m_Options.parse_positional({"project", "oplog"}); - m_Options.positional_help("[ []]"); -} - -DropProjectCommand::~DropProjectCommand() -{ -} - -int -DropProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - if (m_ProjectName.empty()) - { - throw OptionParseException("Drop command requires a project"); - } - - HttpClient Http(m_HostName); - if (m_OplogName.empty()) - { - ZEN_CONSOLE("Dropping project '{}' from '{}'", m_ProjectName, m_HostName); - if (HttpClient::Response Result = Http.Delete(fmt::format("/prj/{}", m_ProjectName))) - { - ZEN_CONSOLE("{}", Result); - } - else - { - Result.ThrowError("delete project failed"sv); - return 1; - } - } - else - { - ZEN_CONSOLE("Dropping oplog '{}/{}' from '{}'", m_ProjectName, m_OplogName, m_HostName); - if (HttpClient::Response Result = Http.Delete(fmt::format("/prj/{}/oplog/{}", m_ProjectName, m_OplogName))) - { - ZEN_CONSOLE("{}", Result); - } - else - { - Result.ThrowError("delete oplog failed"sv); - return 1; - } - } - - return 0; -} - -/////////////////////////////////////// - -ProjectInfoCommand::ProjectInfoCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), ""); - m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), ""); - m_Options.parse_positional({"project", "oplog"}); - m_Options.positional_help("[ []]"); -} - -ProjectInfoCommand::~ProjectInfoCommand() -{ -} - -int -ProjectInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - if (!m_OplogName.empty() && m_ProjectName.empty()) - { - throw OptionParseException("an oplog can't be specified without also specifying a project"); - } - - HttpClient Http(m_HostName); - - std::string Url; - if (m_ProjectName.empty()) - { - Url = "/prj"; - ZEN_CONSOLE("Info from '{}'", Url); - } - else if (m_OplogName.empty()) - { - Url = fmt::format("/prj/{}", m_ProjectName); - ZEN_CONSOLE("Info on project '{}' from '{}{}'", m_ProjectName, m_HostName, Url); - } - else - { - Url = fmt::format("/prj/{}/oplog/{}", m_ProjectName, m_OplogName); - ZEN_CONSOLE("Info on oplog '{}/{}' from '{}{}'", m_ProjectName, m_OplogName, m_HostName, Url); - } - - if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) - { - ZEN_CONSOLE("{}", Result.ToText()); - } - else - { - Result.ThrowError("failed to fetch info"sv); - return 1; - } - return 1; -} - -/////////////////////////////////////// - -CreateProjectCommand::CreateProjectCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectId), ""); - m_Options.add_option("", "", "rootdir", "Absolute path to root directory", cxxopts::value(m_RootDir), ""); - m_Options.add_option("", "", "enginedir", "Absolute path to engine root directory", cxxopts::value(m_EngineRootDir), ""); - m_Options.add_option("", "", "projectdir", "Absolute path to project directory", cxxopts::value(m_ProjectRootDir), ""); - m_Options.add_option("", "", "projectfile", "Absolute path to .uproject file", cxxopts::value(m_ProjectFile), ""); - m_Options.add_option("", "f", "force-update", "Force update of existing project", cxxopts::value(m_ForceUpdate), ""); - m_Options.parse_positional({"project", "rootdir", "enginedir", "projectdir", "projectfile"}); -} - -CreateProjectCommand::~CreateProjectCommand() = default; - -int -CreateProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - using namespace std::literals; - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - if (m_ProjectId.empty()) - { - ZEN_ERROR("Project name must be given"); - return 1; - } - - HttpClient Http(m_HostName); - - std::string Url = fmt::format("/prj/{}", m_ProjectId); - - if (!m_ForceUpdate) - { - if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) - { - ZEN_CONSOLE("Project already exists.\n{}", Result.ToText()); - return 1; - } - } - - IoBuffer Payload = MakeCbObjectPayload([&](CbObjectWriter& Writer) { - Writer.AddString("id"sv, m_ProjectId); - Writer.AddString("root"sv, m_RootDir); - Writer.AddString("engine"sv, m_EngineRootDir); - Writer.AddString("project"sv, m_ProjectRootDir); - Writer.AddString("projectfile"sv, m_ProjectFile); - }); - if (HttpClient::Response Result = m_ForceUpdate ? Http.Put(Url, Payload, HttpClient::Accept(ZenContentType::kText)) - : Http.Post(Url, Payload, HttpClient::Accept(ZenContentType::kText))) - { - ZEN_CONSOLE("{}", Result); - return 0; - } - else - { - Result.ThrowError("failed to create project"sv); - return 1; - } -} - -/////////////////////////////////////// - -DeleteProjectCommand::DeleteProjectCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectId), ""); -} - -DeleteProjectCommand::~DeleteProjectCommand() = default; - -int -DeleteProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - using namespace std::literals; - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - if (m_ProjectId.empty()) - { - ZEN_ERROR("Project name must be given"); - return 1; - } - - HttpClient Http(m_HostName); - - std::string Url = fmt::format("/prj/{}", m_ProjectId); - - if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON)); !Result) - { - Result.ThrowError("failed deleting project"sv); - return 1; - } - - if (HttpClient::Response Result = Http.Delete(Url, HttpClient::Accept(ZenContentType::kText))) - { - ZEN_CONSOLE("{}", Result); - return 0; - } - else - { - Result.ThrowError("failed deleting project"sv); - return 1; - } -} - -/////////////////////////////////////// - -CreateOplogCommand::CreateOplogCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectId), ""); - m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogId), ""); - m_Options.add_option("", "", "gcpath", "Absolute path to oplog lifetime marker file", cxxopts::value(m_GcPath), ""); - m_Options.add_option("", "f", "force-update", "Force update of existing oplog", cxxopts::value(m_ForceUpdate), ""); - m_Options.parse_positional({"project", "oplog", "gcpath"}); -} - -CreateOplogCommand::~CreateOplogCommand() = default; - -int -CreateOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - using namespace std::literals; - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - if (m_ProjectId.empty()) - { - throw OptionParseException("project name must be specified"); - } - - if (m_OplogId.empty()) - { - throw OptionParseException("oplog name must be specified"); - } - - HttpClient Http(m_HostName); - - std::string Url = fmt::format("/prj/{}/oplog/{}", m_ProjectId, m_OplogId); - if (!m_ForceUpdate) - { - if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) - { - ZEN_CONSOLE("Oplog already exists.\n{}", Result.ToText()); - return 1; - } - } - - IoBuffer OplogPayload; - if (!m_GcPath.empty()) - { - OplogPayload = MakeCbObjectPayload([&](CbObjectWriter& Writer) { Writer.AddString("gcpath"sv, m_GcPath); }); - } - - if (HttpClient::Response Result = m_ForceUpdate ? Http.Put(Url, OplogPayload, HttpClient::Accept(ZenContentType::kText)) - : Http.Post(Url, OplogPayload, HttpClient::Accept(ZenContentType::kText))) - { - ZEN_CONSOLE("{}", Result); - return 0; - } - else - { - Result.ThrowError("failed to create oplog"sv); - return 1; - } -} - -/////////////////////////////////////// - -DeleteOplogCommand::DeleteOplogCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectId), ""); - m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogId), ""); - m_Options.parse_positional({"project", "oplog", "gcpath"}); -} - -DeleteOplogCommand::~DeleteOplogCommand() = default; - -int -DeleteOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - using namespace std::literals; - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - if (m_ProjectId.empty()) - { - throw OptionParseException("project name must be specified"); - } - - if (m_OplogId.empty()) - { - throw OptionParseException("oplog name must be specified"); - } - - HttpClient Http(m_HostName); - std::string Url = fmt::format("/prj/{}/oplog/{}", m_ProjectId, m_OplogId); - - if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON)); !Result) - { - Result.ThrowError("failed deleting oplog"sv); - return 1; - } - - if (HttpClient::Response Result = Http.Delete(Url, HttpClient::Accept(ZenContentType::kText))) - { - ZEN_CONSOLE("{}", Result); - return 0; - } - else - { - Result.ThrowError("failed deleting oplog"sv); - return 1; - } -} - -/////////////////////////////////////// - -ExportOplogCommand::ExportOplogCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), ""); - m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), ""); - m_Options.add_option("", "", "maxblocksize", "Max size for bundled attachments", cxxopts::value(m_MaxBlockSize), ""); - m_Options.add_option("", - "", - "maxchunkembedsize", - "Max size for attachment to be bundled", - cxxopts::value(m_MaxChunkEmbedSize), - ""); - m_Options.add_option("", - "", - "embedloosefiles", - "Export additional files referenced by path as attachments", - cxxopts::value(m_EmbedLooseFiles), - ""); - m_Options.add_option("", "f", "force", "Force export of all attachments", cxxopts::value(m_Force), ""); - m_Options.add_option("", - "", - "disableblocks", - "Disable block creation and save all attachments individually (applies to file and cloud target)", - cxxopts::value(m_DisableBlocks), - ""); - m_Options.add_option("", "a", "async", "Trigger export but don't wait for completion", cxxopts::value(m_Async), ""); - - m_Options.add_option("", "", "cloud", "Cloud Storage URL", cxxopts::value(m_CloudUrl), ""); - m_Options.add_option("cloud", "", "namespace", "Cloud Storage namespace", cxxopts::value(m_CloudNamespace), ""); - m_Options.add_option("cloud", "", "bucket", "Cloud Storage bucket", cxxopts::value(m_CloudBucket), ""); - m_Options.add_option("cloud", "", "key", "Cloud Storage key", cxxopts::value(m_CloudKey), ""); - m_Options.add_option("cloud", - "", - "basekey", - "Optional Base Cloud Storage key for incremental export", - cxxopts::value(m_BaseCloudKey), - ""); - m_Options - .add_option("cloud", "", "openid-provider", "Cloud Storage openid provider", cxxopts::value(m_CloudOpenIdProvider), ""); - m_Options.add_option("cloud", "", "access-token", "Cloud Storage access token", cxxopts::value(m_CloudAccessToken), ""); - m_Options.add_option("cloud", - "", - "access-token-env", - "Name of environment variable that holds the cloud Storage access token", - cxxopts::value(m_CloudAccessTokenEnv)->default_value(DefaultCloudAccessTokenEnvVariableName), - ""); - m_Options.add_option("cloud", - "", - "assume-http2", - "Assume that the cloud endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake", - cxxopts::value(m_CloudAssumeHttp2), - ""); - m_Options.add_option("cloud", - "", - "disabletempblocks", - "Disable temp block creation and upload blocks without waiting for oplog container to be uploaded", - cxxopts::value(m_CloudDisableTempBlocks), - ""); - - m_Options.add_option("", "", "zen", "Zen service upload address", cxxopts::value(m_ZenUrl), ""); - m_Options.add_option("zen", "", "target-project", "Zen target project name", cxxopts::value(m_ZenProjectName), ""); - m_Options.add_option("zen", "", "target-oplog", "Zen target oplog name", cxxopts::value(m_ZenOplogName), ""); - m_Options.add_option("zen", "", "clean", "Delete existing target Zen oplog", cxxopts::value(m_ZenClean), ""); - - m_Options.add_option("", "", "file", "Local folder path", cxxopts::value(m_FileDirectoryPath), ""); - m_Options.add_option("file", "", "name", "Local file name", cxxopts::value(m_FileName), ""); - m_Options.add_option("file", - "", - "basename", - "Local base file name for incremental oplog export", - cxxopts::value(m_BaseFileName), - ""); - m_Options.add_option("file", - "", - "forcetempblocks", - "Force creation of temp attachment blocks", - cxxopts::value(m_FileForceEnableTempBlocks), - ""); - - m_Options.parse_positional({"project", "oplog"}); -} - -ExportOplogCommand::~ExportOplogCommand() -{ -} - -int -ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - using namespace std::literals; - - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - if (m_ProjectName.empty()) - { - throw OptionParseException("project name must be specified"); - } - - if (m_OplogName.empty()) - { - throw OptionParseException("oplog identifier must be specified"); - } - - size_t TargetCount = 0; - TargetCount += m_CloudUrl.empty() ? 0 : 1; - TargetCount += m_ZenUrl.empty() ? 0 : 1; - TargetCount += m_FileDirectoryPath.empty() ? 0 : 1; - if (TargetCount != 1) - { - if (TargetCount == 0) - { - throw OptionParseException("an export target must be specified"); - } - else - { - throw OptionParseException("a single export target must be specified"); - } - } - - if (!m_CloudUrl.empty()) - { - if (m_CloudNamespace.empty() || m_CloudBucket.empty()) - { - ZEN_ERROR("Options for cloud target are missing"); - ZEN_CONSOLE("{}", m_Options.help({"cloud"}).c_str()); - return 1; - } - if (m_CloudKey.empty()) - { - std::string KeyString = fmt::format("{}/{}/{}/{}", m_ProjectName, m_OplogName, m_CloudNamespace, m_CloudBucket); - IoHash Key = IoHash::HashBuffer(KeyString.data(), KeyString.size()); - m_CloudKey = Key.ToHexString(); - ZEN_WARN("Using auto generated cloud key '{}'", m_CloudKey); - } - } - - if (!m_ZenUrl.empty()) - { - if (m_ZenProjectName.empty()) - { - m_ZenProjectName = m_ProjectName; - ZEN_WARN("Using default zen target project id '{}'", m_ZenProjectName); - } - if (m_ZenOplogName.empty()) - { - m_ZenOplogName = m_OplogName; - ZEN_WARN("Using default zen target oplog id '{}'", m_ZenOplogName); - } - - std::string TargetUrlBase = m_ZenUrl; - if (TargetUrlBase.find("://") == std::string::npos) - { - // Assume https URL - TargetUrlBase = fmt::format("http://{}", TargetUrlBase); - } - - HttpClient Http(TargetUrlBase); - std::string Url = fmt::format("/prj/{}/oplog/{}", m_ZenProjectName, m_ZenOplogName); - - bool CreateOplog = false; - if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) - { - if (m_ZenClean) - { - ZEN_WARN("Deleting zen remote oplog '{}/{}'", m_ZenProjectName, m_ZenOplogName) - Result = Http.Delete(Url, HttpClient::Accept(ZenContentType::kJSON)); - if (!Result) - { - Result.ThrowError("failed deleting existing zen remote oplog"sv); - return 1; - } - CreateOplog = true; - } - } - else if (Result.StatusCode == HttpResponseCode::NotFound) - { - CreateOplog = true; - } - else - { - Result.ThrowError("failed checking zen remote oplog"sv); - return 1; - } - - if (CreateOplog) - { - ZEN_WARN("Creating zen remote oplog '{}/{}'", m_ZenProjectName, m_ZenOplogName); - if (HttpClient::Response Result = Http.Post(Url); !Result) - { - Result.ThrowError("failed creating zen remote oplog"sv); - return 1; - } - } - } - - if (!m_FileDirectoryPath.empty()) - { - if (m_FileName.empty()) - { - m_FileName = m_OplogName; - ZEN_WARN("Using default file name '{}'", m_FileName); - } - } - - std::string TargetDescription; - - IoBuffer Payload = MakeCbObjectPayload([&](CbObjectWriter& Writer) { - Writer.AddString("method"sv, "export"sv); - Writer.BeginObject("params"sv); - { - if (m_MaxBlockSize != 0) - { - Writer.AddInteger("maxblocksize"sv, m_MaxBlockSize); - } - if (m_MaxChunkEmbedSize != 0) - { - Writer.AddInteger("maxchunkembedsize"sv, m_MaxChunkEmbedSize); - } - if (m_EmbedLooseFiles) - { - Writer.AddBool("embedloosefiles"sv, true); - } - if (m_Force) - { - Writer.AddBool("force"sv, true); - } - Writer.AddBool("async"sv, true); - if (!m_FileDirectoryPath.empty()) - { - Writer.BeginObject("file"sv); - { - Writer.AddString("path"sv, m_FileDirectoryPath); - Writer.AddString("name"sv, m_FileName); - if (!m_BaseFileName.empty()) - { - Writer.AddString("basename"sv, m_BaseFileName); - } - if (m_DisableBlocks) - { - Writer.AddBool("disableblocks"sv, true); - } - if (m_FileForceEnableTempBlocks) - { - Writer.AddBool("enabletempblocks"sv, true); - } - } - Writer.EndObject(); // "file" - TargetDescription = fmt::format("[file] {}/{}{}{}", - m_FileDirectoryPath, - m_FileName, - m_BaseFileName.empty() ? "" : " Base: ", - m_BaseFileName); - } - if (!m_CloudUrl.empty()) - { - Writer.BeginObject("cloud"sv); - { - Writer.AddString("url"sv, m_CloudUrl); - Writer.AddString("namespace"sv, m_CloudNamespace); - Writer.AddString("bucket"sv, m_CloudBucket); - Writer.AddString("key"sv, m_CloudKey); - if (!m_BaseCloudKey.empty()) - { - Writer.AddString("basekey"sv, m_BaseCloudKey); - } - if (!m_CloudOpenIdProvider.empty()) - { - Writer.AddString("openid-provider"sv, m_CloudOpenIdProvider); - } - if (!m_CloudAccessToken.empty()) - { - Writer.AddString("access-token"sv, m_CloudAccessToken); - } - if (!m_CloudAccessTokenEnv.empty()) - { - std::string ResolvedCloudAccessTokenEnv = GetEnvVariable(m_CloudAccessTokenEnv); - - if (!ResolvedCloudAccessTokenEnv.empty()) - { - Writer.AddString("access-token"sv, ResolvedCloudAccessTokenEnv); - } - else - { - Writer.AddString("access-token-env"sv, m_CloudAccessTokenEnv); - } - } - if (m_CloudAssumeHttp2) - { - Writer.AddBool("assumehttp2"sv, true); - } - if (m_DisableBlocks) - { - Writer.AddBool("disableblocks"sv, true); - } - if (m_CloudDisableTempBlocks) - { - Writer.AddBool("disabletempblocks"sv, true); - } - } - Writer.EndObject(); // "cloud" - TargetDescription = fmt::format("[cloud] {}/{}/{}/{}{}{}", - m_CloudUrl, - m_CloudNamespace, - m_CloudBucket, - m_CloudKey, - m_BaseCloudKey.empty() ? "" : " Base: ", - m_BaseCloudKey); - } - if (!m_ZenUrl.empty()) - { - Writer.BeginObject("zen"sv); - { - Writer.AddString("url"sv, m_ZenUrl); - Writer.AddString("project"sv, m_ZenProjectName); - Writer.AddString("oplog"sv, m_ZenOplogName); - } - Writer.EndObject(); // "zen" - - TargetDescription = fmt::format("[zen] {}/{}/{}", m_ZenUrl, m_ZenProjectName, m_ZenOplogName); - } - } - Writer.EndObject(); // "params" - }); - - ZEN_CONSOLE("Saving oplog '{}/{}' from '{}' to {}", m_ProjectName, m_OplogName, m_HostName, TargetDescription); - - HttpClient Http(m_HostName); - if (m_Async) - { - if (HttpClient::Response Result = Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), - std::move(Payload), - HttpClient::Accept(ZenContentType::kJSON)); - Result) - { - ZEN_CONSOLE("{}", Result.AsText()); - } - else - { - Result.ThrowError("failed requesting loading oplog export"sv); - return 1; - } - } - else - { - AsyncPost(Http, fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), std::move(Payload)); - } - return 0; -} - -//////////////////////////// - -ImportOplogCommand::ImportOplogCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), ""); - m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), ""); - m_Options.add_option("", "", "maxblocksize", "Max size for bundled attachments", cxxopts::value(m_MaxBlockSize), ""); - m_Options.add_option("", - "", - "maxchunkembedsize", - "Max size for attachment to be bundled", - cxxopts::value(m_MaxChunkEmbedSize), - ""); - m_Options.add_option("", "f", "force", "Force import of all attachments", cxxopts::value(m_Force), ""); - m_Options.add_option("", "a", "async", "Trigger import but don't wait for completion", cxxopts::value(m_Async), ""); - - m_Options.add_option("", "", "cloud", "Cloud Storage URL", cxxopts::value(m_CloudUrl), ""); - m_Options.add_option("cloud", "", "namespace", "Cloud Storage namespace", cxxopts::value(m_CloudNamespace), ""); - m_Options.add_option("cloud", "", "bucket", "Cloud Storage bucket", cxxopts::value(m_CloudBucket), ""); - m_Options.add_option("cloud", "", "key", "Cloud Storage key", cxxopts::value(m_CloudKey), ""); - m_Options - .add_option("cloud", "", "openid-provider", "Cloud Storage openid provider", cxxopts::value(m_CloudOpenIdProvider), ""); - m_Options.add_option("cloud", "", "access-token", "Cloud Storage access token", cxxopts::value(m_CloudAccessToken), ""); - m_Options.add_option("cloud", - "", - "access-token-env", - "Name of environment variable that holds the cloud Storage access token", - cxxopts::value(m_CloudAccessTokenEnv)->default_value(DefaultCloudAccessTokenEnvVariableName), - ""); - m_Options.add_option("cloud", - "", - "assume-http2", - "Assume that the cloud endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake", - cxxopts::value(m_CloudAssumeHttp2), - ""); - - m_Options.add_option("", "", "zen", "Zen service upload address", cxxopts::value(m_ZenUrl), ""); - m_Options.add_option("zen", "", "source-project", "Zen source project name", cxxopts::value(m_ZenProjectName), ""); - m_Options.add_option("zen", "", "source-oplog", "Zen source oplog name", cxxopts::value(m_ZenOplogName), ""); - m_Options.add_option("zen", "", "clean", "Delete existing target Zen oplog", cxxopts::value(m_ZenClean), ""); - - m_Options.add_option("", "", "file", "Local folder path", cxxopts::value(m_FileDirectoryPath), ""); - m_Options.add_option("file", "", "name", "Local file name", cxxopts::value(m_FileName), ""); - - m_Options.parse_positional({"project", "oplog"}); -} - -ImportOplogCommand::~ImportOplogCommand() -{ -} - -int -ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - using namespace std::literals; - - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - if (m_ProjectName.empty()) - { - ZEN_ERROR("Project name must be given"); - return 1; - } - - if (m_OplogName.empty()) - { - ZEN_ERROR("Oplog name must be given"); - return 1; - } - - size_t TargetCount = 0; - TargetCount += m_CloudUrl.empty() ? 0 : 1; - TargetCount += m_ZenUrl.empty() ? 0 : 1; - TargetCount += m_FileDirectoryPath.empty() ? 0 : 1; - if (TargetCount != 1) - { - ZEN_ERROR("Provide one source only"); - ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); - return 1; - } - - if (!m_CloudUrl.empty()) - { - if (m_CloudNamespace.empty() || m_CloudBucket.empty()) - { - ZEN_ERROR("Options for cloud source are missing"); - ZEN_CONSOLE("{}", m_Options.help({"cloud"}).c_str()); - return 1; - } - if (m_CloudKey.empty()) - { - std::string KeyString = fmt::format("{}/{}/{}/{}", m_ProjectName, m_OplogName, m_CloudNamespace, m_CloudBucket); - IoHash Key = IoHash::HashBuffer(KeyString.data(), KeyString.size()); - m_CloudKey = Key.ToHexString(); - ZEN_WARN("Using auto generated cloud key '{}'", m_CloudKey); - } - } - - if (!m_ZenUrl.empty()) - { - if (m_ZenProjectName.empty()) - { - m_ZenProjectName = m_ProjectName; - ZEN_WARN("Using default zen target project id '{}'", m_ZenProjectName); - } - if (m_ZenOplogName.empty()) - { - m_ZenOplogName = m_OplogName; - ZEN_WARN("Using default zen target oplog id '{}'", m_ZenOplogName); - } - } - - if (!m_FileDirectoryPath.empty()) - { - if (m_FileName.empty()) - { - m_FileName = m_OplogName; - ZEN_WARN("Using auto generated file name '{}'", m_FileName); - } - } - - HttpClient Http(m_HostName); - std::string Url = fmt::format("/prj/{}/oplog/{}", m_ProjectName, m_OplogName); - - bool CreateOplog = false; - if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) - { - if (m_ZenClean) - { - ZEN_WARN("Deleting oplog '{}/{}'", m_ProjectName, m_OplogName) - Result = Http.Delete(Url, HttpClient::Accept(ZenContentType::kJSON)); - if (!Result) - { - Result.ThrowError("failed deleting existing oplog"sv); - return 1; - } - CreateOplog = true; - } - } - else if (Result.StatusCode == HttpResponseCode::NotFound) - { - CreateOplog = true; - } - else - { - Result.ThrowError("failed checking oplog"sv); - return 1; - } - - if (CreateOplog) - { - ZEN_WARN("Creating oplog '{}/{}'", m_ProjectName, m_OplogName); - if (HttpClient::Response Result = Http.Post(Url); !Result) - { - Result.ThrowError("failed creating oplog"sv); - return 1; - } - } - - std::string SourceDescription; - - IoBuffer Payload = MakeCbObjectPayload([&](CbObjectWriter& Writer) { - Writer.AddString("method"sv, "import"sv); - Writer.BeginObject("params"sv); - { - if (m_Force) - { - Writer.AddBool("force"sv, true); - } - if (!m_FileDirectoryPath.empty()) - { - Writer.BeginObject("file"sv); - { - Writer.AddString("path"sv, m_FileDirectoryPath); - Writer.AddString("name"sv, m_FileName); - } - Writer.EndObject(); // "file" - SourceDescription = fmt::format("[file] {}/{}", m_FileDirectoryPath, m_FileName); - } - if (!m_CloudUrl.empty()) - { - Writer.BeginObject("cloud"sv); - { - Writer.AddString("url"sv, m_CloudUrl); - Writer.AddString("namespace"sv, m_CloudNamespace); - Writer.AddString("bucket"sv, m_CloudBucket); - Writer.AddString("key"sv, m_CloudKey); - if (!m_CloudOpenIdProvider.empty()) - { - Writer.AddString("openid-provider"sv, m_CloudOpenIdProvider); - } - if (!m_CloudAccessToken.empty()) - { - Writer.AddString("access-token"sv, m_CloudAccessToken); - } - if (!m_CloudAccessTokenEnv.empty()) - { - std::string ResolvedCloudAccessTokenEnv = GetEnvVariable(m_CloudAccessTokenEnv); - - if (!ResolvedCloudAccessTokenEnv.empty()) - { - Writer.AddString("access-token"sv, ResolvedCloudAccessTokenEnv); - } - else - { - Writer.AddString("access-token-env"sv, m_CloudAccessTokenEnv); - } - } - if (m_CloudAssumeHttp2) - { - Writer.AddBool("assumehttp2"sv, true); - } - } - Writer.EndObject(); // "cloud" - SourceDescription = fmt::format("[cloud] {}/{}/{}/{}", m_CloudUrl, m_CloudNamespace, m_CloudBucket, m_CloudKey); - } - if (!m_ZenUrl.empty()) - { - Writer.BeginObject("zen"sv); - { - Writer.AddString("url"sv, m_ZenUrl); - Writer.AddString("project"sv, m_ZenProjectName); - Writer.AddString("oplog"sv, m_ZenOplogName); - } - Writer.EndObject(); // "zen" - SourceDescription = fmt::format("[zen] {}", m_ZenUrl); - } - } - Writer.EndObject(); // "params" - }); - - ZEN_CONSOLE("Loading oplog '{}/{}' from '{}' to {}", m_ProjectName, m_OplogName, SourceDescription, m_HostName); - - if (m_Async) - { - if (HttpClient::Response Result = Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), - std::move(Payload), - HttpClient::Accept(ZenContentType::kJSON)); - Result) - { - ZEN_CONSOLE("{}", Result.AsText()); - } - else - { - Result.ThrowError("failed requesting loading oplog import"sv); - return 1; - } - } - else - { - AsyncPost(Http, fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), std::move(Payload)); - } - return 0; -} - -//////////////////////////// - -SnapshotOplogCommand::SnapshotOplogCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), ""); - m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), ""); - - m_Options.parse_positional({"project", "oplog"}); -} - -SnapshotOplogCommand::~SnapshotOplogCommand() -{ -} - -int -SnapshotOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - using namespace std::literals; - - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - if (m_ProjectName.empty()) - { - ZEN_ERROR("Project name must be given"); - return 1; - } - - if (m_OplogName.empty()) - { - ZEN_ERROR("Oplog name must be given"); - return 1; - } - - IoBuffer Payload = MakeCbObjectPayload([&](CbObjectWriter& Writer) { Writer.AddString("method"sv, "snapshot"sv); }); - - HttpClient Http(m_HostName); - - ZEN_CONSOLE("Snapshotting oplog '{}/{}' to {}", m_ProjectName, m_OplogName, m_HostName); - if (HttpClient::Response Result = - Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), Payload, HttpClient::Accept(ZenContentType::kJSON))) - { - ZEN_CONSOLE("{}", Result); - return 0; - } - else - { - Result.ThrowError("failed to create project"sv); - return 1; - } -} - -//////////////////////////// - -ProjectStatsCommand::ProjectStatsCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); -} - -ProjectStatsCommand::~ProjectStatsCommand() -{ -} - -int -ProjectStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - HttpClient Http(m_HostName); - if (HttpClient::Response Result = Http.Get("/stats/prj", HttpClient::Accept(ZenContentType::kJSON))) - { - ZEN_CONSOLE("{}", Result.AsText()); - return 0; - } - else - { - Result.ThrowError("failed to get project stats"sv); - return 1; - } -} - -//////////////////////////// - -ProjectDetailsCommand::ProjectDetailsCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "c", "csv", "Output in CSV format (default is JSon)", cxxopts::value(m_CSV), ""); - m_Options.add_option("", "d", "details", "Detailed info on oplog", cxxopts::value(m_Details), "
"); - m_Options.add_option("", "o", "opdetails", "Details info on oplog body", cxxopts::value(m_OpDetails), ""); - m_Options.add_option("", "p", "project", "Project name to get info from", cxxopts::value(m_ProjectName), ""); - m_Options.add_option("", "l", "oplog", "Oplog name to get info from", cxxopts::value(m_OplogName), ""); - m_Options.add_option("", "i", "opid", "Oid of a specific op info for", cxxopts::value(m_OpId), ""); - m_Options.add_option("", - "a", - "attachmentdetails", - "Get detailed information about attachments", - cxxopts::value(m_AttachmentDetails), - ""); -} - -ProjectDetailsCommand::~ProjectDetailsCommand() -{ -} - -int -ProjectDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - if (!m_OpId.empty()) - { - if (m_ProjectName.empty() || m_OplogName.empty()) - { - ZEN_ERROR("Provide project and oplog name"); - ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); - return 1; - } - } - else if (!m_OplogName.empty()) - { - if (m_ProjectName.empty()) - { - ZEN_ERROR("Provide project name"); - ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); - return 1; - } - } - - HttpClient Http(m_HostName); - - ExtendableStringBuilder<128> Url; - Url.Append("/prj/details$"); - if (!m_ProjectName.empty()) - { - Url.Append("/"); - Url.Append(m_ProjectName); - } - if (!m_OplogName.empty()) - { - Url.Append("/"); - Url.Append(m_OplogName); - } - if (!m_OpId.empty()) - { - Url.Append("/"); - Url.Append(m_OpId); - } - - if (HttpClient::Response Result = - Http.Get(Url, - m_CSV ? HttpClient::Accept(ZenContentType::kText) : HttpClient::Accept(ZenContentType::kJSON), - {{"opdetails", m_OpDetails ? "true" : "false"}, - {"details", m_Details ? "true" : "false"}, - {"attachmentdetails", m_AttachmentDetails ? "true" : "false"}, - {"csv", m_CSV ? "true" : "false"}})) - { - ZEN_CONSOLE("{}", Result.AsText()); - return 0; - } - else - { - Result.ThrowError("failed to get project details"sv); - return 1; - } -} - -//////////////////////////// - -OplogMirrorCommand::OplogMirrorCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "p", "project", "Project name to get info from", cxxopts::value(m_ProjectName), ""); - m_Options.add_option("", "l", "oplog", "Oplog name to get info from", cxxopts::value(m_OplogName), ""); - m_Options.add_option("", "t", "target", "Target directory for mirror", cxxopts::value(m_MirrorRootPath), ""); - - m_Options.parse_positional({"project", "oplog", "target"}); - m_Options.positional_help("[ ]"); -} - -OplogMirrorCommand::~OplogMirrorCommand() -{ -} - -int -OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - if (m_ProjectName.empty()) - { - throw OptionParseException("a project must be specified"); - } - - if (m_OplogName.empty()) - { - throw OptionParseException("an oplog must be specified"); - } - - if (m_MirrorRootPath.empty()) - { - throw OptionParseException("a target path must be specified"); - } - - ZEN_CONSOLE("Emitting file data from oplog '{}/{}' to '{}'", m_ProjectName, m_OplogName, m_MirrorRootPath); - - HttpClient Http(m_HostName); - - if (HttpClient::Response Result = Http.Get(fmt::format("/prj/{}/oplog/{}", m_ProjectName, m_OplogName))) - { - // The info requested is not really used at this moment, we just use the probe to be able to provide - // better diagnostics up front - } - else - { - Result.ThrowError("oplog info fetch failed"sv); - - return 1; - } - - // Emit file data to target directory - - std::filesystem::path RootPath{m_MirrorRootPath}; - CreateDirectories(RootPath); - - std::filesystem::path TmpPath = RootPath / ".tmp"; - CreateDirectories(TmpPath); - - std::atomic_int64_t FileCount = 0; - int OplogEntryCount = 0; - - size_t WorkerCount = Min(std::thread::hardware_concurrency(), 16u); - WorkerThreadPool WorkerPool(gsl::narrow(WorkerCount)); - Latch WorkRemaining(1); - - std::unordered_set FileNames; - - auto EmitFilesForDataArray = [&](CbArrayView DataArray) { - for (auto DataIter : DataArray) - { - if (CbObjectView Data = DataIter.AsObjectView()) - { - std::string FileName = std::string(Data["filename"sv].AsString()); - Oid ChunkId = Data["id"sv].AsObjectId(); - if (!FileNames.insert(FileName).second) - { - continue; - } - WorkRemaining.AddCount(1); - WorkerPool.ScheduleWork([this, &RootPath, FileName, &FileCount, ChunkId, &Http, TmpPath, &WorkRemaining]() { - auto _ = MakeGuard([&WorkRemaining]() { WorkRemaining.CountDown(); }); - if (HttpClient::Response ChunkResponse = - Http.Download(fmt::format("/prj/{}/oplog/{}/{}"sv, m_ProjectName, m_OplogName, ChunkId), TmpPath)) - { - IoBuffer ChunkData = ChunkResponse.ResponsePayload; - std::filesystem::path TargetPath = RootPath / FileName; - if (!MoveToFile(TargetPath, ChunkData)) - { - WriteFile(TargetPath, ChunkData); - } - ++FileCount; - } - else - { - ZEN_CONSOLE("Unable to fetch '{}' (chunk {}). Reason: '{}'", FileName, ChunkId, ChunkResponse.ErrorMessage(""sv)); - } - }); - } - } - }; - - if (HttpClient::Response Response = Http.Get(fmt::format("/prj/{}/oplog/{}/entries"sv, m_ProjectName, m_OplogName))) - { - if (CbObject ResponseObject = Response.AsObject()) - { - for (auto EntryIter : ResponseObject["entries"sv]) - { - CbObjectView Entry = EntryIter.AsObjectView(); - - EmitFilesForDataArray(Entry["packagedata"sv].AsArrayView()); - EmitFilesForDataArray(Entry["bulkdata"sv].AsArrayView()); - - ++OplogEntryCount; - } - } - else - { - ZEN_ERROR("unknown format response to oplog entries request"); - } - } - else - { - Response.ThrowError("oplog entries fetch failed"); - - return 1; - } - WorkRemaining.CountDown(); - WorkRemaining.Wait(); - - std::filesystem::remove_all(TmpPath); - - ZEN_CONSOLE("mirrored {} files from {} oplog entries successfully", FileCount.load(), OplogEntryCount); - - return 0; -} - -} // namespace zen diff --git a/src/zen/cmds/projectstore.h b/src/zen/cmds/projectstore.h deleted file mode 100644 index fd1590423..000000000 --- a/src/zen/cmds/projectstore.h +++ /dev/null @@ -1,256 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -class DropProjectCommand : public ZenCmdBase -{ -public: - DropProjectCommand(); - ~DropProjectCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"project-drop", "Drop project or project oplog"}; - std::string m_HostName; - std::string m_ProjectName; - std::string m_OplogName; -}; - -class ProjectInfoCommand : public ZenCmdBase -{ -public: - ProjectInfoCommand(); - ~ProjectInfoCommand(); - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"project-info", "Info on project or project oplog"}; - std::string m_HostName; - std::string m_ProjectName; - std::string m_OplogName; -}; - -class CreateProjectCommand : public ZenCmdBase -{ -public: - CreateProjectCommand(); - ~CreateProjectCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"project-create", "Create project, the project must not already exist."}; - std::string m_HostName; - std::string m_ProjectId; - std::string m_RootDir; - std::string m_EngineRootDir; - std::string m_ProjectRootDir; - std::string m_ProjectFile; - bool m_ForceUpdate = false; -}; - -class DeleteProjectCommand : public ZenCmdBase -{ -public: - DeleteProjectCommand(); - ~DeleteProjectCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"project-delete", "Delete project and all its oplogs"}; - std::string m_HostName; - std::string m_ProjectId; -}; - -class CreateOplogCommand : public ZenCmdBase -{ -public: - CreateOplogCommand(); - ~CreateOplogCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"oplog-create", "Create oplog in an existing project, the oplog must not already exist."}; - std::string m_HostName; - std::string m_ProjectId; - std::string m_OplogId; - std::string m_GcPath; - bool m_ForceUpdate = false; -}; - -class DeleteOplogCommand : public ZenCmdBase -{ -public: - DeleteOplogCommand(); - ~DeleteOplogCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"oplog-delete", "Delete oplog and all its data"}; - std::string m_HostName; - std::string m_ProjectId; - std::string m_OplogId; -}; - -class ExportOplogCommand : public ZenCmdBase -{ -public: - ExportOplogCommand(); - ~ExportOplogCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"oplog-export", - "Export project store oplog to cloud (--cloud), file system (--file) or other Zen instance (--zen)"}; - std::string m_HostName; - std::string m_ProjectName; - std::string m_OplogName; - uint64_t m_MaxBlockSize = 0; - uint64_t m_MaxChunkEmbedSize = 0; - bool m_EmbedLooseFiles = false; - bool m_Force = false; - bool m_DisableBlocks = false; - bool m_Async = false; - - std::string m_CloudUrl; - std::string m_CloudNamespace; - std::string m_CloudBucket; - std::string m_CloudKey; - std::string m_BaseCloudKey; - std::string m_CloudOpenIdProvider; - std::string m_CloudAccessToken; - std::string m_CloudAccessTokenEnv; - bool m_CloudAssumeHttp2 = false; - bool m_CloudDisableTempBlocks = false; - - std::string m_ZenUrl; - std::string m_ZenProjectName; - std::string m_ZenOplogName; - bool m_ZenClean; - - std::string m_FileDirectoryPath; - std::string m_FileName; - std::string m_BaseFileName; - bool m_FileForceEnableTempBlocks = false; -}; - -class ImportOplogCommand : public ZenCmdBase -{ -public: - ImportOplogCommand(); - ~ImportOplogCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"oplog-import", - "Import project store oplog from cloud (--cloud), file system (--file) or other Zen instance (--zen)"}; - std::string m_HostName; - std::string m_ProjectName; - std::string m_OplogName; - size_t m_MaxBlockSize = 0; - size_t m_MaxChunkEmbedSize = 0; - bool m_Force = false; - bool m_Async = false; - - std::string m_CloudUrl; - std::string m_CloudNamespace; - std::string m_CloudBucket; - std::string m_CloudKey; - std::string m_CloudOpenIdProvider; - std::string m_CloudAccessToken; - std::string m_CloudAccessTokenEnv; - bool m_CloudAssumeHttp2 = false; - - std::string m_ZenUrl; - std::string m_ZenProjectName; - std::string m_ZenOplogName; - bool m_ZenClean; - - std::string m_FileDirectoryPath; - std::string m_FileName; -}; - -class SnapshotOplogCommand : public ZenCmdBase -{ -public: - SnapshotOplogCommand(); - ~SnapshotOplogCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"oplog-snapshot", "Snapshot external file references in project store oplog into zen"}; - std::string m_HostName; - std::string m_ProjectName; - std::string m_OplogName; -}; - -class ProjectStatsCommand : public ZenCmdBase -{ -public: - ProjectStatsCommand(); - ~ProjectStatsCommand(); - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"project-stats", "Stats info on project store"}; - std::string m_HostName; -}; - -class ProjectDetailsCommand : public ZenCmdBase -{ -public: - ProjectDetailsCommand(); - ~ProjectDetailsCommand(); - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"project-details", "Detail info on project store"}; - std::string m_HostName; - bool m_Details; - bool m_OpDetails; - bool m_AttachmentDetails; - bool m_CSV; - std::string m_ProjectName; - std::string m_OplogName; - std::string m_OpId; -}; - -class OplogMirrorCommand : public ZenCmdBase -{ -public: - OplogMirrorCommand(); - ~OplogMirrorCommand(); - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"oplog-mirror", "Mirror oplog to file system"}; - std::string m_HostName; - std::string m_ProjectName; - std::string m_OplogName; - std::string m_MirrorRootPath; -}; - -} // namespace zen diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp new file mode 100644 index 000000000..5795b3190 --- /dev/null +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -0,0 +1,1558 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "projectstore_cmd.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#include + +namespace zen { + +namespace { + + using namespace std::literals; + + const std::string DefaultCloudAccessTokenEnvVariableName( +#if ZEN_PLATFORM_WINDOWS + "UE-CloudDataCacheAccessToken"sv +#endif +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + "UE_CloudDataCacheAccessToken"sv +#endif + + ); + + IoBuffer MakeCbObjectPayload(std::function WriteCB) + { + CbObjectWriter Writer; + WriteCB(Writer); + IoBuffer Payload = Writer.Save().GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + return Payload; + }; + + static std::atomic_uint32_t SignalCounter[NSIG] = {0}; + + static void SignalCallbackHandler(int SigNum) + { + if (SigNum >= 0 && SigNum < NSIG) + { + SignalCounter[SigNum].fetch_add(1); + } + } + + void AsyncPost(HttpClient& Http, std::string_view Url, IoBuffer&& Payload) + { + if (HttpClient::Response Result = Http.Post(Url, Payload)) + { + if (Result.StatusCode == HttpResponseCode::Accepted) + { + signal(SIGINT, SignalCallbackHandler); + bool Cancelled = false; + + std::string_view JobIdText = Result.AsText(); + std::optional JobIdMaybe = ParseInt(JobIdText); + if (!JobIdMaybe) + { + Result.ThrowError("invalid job id"sv); + } + + std::string LastCurrentOp; + uint32_t LastCurrentOpPercentComplete = 0; + + uint64_t JobId = JobIdMaybe.value(); + while (true) + { + HttpClient::Response StatusResult = + Http.Get(fmt::format("/admin/jobs/{}", JobId), HttpClient::Accept(ZenContentType::kCbObject)); + if (!StatusResult) + { + StatusResult.ThrowError("failed to create project"sv); + } + CbObject StatusObject = StatusResult.AsObject(); + std::string_view Status = StatusObject["Status"sv].AsString(); + CbArrayView Messages = StatusObject["Messages"sv].AsArrayView(); + for (auto M : Messages) + { + std::string_view Message = M.AsString(); + ZEN_CONSOLE("{}", Message); + } + if (Status == "Complete") + { + if (Cancelled) + { + ZEN_CONSOLE("Cancelled"); + } + else + { + double QueueTimeS = StatusObject["QueueTimeS"].AsDouble(); + double RuntimeS = StatusObject["RunTimeS"].AsDouble(); + ZEN_CONSOLE("Completed: QueueTime: {:.3} s, RunTime: {:.3} s", QueueTimeS, RuntimeS); + } + break; + } + if (Status == "Aborted") + { + Result.ThrowError("Aborted"); + break; + } + if (Status == "Queued") + { + double QueueTimeS = StatusObject["QueueTimeS"].AsDouble(); + ZEN_CONSOLE("Queued, waited {:.3} s...", QueueTimeS); + } + if (Status == "Running") + { + std::string_view CurrentOp = StatusObject["CurrentOp"sv].AsString(); + uint32_t CurrentOpPercentComplete = StatusObject["CurrentOpPercentComplete"sv].AsUInt32(); + if (CurrentOp != LastCurrentOp || CurrentOpPercentComplete != LastCurrentOpPercentComplete) + { + LastCurrentOp = CurrentOp; + LastCurrentOpPercentComplete = CurrentOpPercentComplete; + ZEN_CONSOLE("{} {}%", CurrentOp, CurrentOpPercentComplete); + } + } + uint32_t AbortCounter = SignalCounter[SIGINT].load(); + if (SignalCounter[SIGINT] > 0) + { + SignalCounter[SIGINT].fetch_sub(AbortCounter); + if (HttpClient::Response DeleteResult = Http.Delete(fmt::format("/admin/jobs/{}", JobId))) + { + ZEN_CONSOLE("Requested cancel..."); + Cancelled = true; + } + else + { + ZEN_CONSOLE("Failed cancelling job {}", DeleteResult); + } + continue; + } + Sleep(100); + } + } + else + { + ZEN_CONSOLE("{}", Result); + } + } + else + { + Result.ThrowError("failed to start operation"sv); + } + } + +} // namespace + +/////////////////////////////////////// + +DropProjectCommand::DropProjectCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), ""); + m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), ""); + m_Options.parse_positional({"project", "oplog"}); + m_Options.positional_help("[ []]"); +} + +DropProjectCommand::~DropProjectCommand() +{ +} + +int +DropProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + if (m_ProjectName.empty()) + { + throw OptionParseException("Drop command requires a project"); + } + + HttpClient Http(m_HostName); + if (m_OplogName.empty()) + { + ZEN_CONSOLE("Dropping project '{}' from '{}'", m_ProjectName, m_HostName); + if (HttpClient::Response Result = Http.Delete(fmt::format("/prj/{}", m_ProjectName))) + { + ZEN_CONSOLE("{}", Result); + } + else + { + Result.ThrowError("delete project failed"sv); + return 1; + } + } + else + { + ZEN_CONSOLE("Dropping oplog '{}/{}' from '{}'", m_ProjectName, m_OplogName, m_HostName); + if (HttpClient::Response Result = Http.Delete(fmt::format("/prj/{}/oplog/{}", m_ProjectName, m_OplogName))) + { + ZEN_CONSOLE("{}", Result); + } + else + { + Result.ThrowError("delete oplog failed"sv); + return 1; + } + } + + return 0; +} + +/////////////////////////////////////// + +ProjectInfoCommand::ProjectInfoCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), ""); + m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), ""); + m_Options.parse_positional({"project", "oplog"}); + m_Options.positional_help("[ []]"); +} + +ProjectInfoCommand::~ProjectInfoCommand() +{ +} + +int +ProjectInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + if (!m_OplogName.empty() && m_ProjectName.empty()) + { + throw OptionParseException("an oplog can't be specified without also specifying a project"); + } + + HttpClient Http(m_HostName); + + std::string Url; + if (m_ProjectName.empty()) + { + Url = "/prj"; + ZEN_CONSOLE("Info from '{}'", Url); + } + else if (m_OplogName.empty()) + { + Url = fmt::format("/prj/{}", m_ProjectName); + ZEN_CONSOLE("Info on project '{}' from '{}{}'", m_ProjectName, m_HostName, Url); + } + else + { + Url = fmt::format("/prj/{}/oplog/{}", m_ProjectName, m_OplogName); + ZEN_CONSOLE("Info on oplog '{}/{}' from '{}{}'", m_ProjectName, m_OplogName, m_HostName, Url); + } + + if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) + { + ZEN_CONSOLE("{}", Result.ToText()); + } + else + { + Result.ThrowError("failed to fetch info"sv); + return 1; + } + return 1; +} + +/////////////////////////////////////// + +CreateProjectCommand::CreateProjectCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectId), ""); + m_Options.add_option("", "", "rootdir", "Absolute path to root directory", cxxopts::value(m_RootDir), ""); + m_Options.add_option("", "", "enginedir", "Absolute path to engine root directory", cxxopts::value(m_EngineRootDir), ""); + m_Options.add_option("", "", "projectdir", "Absolute path to project directory", cxxopts::value(m_ProjectRootDir), ""); + m_Options.add_option("", "", "projectfile", "Absolute path to .uproject file", cxxopts::value(m_ProjectFile), ""); + m_Options.add_option("", "f", "force-update", "Force update of existing project", cxxopts::value(m_ForceUpdate), ""); + m_Options.parse_positional({"project", "rootdir", "enginedir", "projectdir", "projectfile"}); +} + +CreateProjectCommand::~CreateProjectCommand() = default; + +int +CreateProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + using namespace std::literals; + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + if (m_ProjectId.empty()) + { + ZEN_ERROR("Project name must be given"); + return 1; + } + + HttpClient Http(m_HostName); + + std::string Url = fmt::format("/prj/{}", m_ProjectId); + + if (!m_ForceUpdate) + { + if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) + { + ZEN_CONSOLE("Project already exists.\n{}", Result.ToText()); + return 1; + } + } + + IoBuffer Payload = MakeCbObjectPayload([&](CbObjectWriter& Writer) { + Writer.AddString("id"sv, m_ProjectId); + Writer.AddString("root"sv, m_RootDir); + Writer.AddString("engine"sv, m_EngineRootDir); + Writer.AddString("project"sv, m_ProjectRootDir); + Writer.AddString("projectfile"sv, m_ProjectFile); + }); + if (HttpClient::Response Result = m_ForceUpdate ? Http.Put(Url, Payload, HttpClient::Accept(ZenContentType::kText)) + : Http.Post(Url, Payload, HttpClient::Accept(ZenContentType::kText))) + { + ZEN_CONSOLE("{}", Result); + return 0; + } + else + { + Result.ThrowError("failed to create project"sv); + return 1; + } +} + +/////////////////////////////////////// + +DeleteProjectCommand::DeleteProjectCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectId), ""); +} + +DeleteProjectCommand::~DeleteProjectCommand() = default; + +int +DeleteProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + using namespace std::literals; + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + if (m_ProjectId.empty()) + { + ZEN_ERROR("Project name must be given"); + return 1; + } + + HttpClient Http(m_HostName); + + std::string Url = fmt::format("/prj/{}", m_ProjectId); + + if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON)); !Result) + { + Result.ThrowError("failed deleting project"sv); + return 1; + } + + if (HttpClient::Response Result = Http.Delete(Url, HttpClient::Accept(ZenContentType::kText))) + { + ZEN_CONSOLE("{}", Result); + return 0; + } + else + { + Result.ThrowError("failed deleting project"sv); + return 1; + } +} + +/////////////////////////////////////// + +CreateOplogCommand::CreateOplogCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectId), ""); + m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogId), ""); + m_Options.add_option("", "", "gcpath", "Absolute path to oplog lifetime marker file", cxxopts::value(m_GcPath), ""); + m_Options.add_option("", "f", "force-update", "Force update of existing oplog", cxxopts::value(m_ForceUpdate), ""); + m_Options.parse_positional({"project", "oplog", "gcpath"}); +} + +CreateOplogCommand::~CreateOplogCommand() = default; + +int +CreateOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + using namespace std::literals; + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + if (m_ProjectId.empty()) + { + throw OptionParseException("project name must be specified"); + } + + if (m_OplogId.empty()) + { + throw OptionParseException("oplog name must be specified"); + } + + HttpClient Http(m_HostName); + + std::string Url = fmt::format("/prj/{}/oplog/{}", m_ProjectId, m_OplogId); + if (!m_ForceUpdate) + { + if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) + { + ZEN_CONSOLE("Oplog already exists.\n{}", Result.ToText()); + return 1; + } + } + + IoBuffer OplogPayload; + if (!m_GcPath.empty()) + { + OplogPayload = MakeCbObjectPayload([&](CbObjectWriter& Writer) { Writer.AddString("gcpath"sv, m_GcPath); }); + } + + if (HttpClient::Response Result = m_ForceUpdate ? Http.Put(Url, OplogPayload, HttpClient::Accept(ZenContentType::kText)) + : Http.Post(Url, OplogPayload, HttpClient::Accept(ZenContentType::kText))) + { + ZEN_CONSOLE("{}", Result); + return 0; + } + else + { + Result.ThrowError("failed to create oplog"sv); + return 1; + } +} + +/////////////////////////////////////// + +DeleteOplogCommand::DeleteOplogCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectId), ""); + m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogId), ""); + m_Options.parse_positional({"project", "oplog", "gcpath"}); +} + +DeleteOplogCommand::~DeleteOplogCommand() = default; + +int +DeleteOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + using namespace std::literals; + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + if (m_ProjectId.empty()) + { + throw OptionParseException("project name must be specified"); + } + + if (m_OplogId.empty()) + { + throw OptionParseException("oplog name must be specified"); + } + + HttpClient Http(m_HostName); + std::string Url = fmt::format("/prj/{}/oplog/{}", m_ProjectId, m_OplogId); + + if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON)); !Result) + { + Result.ThrowError("failed deleting oplog"sv); + return 1; + } + + if (HttpClient::Response Result = Http.Delete(Url, HttpClient::Accept(ZenContentType::kText))) + { + ZEN_CONSOLE("{}", Result); + return 0; + } + else + { + Result.ThrowError("failed deleting oplog"sv); + return 1; + } +} + +/////////////////////////////////////// + +ExportOplogCommand::ExportOplogCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), ""); + m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), ""); + m_Options.add_option("", "", "maxblocksize", "Max size for bundled attachments", cxxopts::value(m_MaxBlockSize), ""); + m_Options.add_option("", + "", + "maxchunkembedsize", + "Max size for attachment to be bundled", + cxxopts::value(m_MaxChunkEmbedSize), + ""); + m_Options.add_option("", + "", + "embedloosefiles", + "Export additional files referenced by path as attachments", + cxxopts::value(m_EmbedLooseFiles), + ""); + m_Options.add_option("", "f", "force", "Force export of all attachments", cxxopts::value(m_Force), ""); + m_Options.add_option("", + "", + "disableblocks", + "Disable block creation and save all attachments individually (applies to file and cloud target)", + cxxopts::value(m_DisableBlocks), + ""); + m_Options.add_option("", "a", "async", "Trigger export but don't wait for completion", cxxopts::value(m_Async), ""); + + m_Options.add_option("", "", "cloud", "Cloud Storage URL", cxxopts::value(m_CloudUrl), ""); + m_Options.add_option("cloud", "", "namespace", "Cloud Storage namespace", cxxopts::value(m_CloudNamespace), ""); + m_Options.add_option("cloud", "", "bucket", "Cloud Storage bucket", cxxopts::value(m_CloudBucket), ""); + m_Options.add_option("cloud", "", "key", "Cloud Storage key", cxxopts::value(m_CloudKey), ""); + m_Options.add_option("cloud", + "", + "basekey", + "Optional Base Cloud Storage key for incremental export", + cxxopts::value(m_BaseCloudKey), + ""); + m_Options + .add_option("cloud", "", "openid-provider", "Cloud Storage openid provider", cxxopts::value(m_CloudOpenIdProvider), ""); + m_Options.add_option("cloud", "", "access-token", "Cloud Storage access token", cxxopts::value(m_CloudAccessToken), ""); + m_Options.add_option("cloud", + "", + "access-token-env", + "Name of environment variable that holds the cloud Storage access token", + cxxopts::value(m_CloudAccessTokenEnv)->default_value(DefaultCloudAccessTokenEnvVariableName), + ""); + m_Options.add_option("cloud", + "", + "assume-http2", + "Assume that the cloud endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake", + cxxopts::value(m_CloudAssumeHttp2), + ""); + m_Options.add_option("cloud", + "", + "disabletempblocks", + "Disable temp block creation and upload blocks without waiting for oplog container to be uploaded", + cxxopts::value(m_CloudDisableTempBlocks), + ""); + + m_Options.add_option("", "", "zen", "Zen service upload address", cxxopts::value(m_ZenUrl), ""); + m_Options.add_option("zen", "", "target-project", "Zen target project name", cxxopts::value(m_ZenProjectName), ""); + m_Options.add_option("zen", "", "target-oplog", "Zen target oplog name", cxxopts::value(m_ZenOplogName), ""); + m_Options.add_option("zen", "", "clean", "Delete existing target Zen oplog", cxxopts::value(m_ZenClean), ""); + + m_Options.add_option("", "", "file", "Local folder path", cxxopts::value(m_FileDirectoryPath), ""); + m_Options.add_option("file", "", "name", "Local file name", cxxopts::value(m_FileName), ""); + m_Options.add_option("file", + "", + "basename", + "Local base file name for incremental oplog export", + cxxopts::value(m_BaseFileName), + ""); + m_Options.add_option("file", + "", + "forcetempblocks", + "Force creation of temp attachment blocks", + cxxopts::value(m_FileForceEnableTempBlocks), + ""); + + m_Options.parse_positional({"project", "oplog"}); +} + +ExportOplogCommand::~ExportOplogCommand() +{ +} + +int +ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + using namespace std::literals; + + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + if (m_ProjectName.empty()) + { + throw OptionParseException("project name must be specified"); + } + + if (m_OplogName.empty()) + { + throw OptionParseException("oplog identifier must be specified"); + } + + size_t TargetCount = 0; + TargetCount += m_CloudUrl.empty() ? 0 : 1; + TargetCount += m_ZenUrl.empty() ? 0 : 1; + TargetCount += m_FileDirectoryPath.empty() ? 0 : 1; + if (TargetCount != 1) + { + if (TargetCount == 0) + { + throw OptionParseException("an export target must be specified"); + } + else + { + throw OptionParseException("a single export target must be specified"); + } + } + + if (!m_CloudUrl.empty()) + { + if (m_CloudNamespace.empty() || m_CloudBucket.empty()) + { + ZEN_ERROR("Options for cloud target are missing"); + ZEN_CONSOLE("{}", m_Options.help({"cloud"}).c_str()); + return 1; + } + if (m_CloudKey.empty()) + { + std::string KeyString = fmt::format("{}/{}/{}/{}", m_ProjectName, m_OplogName, m_CloudNamespace, m_CloudBucket); + IoHash Key = IoHash::HashBuffer(KeyString.data(), KeyString.size()); + m_CloudKey = Key.ToHexString(); + ZEN_WARN("Using auto generated cloud key '{}'", m_CloudKey); + } + } + + if (!m_ZenUrl.empty()) + { + if (m_ZenProjectName.empty()) + { + m_ZenProjectName = m_ProjectName; + ZEN_WARN("Using default zen target project id '{}'", m_ZenProjectName); + } + if (m_ZenOplogName.empty()) + { + m_ZenOplogName = m_OplogName; + ZEN_WARN("Using default zen target oplog id '{}'", m_ZenOplogName); + } + + std::string TargetUrlBase = m_ZenUrl; + if (TargetUrlBase.find("://") == std::string::npos) + { + // Assume https URL + TargetUrlBase = fmt::format("http://{}", TargetUrlBase); + } + + HttpClient Http(TargetUrlBase); + std::string Url = fmt::format("/prj/{}/oplog/{}", m_ZenProjectName, m_ZenOplogName); + + bool CreateOplog = false; + if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) + { + if (m_ZenClean) + { + ZEN_WARN("Deleting zen remote oplog '{}/{}'", m_ZenProjectName, m_ZenOplogName) + Result = Http.Delete(Url, HttpClient::Accept(ZenContentType::kJSON)); + if (!Result) + { + Result.ThrowError("failed deleting existing zen remote oplog"sv); + return 1; + } + CreateOplog = true; + } + } + else if (Result.StatusCode == HttpResponseCode::NotFound) + { + CreateOplog = true; + } + else + { + Result.ThrowError("failed checking zen remote oplog"sv); + return 1; + } + + if (CreateOplog) + { + ZEN_WARN("Creating zen remote oplog '{}/{}'", m_ZenProjectName, m_ZenOplogName); + if (HttpClient::Response Result = Http.Post(Url); !Result) + { + Result.ThrowError("failed creating zen remote oplog"sv); + return 1; + } + } + } + + if (!m_FileDirectoryPath.empty()) + { + if (m_FileName.empty()) + { + m_FileName = m_OplogName; + ZEN_WARN("Using default file name '{}'", m_FileName); + } + } + + std::string TargetDescription; + + IoBuffer Payload = MakeCbObjectPayload([&](CbObjectWriter& Writer) { + Writer.AddString("method"sv, "export"sv); + Writer.BeginObject("params"sv); + { + if (m_MaxBlockSize != 0) + { + Writer.AddInteger("maxblocksize"sv, m_MaxBlockSize); + } + if (m_MaxChunkEmbedSize != 0) + { + Writer.AddInteger("maxchunkembedsize"sv, m_MaxChunkEmbedSize); + } + if (m_EmbedLooseFiles) + { + Writer.AddBool("embedloosefiles"sv, true); + } + if (m_Force) + { + Writer.AddBool("force"sv, true); + } + Writer.AddBool("async"sv, true); + if (!m_FileDirectoryPath.empty()) + { + Writer.BeginObject("file"sv); + { + Writer.AddString("path"sv, m_FileDirectoryPath); + Writer.AddString("name"sv, m_FileName); + if (!m_BaseFileName.empty()) + { + Writer.AddString("basename"sv, m_BaseFileName); + } + if (m_DisableBlocks) + { + Writer.AddBool("disableblocks"sv, true); + } + if (m_FileForceEnableTempBlocks) + { + Writer.AddBool("enabletempblocks"sv, true); + } + } + Writer.EndObject(); // "file" + TargetDescription = fmt::format("[file] {}/{}{}{}", + m_FileDirectoryPath, + m_FileName, + m_BaseFileName.empty() ? "" : " Base: ", + m_BaseFileName); + } + if (!m_CloudUrl.empty()) + { + Writer.BeginObject("cloud"sv); + { + Writer.AddString("url"sv, m_CloudUrl); + Writer.AddString("namespace"sv, m_CloudNamespace); + Writer.AddString("bucket"sv, m_CloudBucket); + Writer.AddString("key"sv, m_CloudKey); + if (!m_BaseCloudKey.empty()) + { + Writer.AddString("basekey"sv, m_BaseCloudKey); + } + if (!m_CloudOpenIdProvider.empty()) + { + Writer.AddString("openid-provider"sv, m_CloudOpenIdProvider); + } + if (!m_CloudAccessToken.empty()) + { + Writer.AddString("access-token"sv, m_CloudAccessToken); + } + if (!m_CloudAccessTokenEnv.empty()) + { + std::string ResolvedCloudAccessTokenEnv = GetEnvVariable(m_CloudAccessTokenEnv); + + if (!ResolvedCloudAccessTokenEnv.empty()) + { + Writer.AddString("access-token"sv, ResolvedCloudAccessTokenEnv); + } + else + { + Writer.AddString("access-token-env"sv, m_CloudAccessTokenEnv); + } + } + if (m_CloudAssumeHttp2) + { + Writer.AddBool("assumehttp2"sv, true); + } + if (m_DisableBlocks) + { + Writer.AddBool("disableblocks"sv, true); + } + if (m_CloudDisableTempBlocks) + { + Writer.AddBool("disabletempblocks"sv, true); + } + } + Writer.EndObject(); // "cloud" + TargetDescription = fmt::format("[cloud] {}/{}/{}/{}{}{}", + m_CloudUrl, + m_CloudNamespace, + m_CloudBucket, + m_CloudKey, + m_BaseCloudKey.empty() ? "" : " Base: ", + m_BaseCloudKey); + } + if (!m_ZenUrl.empty()) + { + Writer.BeginObject("zen"sv); + { + Writer.AddString("url"sv, m_ZenUrl); + Writer.AddString("project"sv, m_ZenProjectName); + Writer.AddString("oplog"sv, m_ZenOplogName); + } + Writer.EndObject(); // "zen" + + TargetDescription = fmt::format("[zen] {}/{}/{}", m_ZenUrl, m_ZenProjectName, m_ZenOplogName); + } + } + Writer.EndObject(); // "params" + }); + + ZEN_CONSOLE("Saving oplog '{}/{}' from '{}' to {}", m_ProjectName, m_OplogName, m_HostName, TargetDescription); + + HttpClient Http(m_HostName); + if (m_Async) + { + if (HttpClient::Response Result = Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), + std::move(Payload), + HttpClient::Accept(ZenContentType::kJSON)); + Result) + { + ZEN_CONSOLE("{}", Result.AsText()); + } + else + { + Result.ThrowError("failed requesting loading oplog export"sv); + return 1; + } + } + else + { + AsyncPost(Http, fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), std::move(Payload)); + } + return 0; +} + +//////////////////////////// + +ImportOplogCommand::ImportOplogCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), ""); + m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), ""); + m_Options.add_option("", "", "maxblocksize", "Max size for bundled attachments", cxxopts::value(m_MaxBlockSize), ""); + m_Options.add_option("", + "", + "maxchunkembedsize", + "Max size for attachment to be bundled", + cxxopts::value(m_MaxChunkEmbedSize), + ""); + m_Options.add_option("", "f", "force", "Force import of all attachments", cxxopts::value(m_Force), ""); + m_Options.add_option("", "a", "async", "Trigger import but don't wait for completion", cxxopts::value(m_Async), ""); + + m_Options.add_option("", "", "cloud", "Cloud Storage URL", cxxopts::value(m_CloudUrl), ""); + m_Options.add_option("cloud", "", "namespace", "Cloud Storage namespace", cxxopts::value(m_CloudNamespace), ""); + m_Options.add_option("cloud", "", "bucket", "Cloud Storage bucket", cxxopts::value(m_CloudBucket), ""); + m_Options.add_option("cloud", "", "key", "Cloud Storage key", cxxopts::value(m_CloudKey), ""); + m_Options + .add_option("cloud", "", "openid-provider", "Cloud Storage openid provider", cxxopts::value(m_CloudOpenIdProvider), ""); + m_Options.add_option("cloud", "", "access-token", "Cloud Storage access token", cxxopts::value(m_CloudAccessToken), ""); + m_Options.add_option("cloud", + "", + "access-token-env", + "Name of environment variable that holds the cloud Storage access token", + cxxopts::value(m_CloudAccessTokenEnv)->default_value(DefaultCloudAccessTokenEnvVariableName), + ""); + m_Options.add_option("cloud", + "", + "assume-http2", + "Assume that the cloud endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake", + cxxopts::value(m_CloudAssumeHttp2), + ""); + + m_Options.add_option("", "", "zen", "Zen service upload address", cxxopts::value(m_ZenUrl), ""); + m_Options.add_option("zen", "", "source-project", "Zen source project name", cxxopts::value(m_ZenProjectName), ""); + m_Options.add_option("zen", "", "source-oplog", "Zen source oplog name", cxxopts::value(m_ZenOplogName), ""); + m_Options.add_option("zen", "", "clean", "Delete existing target Zen oplog", cxxopts::value(m_ZenClean), ""); + + m_Options.add_option("", "", "file", "Local folder path", cxxopts::value(m_FileDirectoryPath), ""); + m_Options.add_option("file", "", "name", "Local file name", cxxopts::value(m_FileName), ""); + + m_Options.parse_positional({"project", "oplog"}); +} + +ImportOplogCommand::~ImportOplogCommand() +{ +} + +int +ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + using namespace std::literals; + + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + if (m_ProjectName.empty()) + { + ZEN_ERROR("Project name must be given"); + return 1; + } + + if (m_OplogName.empty()) + { + ZEN_ERROR("Oplog name must be given"); + return 1; + } + + size_t TargetCount = 0; + TargetCount += m_CloudUrl.empty() ? 0 : 1; + TargetCount += m_ZenUrl.empty() ? 0 : 1; + TargetCount += m_FileDirectoryPath.empty() ? 0 : 1; + if (TargetCount != 1) + { + ZEN_ERROR("Provide one source only"); + ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); + return 1; + } + + if (!m_CloudUrl.empty()) + { + if (m_CloudNamespace.empty() || m_CloudBucket.empty()) + { + ZEN_ERROR("Options for cloud source are missing"); + ZEN_CONSOLE("{}", m_Options.help({"cloud"}).c_str()); + return 1; + } + if (m_CloudKey.empty()) + { + std::string KeyString = fmt::format("{}/{}/{}/{}", m_ProjectName, m_OplogName, m_CloudNamespace, m_CloudBucket); + IoHash Key = IoHash::HashBuffer(KeyString.data(), KeyString.size()); + m_CloudKey = Key.ToHexString(); + ZEN_WARN("Using auto generated cloud key '{}'", m_CloudKey); + } + } + + if (!m_ZenUrl.empty()) + { + if (m_ZenProjectName.empty()) + { + m_ZenProjectName = m_ProjectName; + ZEN_WARN("Using default zen target project id '{}'", m_ZenProjectName); + } + if (m_ZenOplogName.empty()) + { + m_ZenOplogName = m_OplogName; + ZEN_WARN("Using default zen target oplog id '{}'", m_ZenOplogName); + } + } + + if (!m_FileDirectoryPath.empty()) + { + if (m_FileName.empty()) + { + m_FileName = m_OplogName; + ZEN_WARN("Using auto generated file name '{}'", m_FileName); + } + } + + HttpClient Http(m_HostName); + std::string Url = fmt::format("/prj/{}/oplog/{}", m_ProjectName, m_OplogName); + + bool CreateOplog = false; + if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) + { + if (m_ZenClean) + { + ZEN_WARN("Deleting oplog '{}/{}'", m_ProjectName, m_OplogName) + Result = Http.Delete(Url, HttpClient::Accept(ZenContentType::kJSON)); + if (!Result) + { + Result.ThrowError("failed deleting existing oplog"sv); + return 1; + } + CreateOplog = true; + } + } + else if (Result.StatusCode == HttpResponseCode::NotFound) + { + CreateOplog = true; + } + else + { + Result.ThrowError("failed checking oplog"sv); + return 1; + } + + if (CreateOplog) + { + ZEN_WARN("Creating oplog '{}/{}'", m_ProjectName, m_OplogName); + if (HttpClient::Response Result = Http.Post(Url); !Result) + { + Result.ThrowError("failed creating oplog"sv); + return 1; + } + } + + std::string SourceDescription; + + IoBuffer Payload = MakeCbObjectPayload([&](CbObjectWriter& Writer) { + Writer.AddString("method"sv, "import"sv); + Writer.BeginObject("params"sv); + { + if (m_Force) + { + Writer.AddBool("force"sv, true); + } + if (!m_FileDirectoryPath.empty()) + { + Writer.BeginObject("file"sv); + { + Writer.AddString("path"sv, m_FileDirectoryPath); + Writer.AddString("name"sv, m_FileName); + } + Writer.EndObject(); // "file" + SourceDescription = fmt::format("[file] {}/{}", m_FileDirectoryPath, m_FileName); + } + if (!m_CloudUrl.empty()) + { + Writer.BeginObject("cloud"sv); + { + Writer.AddString("url"sv, m_CloudUrl); + Writer.AddString("namespace"sv, m_CloudNamespace); + Writer.AddString("bucket"sv, m_CloudBucket); + Writer.AddString("key"sv, m_CloudKey); + if (!m_CloudOpenIdProvider.empty()) + { + Writer.AddString("openid-provider"sv, m_CloudOpenIdProvider); + } + if (!m_CloudAccessToken.empty()) + { + Writer.AddString("access-token"sv, m_CloudAccessToken); + } + if (!m_CloudAccessTokenEnv.empty()) + { + std::string ResolvedCloudAccessTokenEnv = GetEnvVariable(m_CloudAccessTokenEnv); + + if (!ResolvedCloudAccessTokenEnv.empty()) + { + Writer.AddString("access-token"sv, ResolvedCloudAccessTokenEnv); + } + else + { + Writer.AddString("access-token-env"sv, m_CloudAccessTokenEnv); + } + } + if (m_CloudAssumeHttp2) + { + Writer.AddBool("assumehttp2"sv, true); + } + } + Writer.EndObject(); // "cloud" + SourceDescription = fmt::format("[cloud] {}/{}/{}/{}", m_CloudUrl, m_CloudNamespace, m_CloudBucket, m_CloudKey); + } + if (!m_ZenUrl.empty()) + { + Writer.BeginObject("zen"sv); + { + Writer.AddString("url"sv, m_ZenUrl); + Writer.AddString("project"sv, m_ZenProjectName); + Writer.AddString("oplog"sv, m_ZenOplogName); + } + Writer.EndObject(); // "zen" + SourceDescription = fmt::format("[zen] {}", m_ZenUrl); + } + } + Writer.EndObject(); // "params" + }); + + ZEN_CONSOLE("Loading oplog '{}/{}' from '{}' to {}", m_ProjectName, m_OplogName, SourceDescription, m_HostName); + + if (m_Async) + { + if (HttpClient::Response Result = Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), + std::move(Payload), + HttpClient::Accept(ZenContentType::kJSON)); + Result) + { + ZEN_CONSOLE("{}", Result.AsText()); + } + else + { + Result.ThrowError("failed requesting loading oplog import"sv); + return 1; + } + } + else + { + AsyncPost(Http, fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), std::move(Payload)); + } + return 0; +} + +//////////////////////////// + +SnapshotOplogCommand::SnapshotOplogCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), ""); + m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), ""); + + m_Options.parse_positional({"project", "oplog"}); +} + +SnapshotOplogCommand::~SnapshotOplogCommand() +{ +} + +int +SnapshotOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + using namespace std::literals; + + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + if (m_ProjectName.empty()) + { + ZEN_ERROR("Project name must be given"); + return 1; + } + + if (m_OplogName.empty()) + { + ZEN_ERROR("Oplog name must be given"); + return 1; + } + + IoBuffer Payload = MakeCbObjectPayload([&](CbObjectWriter& Writer) { Writer.AddString("method"sv, "snapshot"sv); }); + + HttpClient Http(m_HostName); + + ZEN_CONSOLE("Snapshotting oplog '{}/{}' to {}", m_ProjectName, m_OplogName, m_HostName); + if (HttpClient::Response Result = + Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", m_ProjectName, m_OplogName), Payload, HttpClient::Accept(ZenContentType::kJSON))) + { + ZEN_CONSOLE("{}", Result); + return 0; + } + else + { + Result.ThrowError("failed to create project"sv); + return 1; + } +} + +//////////////////////////// + +ProjectStatsCommand::ProjectStatsCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); +} + +ProjectStatsCommand::~ProjectStatsCommand() +{ +} + +int +ProjectStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + HttpClient Http(m_HostName); + if (HttpClient::Response Result = Http.Get("/stats/prj", HttpClient::Accept(ZenContentType::kJSON))) + { + ZEN_CONSOLE("{}", Result.AsText()); + return 0; + } + else + { + Result.ThrowError("failed to get project stats"sv); + return 1; + } +} + +//////////////////////////// + +ProjectDetailsCommand::ProjectDetailsCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "c", "csv", "Output in CSV format (default is JSon)", cxxopts::value(m_CSV), ""); + m_Options.add_option("", "d", "details", "Detailed info on oplog", cxxopts::value(m_Details), "
"); + m_Options.add_option("", "o", "opdetails", "Details info on oplog body", cxxopts::value(m_OpDetails), ""); + m_Options.add_option("", "p", "project", "Project name to get info from", cxxopts::value(m_ProjectName), ""); + m_Options.add_option("", "l", "oplog", "Oplog name to get info from", cxxopts::value(m_OplogName), ""); + m_Options.add_option("", "i", "opid", "Oid of a specific op info for", cxxopts::value(m_OpId), ""); + m_Options.add_option("", + "a", + "attachmentdetails", + "Get detailed information about attachments", + cxxopts::value(m_AttachmentDetails), + ""); +} + +ProjectDetailsCommand::~ProjectDetailsCommand() +{ +} + +int +ProjectDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + if (!m_OpId.empty()) + { + if (m_ProjectName.empty() || m_OplogName.empty()) + { + ZEN_ERROR("Provide project and oplog name"); + ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); + return 1; + } + } + else if (!m_OplogName.empty()) + { + if (m_ProjectName.empty()) + { + ZEN_ERROR("Provide project name"); + ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); + return 1; + } + } + + HttpClient Http(m_HostName); + + ExtendableStringBuilder<128> Url; + Url.Append("/prj/details$"); + if (!m_ProjectName.empty()) + { + Url.Append("/"); + Url.Append(m_ProjectName); + } + if (!m_OplogName.empty()) + { + Url.Append("/"); + Url.Append(m_OplogName); + } + if (!m_OpId.empty()) + { + Url.Append("/"); + Url.Append(m_OpId); + } + + if (HttpClient::Response Result = + Http.Get(Url, + m_CSV ? HttpClient::Accept(ZenContentType::kText) : HttpClient::Accept(ZenContentType::kJSON), + {{"opdetails", m_OpDetails ? "true" : "false"}, + {"details", m_Details ? "true" : "false"}, + {"attachmentdetails", m_AttachmentDetails ? "true" : "false"}, + {"csv", m_CSV ? "true" : "false"}})) + { + ZEN_CONSOLE("{}", Result.AsText()); + return 0; + } + else + { + Result.ThrowError("failed to get project details"sv); + return 1; + } +} + +//////////////////////////// + +OplogMirrorCommand::OplogMirrorCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "p", "project", "Project name to get info from", cxxopts::value(m_ProjectName), ""); + m_Options.add_option("", "l", "oplog", "Oplog name to get info from", cxxopts::value(m_OplogName), ""); + m_Options.add_option("", "t", "target", "Target directory for mirror", cxxopts::value(m_MirrorRootPath), ""); + + m_Options.parse_positional({"project", "oplog", "target"}); + m_Options.positional_help("[ ]"); +} + +OplogMirrorCommand::~OplogMirrorCommand() +{ +} + +int +OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + if (m_ProjectName.empty()) + { + throw OptionParseException("a project must be specified"); + } + + if (m_OplogName.empty()) + { + throw OptionParseException("an oplog must be specified"); + } + + if (m_MirrorRootPath.empty()) + { + throw OptionParseException("a target path must be specified"); + } + + ZEN_CONSOLE("Emitting file data from oplog '{}/{}' to '{}'", m_ProjectName, m_OplogName, m_MirrorRootPath); + + HttpClient Http(m_HostName); + + if (HttpClient::Response Result = Http.Get(fmt::format("/prj/{}/oplog/{}", m_ProjectName, m_OplogName))) + { + // The info requested is not really used at this moment, we just use the probe to be able to provide + // better diagnostics up front + } + else + { + Result.ThrowError("oplog info fetch failed"sv); + + return 1; + } + + // Emit file data to target directory + + std::filesystem::path RootPath{m_MirrorRootPath}; + CreateDirectories(RootPath); + + std::filesystem::path TmpPath = RootPath / ".tmp"; + CreateDirectories(TmpPath); + + std::atomic_int64_t FileCount = 0; + int OplogEntryCount = 0; + + size_t WorkerCount = Min(std::thread::hardware_concurrency(), 16u); + WorkerThreadPool WorkerPool(gsl::narrow(WorkerCount)); + Latch WorkRemaining(1); + + std::unordered_set FileNames; + + auto EmitFilesForDataArray = [&](CbArrayView DataArray) { + for (auto DataIter : DataArray) + { + if (CbObjectView Data = DataIter.AsObjectView()) + { + std::string FileName = std::string(Data["filename"sv].AsString()); + Oid ChunkId = Data["id"sv].AsObjectId(); + if (!FileNames.insert(FileName).second) + { + continue; + } + WorkRemaining.AddCount(1); + WorkerPool.ScheduleWork([this, &RootPath, FileName, &FileCount, ChunkId, &Http, TmpPath, &WorkRemaining]() { + auto _ = MakeGuard([&WorkRemaining]() { WorkRemaining.CountDown(); }); + if (HttpClient::Response ChunkResponse = + Http.Download(fmt::format("/prj/{}/oplog/{}/{}"sv, m_ProjectName, m_OplogName, ChunkId), TmpPath)) + { + IoBuffer ChunkData = ChunkResponse.ResponsePayload; + std::filesystem::path TargetPath = RootPath / FileName; + if (!MoveToFile(TargetPath, ChunkData)) + { + WriteFile(TargetPath, ChunkData); + } + ++FileCount; + } + else + { + ZEN_CONSOLE("Unable to fetch '{}' (chunk {}). Reason: '{}'", FileName, ChunkId, ChunkResponse.ErrorMessage(""sv)); + } + }); + } + } + }; + + if (HttpClient::Response Response = Http.Get(fmt::format("/prj/{}/oplog/{}/entries"sv, m_ProjectName, m_OplogName))) + { + if (CbObject ResponseObject = Response.AsObject()) + { + for (auto EntryIter : ResponseObject["entries"sv]) + { + CbObjectView Entry = EntryIter.AsObjectView(); + + EmitFilesForDataArray(Entry["packagedata"sv].AsArrayView()); + EmitFilesForDataArray(Entry["bulkdata"sv].AsArrayView()); + + ++OplogEntryCount; + } + } + else + { + ZEN_ERROR("unknown format response to oplog entries request"); + } + } + else + { + Response.ThrowError("oplog entries fetch failed"); + + return 1; + } + WorkRemaining.CountDown(); + WorkRemaining.Wait(); + + std::filesystem::remove_all(TmpPath); + + ZEN_CONSOLE("mirrored {} files from {} oplog entries successfully", FileCount.load(), OplogEntryCount); + + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h new file mode 100644 index 000000000..fd1590423 --- /dev/null +++ b/src/zen/cmds/projectstore_cmd.h @@ -0,0 +1,256 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +class DropProjectCommand : public ZenCmdBase +{ +public: + DropProjectCommand(); + ~DropProjectCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"project-drop", "Drop project or project oplog"}; + std::string m_HostName; + std::string m_ProjectName; + std::string m_OplogName; +}; + +class ProjectInfoCommand : public ZenCmdBase +{ +public: + ProjectInfoCommand(); + ~ProjectInfoCommand(); + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"project-info", "Info on project or project oplog"}; + std::string m_HostName; + std::string m_ProjectName; + std::string m_OplogName; +}; + +class CreateProjectCommand : public ZenCmdBase +{ +public: + CreateProjectCommand(); + ~CreateProjectCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"project-create", "Create project, the project must not already exist."}; + std::string m_HostName; + std::string m_ProjectId; + std::string m_RootDir; + std::string m_EngineRootDir; + std::string m_ProjectRootDir; + std::string m_ProjectFile; + bool m_ForceUpdate = false; +}; + +class DeleteProjectCommand : public ZenCmdBase +{ +public: + DeleteProjectCommand(); + ~DeleteProjectCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"project-delete", "Delete project and all its oplogs"}; + std::string m_HostName; + std::string m_ProjectId; +}; + +class CreateOplogCommand : public ZenCmdBase +{ +public: + CreateOplogCommand(); + ~CreateOplogCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"oplog-create", "Create oplog in an existing project, the oplog must not already exist."}; + std::string m_HostName; + std::string m_ProjectId; + std::string m_OplogId; + std::string m_GcPath; + bool m_ForceUpdate = false; +}; + +class DeleteOplogCommand : public ZenCmdBase +{ +public: + DeleteOplogCommand(); + ~DeleteOplogCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"oplog-delete", "Delete oplog and all its data"}; + std::string m_HostName; + std::string m_ProjectId; + std::string m_OplogId; +}; + +class ExportOplogCommand : public ZenCmdBase +{ +public: + ExportOplogCommand(); + ~ExportOplogCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"oplog-export", + "Export project store oplog to cloud (--cloud), file system (--file) or other Zen instance (--zen)"}; + std::string m_HostName; + std::string m_ProjectName; + std::string m_OplogName; + uint64_t m_MaxBlockSize = 0; + uint64_t m_MaxChunkEmbedSize = 0; + bool m_EmbedLooseFiles = false; + bool m_Force = false; + bool m_DisableBlocks = false; + bool m_Async = false; + + std::string m_CloudUrl; + std::string m_CloudNamespace; + std::string m_CloudBucket; + std::string m_CloudKey; + std::string m_BaseCloudKey; + std::string m_CloudOpenIdProvider; + std::string m_CloudAccessToken; + std::string m_CloudAccessTokenEnv; + bool m_CloudAssumeHttp2 = false; + bool m_CloudDisableTempBlocks = false; + + std::string m_ZenUrl; + std::string m_ZenProjectName; + std::string m_ZenOplogName; + bool m_ZenClean; + + std::string m_FileDirectoryPath; + std::string m_FileName; + std::string m_BaseFileName; + bool m_FileForceEnableTempBlocks = false; +}; + +class ImportOplogCommand : public ZenCmdBase +{ +public: + ImportOplogCommand(); + ~ImportOplogCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"oplog-import", + "Import project store oplog from cloud (--cloud), file system (--file) or other Zen instance (--zen)"}; + std::string m_HostName; + std::string m_ProjectName; + std::string m_OplogName; + size_t m_MaxBlockSize = 0; + size_t m_MaxChunkEmbedSize = 0; + bool m_Force = false; + bool m_Async = false; + + std::string m_CloudUrl; + std::string m_CloudNamespace; + std::string m_CloudBucket; + std::string m_CloudKey; + std::string m_CloudOpenIdProvider; + std::string m_CloudAccessToken; + std::string m_CloudAccessTokenEnv; + bool m_CloudAssumeHttp2 = false; + + std::string m_ZenUrl; + std::string m_ZenProjectName; + std::string m_ZenOplogName; + bool m_ZenClean; + + std::string m_FileDirectoryPath; + std::string m_FileName; +}; + +class SnapshotOplogCommand : public ZenCmdBase +{ +public: + SnapshotOplogCommand(); + ~SnapshotOplogCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"oplog-snapshot", "Snapshot external file references in project store oplog into zen"}; + std::string m_HostName; + std::string m_ProjectName; + std::string m_OplogName; +}; + +class ProjectStatsCommand : public ZenCmdBase +{ +public: + ProjectStatsCommand(); + ~ProjectStatsCommand(); + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"project-stats", "Stats info on project store"}; + std::string m_HostName; +}; + +class ProjectDetailsCommand : public ZenCmdBase +{ +public: + ProjectDetailsCommand(); + ~ProjectDetailsCommand(); + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"project-details", "Detail info on project store"}; + std::string m_HostName; + bool m_Details; + bool m_OpDetails; + bool m_AttachmentDetails; + bool m_CSV; + std::string m_ProjectName; + std::string m_OplogName; + std::string m_OpId; +}; + +class OplogMirrorCommand : public ZenCmdBase +{ +public: + OplogMirrorCommand(); + ~OplogMirrorCommand(); + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"oplog-mirror", "Mirror oplog to file system"}; + std::string m_HostName; + std::string m_ProjectName; + std::string m_OplogName; + std::string m_MirrorRootPath; +}; + +} // namespace zen diff --git a/src/zen/cmds/rpcreplay.cpp b/src/zen/cmds/rpcreplay.cpp deleted file mode 100644 index 349025791..000000000 --- a/src/zen/cmds/rpcreplay.cpp +++ /dev/null @@ -1,438 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "rpcreplay.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -ZEN_THIRD_PARTY_INCLUDES_START -#include -#include -#include -ZEN_THIRD_PARTY_INCLUDES_END - -#include - -namespace zen { - -using namespace std::literals; - -RpcStartRecordingCommand::RpcStartRecordingCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "p", "path", "Recording file path", cxxopts::value(m_RecordingPath), ""); - - m_Options.parse_positional("path"); -} - -RpcStartRecordingCommand::~RpcStartRecordingCommand() = default; - -int -RpcStartRecordingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions, argc, argv); - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw zen::OptionParseException("unable to resolve server specification"); - } - - if (m_RecordingPath.empty()) - { - throw zen::OptionParseException("Rpc start recording command requires a path"); - } - - cpr::Session Session; - Session.SetUrl(fmt::format("{}/z$/exec$/start-recording"sv, m_HostName)); - Session.SetParameters({{"path", m_RecordingPath}}); - cpr::Response Response = Session.Post(); - ZEN_CONSOLE("{}", FormatHttpResponse(Response)); - return MapHttpToCommandReturnCode(Response); -} - -//////////////////////////////////////////////////// - -RpcStopRecordingCommand::RpcStopRecordingCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); -} - -RpcStopRecordingCommand::~RpcStopRecordingCommand() = default; - -int -RpcStopRecordingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions, argc, argv); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw zen::OptionParseException("unable to resolve server specification"); - } - - cpr::Session Session; - Session.SetUrl(fmt::format("{}/z$/exec$/stop-recording"sv, m_HostName)); - cpr::Response Response = Session.Post(); - ZEN_CONSOLE("{}", FormatHttpResponse(Response)); - return MapHttpToCommandReturnCode(Response); -} - -//////////////////////////////////////////////////// - -RpcReplayCommand::RpcReplayCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "p", "path", "Recording file path", cxxopts::value(m_RecordingPath), ""); - m_Options.add_option("", - "w", - "numthreads", - "Number of worker threads per process", - cxxopts::value(m_ThreadCount)->default_value(fmt::format("{}", std::thread::hardware_concurrency())), - ""); - m_Options.add_option("", "", "onhost", "Replay on host, bypassing http/network layer", cxxopts::value(m_OnHost), ""); - m_Options.add_option("", - "", - "showmethodstats", - "Show statistics of which RPC methods are used", - cxxopts::value(m_ShowMethodStats), - ""); - m_Options.add_option("", - "", - "offset", - "Offset into request recording to start replay", - cxxopts::value(m_Offset)->default_value("0"), - ""); - m_Options.add_option("", - "", - "stride", - "Stride for request recording when replaying requests", - cxxopts::value(m_Stride)->default_value("1"), - ""); - m_Options.add_option("", "", "numproc", "Number of worker processes", cxxopts::value(m_ProcessCount)->default_value("1"), ""); - m_Options.add_option("", - "", - "forceallowlocalrefs", - "Force enable local refs in requests", - cxxopts::value(m_ForceAllowLocalRefs), - ""); - m_Options - .add_option("", "", "disablelocalrefs", "Force disable local refs in requests", cxxopts::value(m_DisableLocalRefs), ""); - m_Options.add_option("", - "", - "forceallowlocalhandlerefs", - "Force enable local refs as handles in requests", - cxxopts::value(m_ForceAllowLocalHandleRef), - ""); - m_Options.add_option("", - "", - "disablelocalhandlerefs", - "Force disable local refs as handles in requests", - cxxopts::value(m_DisableLocalHandleRefs), - ""); - m_Options.add_option("", - "", - "forceallowpartiallocalrefs", - "Force enable local refs for all sizes", - cxxopts::value(m_ForceAllowPartialLocalRefs), - ""); - m_Options.add_option("", - "", - "disablepartiallocalrefs", - "Force disable local refs for all sizes", - cxxopts::value(m_DisablePartialLocalRefs), - ""); - - m_Options.parse_positional("path"); -} - -RpcReplayCommand::~RpcReplayCommand() = default; - -int -RpcReplayCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions, argc, argv); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw zen::OptionParseException("unable to resolve server specification"); - } - - if (m_RecordingPath.empty()) - { - throw zen::OptionParseException("Rpc replay command requires a path"); - } - - if (m_OnHost) - { - cpr::Session Session; - Session.SetUrl(fmt::format("{}/z$/exec$/replay-recording"sv, m_HostName)); - Session.SetParameters({{"path", m_RecordingPath}, {"thread-count", fmt::format("{}", m_ThreadCount)}}); - cpr::Response Response = Session.Post(); - ZEN_CONSOLE("{}", FormatHttpResponse(Response)); - return MapHttpToCommandReturnCode(Response); - } - - std::unique_ptr Replayer = cache::MakeDiskRequestReplayer(m_RecordingPath, true); - uint64_t EntryCount = Replayer->GetRequestCount(); - - std::atomic_uint64_t EntryOffset = m_Offset; - std::atomic_uint64_t BytesSent = 0; - std::atomic_uint64_t BytesReceived = 0; - - Stopwatch Timer; - - if (m_ProcessCount > 1) - { - std::vector> WorkerProcesses; - WorkerProcesses.resize(m_ProcessCount); - - ProcessMonitor Monitor; - for (int ProcessIndex = 0; ProcessIndex < m_ProcessCount; ++ProcessIndex) - { - std::string CommandLine = - fmt::format("{} rpc-record-replay --hosturl {} --path \"{}\" --offset {} --stride {} --numthreads {} --numproc {}"sv, - argv[0], - m_HostName, - m_RecordingPath, - m_Stride == 1 ? 0 : m_Offset + ProcessIndex, - m_Stride, - m_ThreadCount, - 1); - CreateProcResult Result(CreateProc(std::filesystem::path(std::string(argv[0])), CommandLine)); - WorkerProcesses[ProcessIndex] = std::make_unique(); - WorkerProcesses[ProcessIndex]->Initialize(Result); - Monitor.AddPid(WorkerProcesses[ProcessIndex]->Pid()); - } - while (Monitor.IsRunning()) - { - ZEN_CONSOLE("Waiting for worker processes..."); - Sleep(1000); - } - return 0; - } - else - { - std::map MethodTypes; - RwLock MethodTypesLock; - - WorkerThreadPool WorkerPool(m_ThreadCount); - - Latch WorkLatch(m_ThreadCount); - for (int WorkerIndex = 0; WorkerIndex < m_ThreadCount; ++WorkerIndex) - { - WorkerPool.ScheduleWork( - [this, &WorkLatch, EntryCount, &EntryOffset, &Replayer, &BytesSent, &BytesReceived, &MethodTypes, &MethodTypesLock]() { - auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); - - cpr::Session Session; - Session.SetUrl(fmt::format("{}/z$/$rpc"sv, m_HostName)); - - uint64_t EntryIndex = EntryOffset.fetch_add(m_Stride); - while (EntryIndex < EntryCount) - { - IoBuffer Payload; - std::pair Types = Replayer->GetRequest(EntryIndex, Payload); - ZenContentType RequestContentType = Types.first; - ZenContentType AcceptContentType = Types.second; - - CbPackage RequestPackage; - CbObject Request; - switch (RequestContentType) - { - case ZenContentType::kCbPackage: - { - if (ParsePackageMessageWithLegacyFallback(Payload, RequestPackage)) - { - Request = RequestPackage.GetObject(); - } - } - break; - case ZenContentType::kCbObject: - { - Request = LoadCompactBinaryObject(Payload); - } - break; - } - - RpcAcceptOptions OriginalAcceptOptions = static_cast(Request["AcceptFlags"sv].AsUInt16(0u)); - int OriginalProcessPid = Request["Pid"sv].AsInt32(0); - - int AdjustedPid = 0; - RpcAcceptOptions AdjustedAcceptOptions = RpcAcceptOptions::kNone; - if (!m_DisableLocalRefs) - { - if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowLocalReferences) || m_ForceAllowLocalRefs) - { - AdjustedAcceptOptions |= RpcAcceptOptions::kAllowLocalReferences; - if (!m_DisablePartialLocalRefs) - { - if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowPartialLocalReferences) || - m_ForceAllowPartialLocalRefs) - { - AdjustedAcceptOptions |= RpcAcceptOptions::kAllowPartialLocalReferences; - } - } - if (!m_DisableLocalHandleRefs) - { - if (OriginalProcessPid != 0 || m_ForceAllowLocalHandleRef) - { - AdjustedPid = GetCurrentProcessId(); - } - } - } - } - - if (m_ShowMethodStats) - { - std::string MethodName = std::string(Request["Method"sv].AsString()); - RwLock::ExclusiveLockScope __(MethodTypesLock); - if (auto It = MethodTypes.find(MethodName); It != MethodTypes.end()) - { - It->second++; - } - else - { - MethodTypes[MethodName] = 1; - } - } - - if (OriginalAcceptOptions != AdjustedAcceptOptions || OriginalProcessPid != AdjustedPid) - { - CbObjectWriter RequestCopyWriter; - for (const CbFieldView& Field : Request) - { - if (!Field.HasName()) - { - RequestCopyWriter.AddField(Field); - continue; - } - std::string_view FieldName = Field.GetName(); - if (FieldName == "Pid"sv) - { - continue; - } - if (FieldName == "AcceptFlags"sv) - { - continue; - } - RequestCopyWriter.AddField(FieldName, Field); - } - if (AdjustedPid != 0) - { - RequestCopyWriter.AddInteger("Pid"sv, AdjustedPid); - } - if (AdjustedAcceptOptions != RpcAcceptOptions::kNone) - { - RequestCopyWriter.AddInteger("AcceptFlags"sv, static_cast(AdjustedAcceptOptions)); - } - - if (RequestContentType == ZenContentType::kCbPackage) - { - RequestPackage.SetObject(RequestCopyWriter.Save()); - std::vector Buffers = FormatPackageMessage(RequestPackage); - std::vector SharedBuffers(Buffers.begin(), Buffers.end()); - Payload = CompositeBuffer(std::move(SharedBuffers)).Flatten().AsIoBuffer(); - } - else - { - RequestCopyWriter.Finalize(); - Payload = IoBuffer(RequestCopyWriter.GetSaveSize()); - RequestCopyWriter.Save(Payload.GetMutableView()); - } - } - - Session.SetHeader({{"Content-Type", std::string(MapContentTypeToString(RequestContentType))}, - {"Accept", std::string(MapContentTypeToString(AcceptContentType))}}); - uint64_t Offset = 0; - auto ReadCallback = [&Payload, &Offset](char* buffer, size_t& size, intptr_t) { - size = Min(size, Payload.GetSize() - Offset); - IoBuffer PayloadRange = IoBuffer(Payload, Offset, size); - MutableMemoryView Data(buffer, size); - Data.CopyFrom(PayloadRange.GetView()); - Offset += size; - return true; - }; - Session.SetReadCallback(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); - cpr::Response Response = Session.Post(); - BytesSent.fetch_add(Payload.GetSize()); - if (Response.error || !(IsHttpSuccessCode(Response.status_code) || - Response.status_code == gsl::narrow(HttpResponseCode::NotFound))) - { - ZEN_CONSOLE("{}", FormatHttpResponse(Response)); - break; - } - BytesReceived.fetch_add(Response.downloaded_bytes); - EntryIndex = EntryOffset.fetch_add(m_Stride); - } - }); - } - - while (!WorkLatch.Wait(1000)) - { - ZEN_CONSOLE("Processing {} requests, {} remaining (sent {}, recevied {})...", - (EntryCount - m_Offset) / m_Stride, - (EntryCount - EntryOffset.load()) / m_Stride, - NiceBytes(BytesSent.load()), - NiceBytes(BytesReceived.load())); - } - if (m_ShowMethodStats) - { - for (const auto& It : MethodTypes) - { - ZEN_CONSOLE("{}: {}", It.first, It.second); - } - } - } - - const uint64_t RequestsSent = (EntryOffset.load() - m_Offset) / m_Stride; - const uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); - const double ElapsedS = ElapsedMS / 1000.500; - const uint64_t Sent = BytesSent.load(); - const uint64_t Received = BytesReceived.load(); - const uint64_t RequestsPerS = static_cast(RequestsSent / ElapsedS); - const uint64_t SentPerS = static_cast(Sent / ElapsedS); - const uint64_t ReceivedPerS = static_cast(Received / ElapsedS); - - ZEN_CONSOLE("Requests sent {} ({}/s), payloads sent {}B ({}B/s), payloads received {}B ({}B/s) in {}", - RequestsSent, - RequestsPerS, - NiceBytes(Sent), - NiceBytes(SentPerS), - NiceBytes(Received), - NiceBytes(ReceivedPerS), - NiceTimeSpanMs(ElapsedMS)); - - return 0; -} - -} // namespace zen diff --git a/src/zen/cmds/rpcreplay.h b/src/zen/cmds/rpcreplay.h deleted file mode 100644 index 742e5ec5b..000000000 --- a/src/zen/cmds/rpcreplay.h +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -class RpcStartRecordingCommand : public ZenCmdBase -{ -public: - RpcStartRecordingCommand(); - ~RpcStartRecordingCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"rpc-record-start", "Starts recording of cache rpc requests on a host"}; - std::string m_HostName; - std::string m_RecordingPath; -}; - -class RpcStopRecordingCommand : public ZenCmdBase -{ -public: - RpcStopRecordingCommand(); - ~RpcStopRecordingCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"rpc-record-stop", "Stops recording of cache rpc requests on a host"}; - std::string m_HostName; -}; - -class RpcReplayCommand : public ZenCmdBase -{ -public: - RpcReplayCommand(); - ~RpcReplayCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"rpc-record-replay", "Replays a previously recorded session of cache rpc requests to a target host"}; - std::string m_HostName; - std::string m_RecordingPath; - bool m_OnHost = false; - bool m_ShowMethodStats = false; - int m_ProcessCount; - int m_ThreadCount; - uint64_t m_Offset; - uint64_t m_Stride; - bool m_ForceAllowLocalRefs; - bool m_DisableLocalRefs; - bool m_ForceAllowLocalHandleRef; - bool m_DisableLocalHandleRefs; - bool m_ForceAllowPartialLocalRefs; - bool m_DisablePartialLocalRefs; -}; - -} // namespace zen diff --git a/src/zen/cmds/rpcreplay_cmd.cpp b/src/zen/cmds/rpcreplay_cmd.cpp new file mode 100644 index 000000000..9e43280e1 --- /dev/null +++ b/src/zen/cmds/rpcreplay_cmd.cpp @@ -0,0 +1,438 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "rpcreplay_cmd.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +#include +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#include + +namespace zen { + +using namespace std::literals; + +RpcStartRecordingCommand::RpcStartRecordingCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "p", "path", "Recording file path", cxxopts::value(m_RecordingPath), ""); + + m_Options.parse_positional("path"); +} + +RpcStartRecordingCommand::~RpcStartRecordingCommand() = default; + +int +RpcStartRecordingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions, argc, argv); + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw zen::OptionParseException("unable to resolve server specification"); + } + + if (m_RecordingPath.empty()) + { + throw zen::OptionParseException("Rpc start recording command requires a path"); + } + + cpr::Session Session; + Session.SetUrl(fmt::format("{}/z$/exec$/start-recording"sv, m_HostName)); + Session.SetParameters({{"path", m_RecordingPath}}); + cpr::Response Response = Session.Post(); + ZEN_CONSOLE("{}", FormatHttpResponse(Response)); + return MapHttpToCommandReturnCode(Response); +} + +//////////////////////////////////////////////////// + +RpcStopRecordingCommand::RpcStopRecordingCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); +} + +RpcStopRecordingCommand::~RpcStopRecordingCommand() = default; + +int +RpcStopRecordingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions, argc, argv); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw zen::OptionParseException("unable to resolve server specification"); + } + + cpr::Session Session; + Session.SetUrl(fmt::format("{}/z$/exec$/stop-recording"sv, m_HostName)); + cpr::Response Response = Session.Post(); + ZEN_CONSOLE("{}", FormatHttpResponse(Response)); + return MapHttpToCommandReturnCode(Response); +} + +//////////////////////////////////////////////////// + +RpcReplayCommand::RpcReplayCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "p", "path", "Recording file path", cxxopts::value(m_RecordingPath), ""); + m_Options.add_option("", + "w", + "numthreads", + "Number of worker threads per process", + cxxopts::value(m_ThreadCount)->default_value(fmt::format("{}", std::thread::hardware_concurrency())), + ""); + m_Options.add_option("", "", "onhost", "Replay on host, bypassing http/network layer", cxxopts::value(m_OnHost), ""); + m_Options.add_option("", + "", + "showmethodstats", + "Show statistics of which RPC methods are used", + cxxopts::value(m_ShowMethodStats), + ""); + m_Options.add_option("", + "", + "offset", + "Offset into request recording to start replay", + cxxopts::value(m_Offset)->default_value("0"), + ""); + m_Options.add_option("", + "", + "stride", + "Stride for request recording when replaying requests", + cxxopts::value(m_Stride)->default_value("1"), + ""); + m_Options.add_option("", "", "numproc", "Number of worker processes", cxxopts::value(m_ProcessCount)->default_value("1"), ""); + m_Options.add_option("", + "", + "forceallowlocalrefs", + "Force enable local refs in requests", + cxxopts::value(m_ForceAllowLocalRefs), + ""); + m_Options + .add_option("", "", "disablelocalrefs", "Force disable local refs in requests", cxxopts::value(m_DisableLocalRefs), ""); + m_Options.add_option("", + "", + "forceallowlocalhandlerefs", + "Force enable local refs as handles in requests", + cxxopts::value(m_ForceAllowLocalHandleRef), + ""); + m_Options.add_option("", + "", + "disablelocalhandlerefs", + "Force disable local refs as handles in requests", + cxxopts::value(m_DisableLocalHandleRefs), + ""); + m_Options.add_option("", + "", + "forceallowpartiallocalrefs", + "Force enable local refs for all sizes", + cxxopts::value(m_ForceAllowPartialLocalRefs), + ""); + m_Options.add_option("", + "", + "disablepartiallocalrefs", + "Force disable local refs for all sizes", + cxxopts::value(m_DisablePartialLocalRefs), + ""); + + m_Options.parse_positional("path"); +} + +RpcReplayCommand::~RpcReplayCommand() = default; + +int +RpcReplayCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions, argc, argv); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw zen::OptionParseException("unable to resolve server specification"); + } + + if (m_RecordingPath.empty()) + { + throw zen::OptionParseException("Rpc replay command requires a path"); + } + + if (m_OnHost) + { + cpr::Session Session; + Session.SetUrl(fmt::format("{}/z$/exec$/replay-recording"sv, m_HostName)); + Session.SetParameters({{"path", m_RecordingPath}, {"thread-count", fmt::format("{}", m_ThreadCount)}}); + cpr::Response Response = Session.Post(); + ZEN_CONSOLE("{}", FormatHttpResponse(Response)); + return MapHttpToCommandReturnCode(Response); + } + + std::unique_ptr Replayer = cache::MakeDiskRequestReplayer(m_RecordingPath, true); + uint64_t EntryCount = Replayer->GetRequestCount(); + + std::atomic_uint64_t EntryOffset = m_Offset; + std::atomic_uint64_t BytesSent = 0; + std::atomic_uint64_t BytesReceived = 0; + + Stopwatch Timer; + + if (m_ProcessCount > 1) + { + std::vector> WorkerProcesses; + WorkerProcesses.resize(m_ProcessCount); + + ProcessMonitor Monitor; + for (int ProcessIndex = 0; ProcessIndex < m_ProcessCount; ++ProcessIndex) + { + std::string CommandLine = + fmt::format("{} rpc-record-replay --hosturl {} --path \"{}\" --offset {} --stride {} --numthreads {} --numproc {}"sv, + argv[0], + m_HostName, + m_RecordingPath, + m_Stride == 1 ? 0 : m_Offset + ProcessIndex, + m_Stride, + m_ThreadCount, + 1); + CreateProcResult Result(CreateProc(std::filesystem::path(std::string(argv[0])), CommandLine)); + WorkerProcesses[ProcessIndex] = std::make_unique(); + WorkerProcesses[ProcessIndex]->Initialize(Result); + Monitor.AddPid(WorkerProcesses[ProcessIndex]->Pid()); + } + while (Monitor.IsRunning()) + { + ZEN_CONSOLE("Waiting for worker processes..."); + Sleep(1000); + } + return 0; + } + else + { + std::map MethodTypes; + RwLock MethodTypesLock; + + WorkerThreadPool WorkerPool(m_ThreadCount); + + Latch WorkLatch(m_ThreadCount); + for (int WorkerIndex = 0; WorkerIndex < m_ThreadCount; ++WorkerIndex) + { + WorkerPool.ScheduleWork( + [this, &WorkLatch, EntryCount, &EntryOffset, &Replayer, &BytesSent, &BytesReceived, &MethodTypes, &MethodTypesLock]() { + auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + + cpr::Session Session; + Session.SetUrl(fmt::format("{}/z$/$rpc"sv, m_HostName)); + + uint64_t EntryIndex = EntryOffset.fetch_add(m_Stride); + while (EntryIndex < EntryCount) + { + IoBuffer Payload; + std::pair Types = Replayer->GetRequest(EntryIndex, Payload); + ZenContentType RequestContentType = Types.first; + ZenContentType AcceptContentType = Types.second; + + CbPackage RequestPackage; + CbObject Request; + switch (RequestContentType) + { + case ZenContentType::kCbPackage: + { + if (ParsePackageMessageWithLegacyFallback(Payload, RequestPackage)) + { + Request = RequestPackage.GetObject(); + } + } + break; + case ZenContentType::kCbObject: + { + Request = LoadCompactBinaryObject(Payload); + } + break; + } + + RpcAcceptOptions OriginalAcceptOptions = static_cast(Request["AcceptFlags"sv].AsUInt16(0u)); + int OriginalProcessPid = Request["Pid"sv].AsInt32(0); + + int AdjustedPid = 0; + RpcAcceptOptions AdjustedAcceptOptions = RpcAcceptOptions::kNone; + if (!m_DisableLocalRefs) + { + if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowLocalReferences) || m_ForceAllowLocalRefs) + { + AdjustedAcceptOptions |= RpcAcceptOptions::kAllowLocalReferences; + if (!m_DisablePartialLocalRefs) + { + if (EnumHasAnyFlags(OriginalAcceptOptions, RpcAcceptOptions::kAllowPartialLocalReferences) || + m_ForceAllowPartialLocalRefs) + { + AdjustedAcceptOptions |= RpcAcceptOptions::kAllowPartialLocalReferences; + } + } + if (!m_DisableLocalHandleRefs) + { + if (OriginalProcessPid != 0 || m_ForceAllowLocalHandleRef) + { + AdjustedPid = GetCurrentProcessId(); + } + } + } + } + + if (m_ShowMethodStats) + { + std::string MethodName = std::string(Request["Method"sv].AsString()); + RwLock::ExclusiveLockScope __(MethodTypesLock); + if (auto It = MethodTypes.find(MethodName); It != MethodTypes.end()) + { + It->second++; + } + else + { + MethodTypes[MethodName] = 1; + } + } + + if (OriginalAcceptOptions != AdjustedAcceptOptions || OriginalProcessPid != AdjustedPid) + { + CbObjectWriter RequestCopyWriter; + for (const CbFieldView& Field : Request) + { + if (!Field.HasName()) + { + RequestCopyWriter.AddField(Field); + continue; + } + std::string_view FieldName = Field.GetName(); + if (FieldName == "Pid"sv) + { + continue; + } + if (FieldName == "AcceptFlags"sv) + { + continue; + } + RequestCopyWriter.AddField(FieldName, Field); + } + if (AdjustedPid != 0) + { + RequestCopyWriter.AddInteger("Pid"sv, AdjustedPid); + } + if (AdjustedAcceptOptions != RpcAcceptOptions::kNone) + { + RequestCopyWriter.AddInteger("AcceptFlags"sv, static_cast(AdjustedAcceptOptions)); + } + + if (RequestContentType == ZenContentType::kCbPackage) + { + RequestPackage.SetObject(RequestCopyWriter.Save()); + std::vector Buffers = FormatPackageMessage(RequestPackage); + std::vector SharedBuffers(Buffers.begin(), Buffers.end()); + Payload = CompositeBuffer(std::move(SharedBuffers)).Flatten().AsIoBuffer(); + } + else + { + RequestCopyWriter.Finalize(); + Payload = IoBuffer(RequestCopyWriter.GetSaveSize()); + RequestCopyWriter.Save(Payload.GetMutableView()); + } + } + + Session.SetHeader({{"Content-Type", std::string(MapContentTypeToString(RequestContentType))}, + {"Accept", std::string(MapContentTypeToString(AcceptContentType))}}); + uint64_t Offset = 0; + auto ReadCallback = [&Payload, &Offset](char* buffer, size_t& size, intptr_t) { + size = Min(size, Payload.GetSize() - Offset); + IoBuffer PayloadRange = IoBuffer(Payload, Offset, size); + MutableMemoryView Data(buffer, size); + Data.CopyFrom(PayloadRange.GetView()); + Offset += size; + return true; + }; + Session.SetReadCallback(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); + cpr::Response Response = Session.Post(); + BytesSent.fetch_add(Payload.GetSize()); + if (Response.error || !(IsHttpSuccessCode(Response.status_code) || + Response.status_code == gsl::narrow(HttpResponseCode::NotFound))) + { + ZEN_CONSOLE("{}", FormatHttpResponse(Response)); + break; + } + BytesReceived.fetch_add(Response.downloaded_bytes); + EntryIndex = EntryOffset.fetch_add(m_Stride); + } + }); + } + + while (!WorkLatch.Wait(1000)) + { + ZEN_CONSOLE("Processing {} requests, {} remaining (sent {}, recevied {})...", + (EntryCount - m_Offset) / m_Stride, + (EntryCount - EntryOffset.load()) / m_Stride, + NiceBytes(BytesSent.load()), + NiceBytes(BytesReceived.load())); + } + if (m_ShowMethodStats) + { + for (const auto& It : MethodTypes) + { + ZEN_CONSOLE("{}: {}", It.first, It.second); + } + } + } + + const uint64_t RequestsSent = (EntryOffset.load() - m_Offset) / m_Stride; + const uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); + const double ElapsedS = ElapsedMS / 1000.500; + const uint64_t Sent = BytesSent.load(); + const uint64_t Received = BytesReceived.load(); + const uint64_t RequestsPerS = static_cast(RequestsSent / ElapsedS); + const uint64_t SentPerS = static_cast(Sent / ElapsedS); + const uint64_t ReceivedPerS = static_cast(Received / ElapsedS); + + ZEN_CONSOLE("Requests sent {} ({}/s), payloads sent {}B ({}B/s), payloads received {}B ({}B/s) in {}", + RequestsSent, + RequestsPerS, + NiceBytes(Sent), + NiceBytes(SentPerS), + NiceBytes(Received), + NiceBytes(ReceivedPerS), + NiceTimeSpanMs(ElapsedMS)); + + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/rpcreplay_cmd.h b/src/zen/cmds/rpcreplay_cmd.h new file mode 100644 index 000000000..742e5ec5b --- /dev/null +++ b/src/zen/cmds/rpcreplay_cmd.h @@ -0,0 +1,65 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +class RpcStartRecordingCommand : public ZenCmdBase +{ +public: + RpcStartRecordingCommand(); + ~RpcStartRecordingCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"rpc-record-start", "Starts recording of cache rpc requests on a host"}; + std::string m_HostName; + std::string m_RecordingPath; +}; + +class RpcStopRecordingCommand : public ZenCmdBase +{ +public: + RpcStopRecordingCommand(); + ~RpcStopRecordingCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"rpc-record-stop", "Stops recording of cache rpc requests on a host"}; + std::string m_HostName; +}; + +class RpcReplayCommand : public ZenCmdBase +{ +public: + RpcReplayCommand(); + ~RpcReplayCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"rpc-record-replay", "Replays a previously recorded session of cache rpc requests to a target host"}; + std::string m_HostName; + std::string m_RecordingPath; + bool m_OnHost = false; + bool m_ShowMethodStats = false; + int m_ProcessCount; + int m_ThreadCount; + uint64_t m_Offset; + uint64_t m_Stride; + bool m_ForceAllowLocalRefs; + bool m_DisableLocalRefs; + bool m_ForceAllowLocalHandleRef; + bool m_DisableLocalHandleRefs; + bool m_ForceAllowPartialLocalRefs; + bool m_DisablePartialLocalRefs; +}; + +} // namespace zen diff --git a/src/zen/cmds/scrub.cpp b/src/zen/cmds/scrub.cpp deleted file mode 100644 index 4b47082a0..000000000 --- a/src/zen/cmds/scrub.cpp +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "scrub.h" -#include -#include -#include - -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END - -using namespace std::literals; - -namespace zen { - -ScrubCommand::ScrubCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); -} - -ScrubCommand::~ScrubCommand() = default; - -int -ScrubCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - zen::HttpClient Http(m_HostName); - - if (zen::HttpClient::Response Response = Http.Post("/admin/scrub"sv)) - { - ZEN_CONSOLE("OK: {}", Response.ToText()); - - return 0; - } - else if (int StatusCode = (int)Response.StatusCode) - { - ZEN_ERROR("scrub start failed: {}: {} ({})", - (int)Response.StatusCode, - ReasonStringForHttpResultCode((int)Response.StatusCode), - Response.AsText()); - } - else - { - ZEN_ERROR("scrub start failed: {}", Response.AsText()); - } - - return 1; -} - -////////////////////////////////////////////////////////////////////////// - -GcCommand::GcCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", - "s", - "smallobjects", - "Collect small objects", - cxxopts::value(m_SmallObjects)->default_value("false"), - ""); - m_Options.add_option("", - "m", - "maxcacheduration", - "Max cache lifetime (in seconds)", - cxxopts::value(m_MaxCacheDuration)->default_value("0"), - ""); - m_Options.add_option("", - "d", - "disksizesoftlimit", - "Max disk usage size (in bytes)", - cxxopts::value(m_DiskSizeSoftLimit)->default_value("0"), - ""); -} - -GcCommand::~GcCommand() -{ -} - -int -GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - cpr::Parameters Params; - if (m_SmallObjects) - { - Params.Add({"smallobjects", "true"}); - } - if (m_MaxCacheDuration != 0) - { - Params.Add({"maxcacheduration", fmt::format("{}", m_MaxCacheDuration)}); - } - if (m_DiskSizeSoftLimit != 0) - { - Params.Add({"disksizesoftlimit", fmt::format("{}", m_DiskSizeSoftLimit)}); - } - - cpr::Session Session; - Session.SetHeader(cpr::Header{{"Accept", "application/json"}}); - Session.SetUrl({fmt::format("{}/admin/gc", m_HostName)}); - Session.SetParameters(Params); - - cpr::Response Result = Session.Post(); - - if (zen::IsHttpSuccessCode(Result.status_code)) - { - ZEN_CONSOLE("OK: {}", Result.text); - return 0; - } - - if (Result.status_code) - { - ZEN_ERROR("GC start failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); - } - else - { - ZEN_ERROR("GC start failed: {}", Result.error.message); - } - - return 1; -} - -GcStatusCommand::GcStatusCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); -} - -GcStatusCommand::~GcStatusCommand() -{ -} - -int -GcStatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - cpr::Session Session; - Session.SetHeader(cpr::Header{{"Accept", "application/json"}}); - Session.SetUrl({fmt::format("{}/admin/gc", m_HostName)}); - - cpr::Response Result = Session.Get(); - - if (zen::IsHttpSuccessCode(Result.status_code)) - { - ZEN_CONSOLE("OK: {}", Result.text); - return 0; - } - - if (Result.status_code) - { - ZEN_ERROR("GC status failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); - } - else - { - ZEN_ERROR("GC status failed: {}", Result.error.message); - } - - return 1; -} - -} // namespace zen diff --git a/src/zen/cmds/scrub.h b/src/zen/cmds/scrub.h deleted file mode 100644 index ee8b4fdbb..000000000 --- a/src/zen/cmds/scrub.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -/** Scrub storage - */ -class ScrubCommand : public ZenCmdBase -{ -public: - ScrubCommand(); - ~ScrubCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"scrub", "Scrub zen storage"}; - std::string m_HostName; -}; - -/** Garbage collect storage - */ -class GcCommand : public ZenCmdBase -{ -public: - GcCommand(); - ~GcCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"gc", "Garbage collect zen storage"}; - std::string m_HostName; - bool m_SmallObjects{false}; - uint64_t m_MaxCacheDuration{0}; - uint64_t m_DiskSizeSoftLimit{0}; -}; - -class GcStatusCommand : public ZenCmdBase -{ -public: - GcStatusCommand(); - ~GcStatusCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"gc-status", "Garbage collect zen storage status check"}; - std::string m_HostName; -}; - -} // namespace zen diff --git a/src/zen/cmds/serve.cpp b/src/zen/cmds/serve.cpp deleted file mode 100644 index 72afc105d..000000000 --- a/src/zen/cmds/serve.cpp +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "serve.h" - -#include -#include -#include -#include -#include -#include -#include - -#if ZEN_PLATFORM_WINDOWS -# include // TEMPORARY HACK -#endif - -namespace zen { - -using namespace std::literals; - -ServeCommand::ServeCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), ""); - m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), ""); - m_Options.add_option("", "", "path", "Root path to directory", cxxopts::value(m_RootPath), ""); - - m_Options.parse_positional({"project", "path"}); - m_Options.positional_help("[ ]"); -} - -ServeCommand::~ServeCommand() -{ -} - -int -ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - if (m_ProjectName.empty()) - { - throw zen::OptionParseException("command requires a project"); - } - - if (m_OplogName.empty()) - { - if (auto pos = m_ProjectName.find_first_of('/'); pos != std::string::npos) - { - m_OplogName = m_ProjectName.substr(pos + 1); - m_ProjectName = m_ProjectName.substr(0, pos); - } - else - { - throw zen::OptionParseException("command requires an oplog"); - } - } - - if (m_RootPath.empty()) - { - throw zen::OptionParseException("command requires a root path"); - } - - if (!std::filesystem::exists(m_RootPath) || !std::filesystem::is_directory(m_RootPath)) - { - throw zen::OptionParseException(fmt::format("path must exist and must be a directory: '{}'", m_RootPath)); - } - - uint16_t ServerPort = 0; - m_HostName = ResolveTargetHostSpec(m_HostName, ServerPort); - - ZenServerEnvironment ServerEnvironment; - std::optional ServerInstance; - - if (m_HostName.empty()) - { - // Spawn a server - - try - { - std::filesystem::path ExePath = zen::GetRunningExecutablePath(); - - ServerEnvironment.Initialize(ExePath.parent_path()); - ServerInstance.emplace(ServerEnvironment); - ServerInstance->SetOwnerPid(zen::GetCurrentProcessId()); - ServerInstance->SpawnServerAndWait(ServerPort); - } - catch (std::exception& Ex) - { - ZEN_CONSOLE("failed to spawn server on port {}: '{}'", ServerPort, Ex.what()); - - throw zen::OptionParseException("unable to resolve server specification (even after spawning server)"); - } - } - else - { - std::filesystem::path ExePath = zen::GetRunningExecutablePath(); - - ServerEnvironment.Initialize(ExePath.parent_path()); - ServerInstance.emplace(ServerEnvironment); - ServerInstance->DisableShutdownOnDestroy(); - ServerInstance->AttachToRunningServer(); - } - - if (ServerInstance) - { - m_HostName = ServerInstance->GetBaseUri(); - ZEN_CONSOLE("base uri: {}", m_HostName); - } - - // Generate manifest for tree - - FileSystemTraversal Traversal; - - struct FsVisitor : public FileSystemTraversal::TreeVisitor - { - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override - { - std::filesystem::path ServerPath = std::filesystem::relative(Parent / File, RootPath); - std::string ServerPathString = reinterpret_cast(ServerPath.generic_u8string().c_str()); - - if (ServerPathString.starts_with("./")) - { - ServerPathString = ServerPathString.substr(2); - } - - Files.emplace_back(FileEntry{ServerPathString, ServerPathString, FileSize}); - } - - virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; } - - struct FileEntry - { - std::string FilePath; - std::string ClientFilePath; - uint64_t FileSize; - }; - - std::filesystem::path RootPath; - std::vector Files; - }; - - FsVisitor Visitor; - Visitor.RootPath = m_RootPath; - Traversal.TraverseFileSystem(m_RootPath, Visitor); - - CbObjectWriter Cbo; - - Cbo << "key" - << "file_manifest"; - - Cbo.BeginArray("files"); - - for (const FsVisitor::FileEntry& Entry : Visitor.Files) - { - ZEN_CONSOLE("file: {}", Entry.FilePath); - - Cbo.BeginObject(); - - BLAKE3 Hash = BLAKE3::HashMemory(Entry.ClientFilePath.data(), Entry.ClientFilePath.size()); - Hash.Hash[11] = 7; // FIoChunkType::ExternalFile - Oid FileChunkId = Oid::FromMemory(Hash.Hash); - - Cbo << "id"sv << FileChunkId; - Cbo << "serverpath"sv << Entry.FilePath; - Cbo << "clientpath"sv << Entry.ClientFilePath; - - Cbo.EndObject(); - } - - Cbo.EndArray(); - - CbObject Manifest = Cbo.Save(); - - // Persist manifest - - const std::string ProjectUri = fmt::format("/prj/{}", m_ProjectName); - const std::string ProjectOplogUri = fmt::format("/prj/{}/oplog/{}", m_ProjectName, m_OplogName); - - HttpClient Client(m_HostName); - - // Ensure project exists - - if (HttpClient::Response ProjectResponse = Client.Get(ProjectUri); !ProjectResponse) - { - // Create project - - CbObjectWriter Project; - - Project << "root" << m_RootPath; - - if (auto NewProjectResponse = Client.Post(ProjectUri, Project.Save()); !NewProjectResponse) - { - // TODO: include details - throw std::runtime_error("failed to create project"); - } - } - - // Ensure oplog exists - - if (HttpClient::Response OplogResponse = Client.Get(ProjectOplogUri); !OplogResponse) - { - // Create oplog - - CbObjectWriter Oplog; - - if (auto NewOplogResponse = Client.Post(ProjectOplogUri, Oplog.Save()); !NewOplogResponse) - { - // TODO: include details - throw std::runtime_error("failed to create oplog"); - } - } - - // Append manifest - - const std::string Uri = fmt::format("/prj/{}/oplog/{}/new", m_ProjectName, m_OplogName); - - HttpClient::Response HttpResponse = Client.Post(Uri, Manifest); - - if (!HttpResponse) - { - ZEN_CONSOLE("error: failed to append manifest!"); - - return 1; - } - - ZEN_CONSOLE("ok serving files now"); - -#if ZEN_PLATFORM_WINDOWS - _getch(); // TEMPORARY HACK -#endif - - return 0; -} - -} // namespace zen diff --git a/src/zen/cmds/serve.h b/src/zen/cmds/serve.h deleted file mode 100644 index 007038d84..000000000 --- a/src/zen/cmds/serve.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -/** File serving - */ -class ServeCommand : public ZenCmdBase -{ -public: - ServeCommand(); - ~ServeCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"serve", "Serve files from a tree"}; - std::string m_HostName; - std::string m_ProjectName; - std::string m_OplogName; - std::string m_RootPath; -}; - -} // namespace zen diff --git a/src/zen/cmds/serve_cmd.cpp b/src/zen/cmds/serve_cmd.cpp new file mode 100644 index 000000000..c8117774b --- /dev/null +++ b/src/zen/cmds/serve_cmd.cpp @@ -0,0 +1,242 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "serve_cmd.h" + +#include +#include +#include +#include +#include +#include +#include + +#if ZEN_PLATFORM_WINDOWS +# include // TEMPORARY HACK +#endif + +namespace zen { + +using namespace std::literals; + +ServeCommand::ServeCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), ""); + m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), ""); + m_Options.add_option("", "", "path", "Root path to directory", cxxopts::value(m_RootPath), ""); + + m_Options.parse_positional({"project", "path"}); + m_Options.positional_help("[ ]"); +} + +ServeCommand::~ServeCommand() +{ +} + +int +ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + if (m_ProjectName.empty()) + { + throw zen::OptionParseException("command requires a project"); + } + + if (m_OplogName.empty()) + { + if (auto pos = m_ProjectName.find_first_of('/'); pos != std::string::npos) + { + m_OplogName = m_ProjectName.substr(pos + 1); + m_ProjectName = m_ProjectName.substr(0, pos); + } + else + { + throw zen::OptionParseException("command requires an oplog"); + } + } + + if (m_RootPath.empty()) + { + throw zen::OptionParseException("command requires a root path"); + } + + if (!std::filesystem::exists(m_RootPath) || !std::filesystem::is_directory(m_RootPath)) + { + throw zen::OptionParseException(fmt::format("path must exist and must be a directory: '{}'", m_RootPath)); + } + + uint16_t ServerPort = 0; + m_HostName = ResolveTargetHostSpec(m_HostName, ServerPort); + + ZenServerEnvironment ServerEnvironment; + std::optional ServerInstance; + + if (m_HostName.empty()) + { + // Spawn a server + + try + { + std::filesystem::path ExePath = zen::GetRunningExecutablePath(); + + ServerEnvironment.Initialize(ExePath.parent_path()); + ServerInstance.emplace(ServerEnvironment); + ServerInstance->SetOwnerPid(zen::GetCurrentProcessId()); + ServerInstance->SpawnServerAndWait(ServerPort); + } + catch (std::exception& Ex) + { + ZEN_CONSOLE("failed to spawn server on port {}: '{}'", ServerPort, Ex.what()); + + throw zen::OptionParseException("unable to resolve server specification (even after spawning server)"); + } + } + else + { + std::filesystem::path ExePath = zen::GetRunningExecutablePath(); + + ServerEnvironment.Initialize(ExePath.parent_path()); + ServerInstance.emplace(ServerEnvironment); + ServerInstance->DisableShutdownOnDestroy(); + ServerInstance->AttachToRunningServer(); + } + + if (ServerInstance) + { + m_HostName = ServerInstance->GetBaseUri(); + ZEN_CONSOLE("base uri: {}", m_HostName); + } + + // Generate manifest for tree + + FileSystemTraversal Traversal; + + struct FsVisitor : public FileSystemTraversal::TreeVisitor + { + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override + { + std::filesystem::path ServerPath = std::filesystem::relative(Parent / File, RootPath); + std::string ServerPathString = reinterpret_cast(ServerPath.generic_u8string().c_str()); + + if (ServerPathString.starts_with("./")) + { + ServerPathString = ServerPathString.substr(2); + } + + Files.emplace_back(FileEntry{ServerPathString, ServerPathString, FileSize}); + } + + virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; } + + struct FileEntry + { + std::string FilePath; + std::string ClientFilePath; + uint64_t FileSize; + }; + + std::filesystem::path RootPath; + std::vector Files; + }; + + FsVisitor Visitor; + Visitor.RootPath = m_RootPath; + Traversal.TraverseFileSystem(m_RootPath, Visitor); + + CbObjectWriter Cbo; + + Cbo << "key" + << "file_manifest"; + + Cbo.BeginArray("files"); + + for (const FsVisitor::FileEntry& Entry : Visitor.Files) + { + ZEN_CONSOLE("file: {}", Entry.FilePath); + + Cbo.BeginObject(); + + BLAKE3 Hash = BLAKE3::HashMemory(Entry.ClientFilePath.data(), Entry.ClientFilePath.size()); + Hash.Hash[11] = 7; // FIoChunkType::ExternalFile + Oid FileChunkId = Oid::FromMemory(Hash.Hash); + + Cbo << "id"sv << FileChunkId; + Cbo << "serverpath"sv << Entry.FilePath; + Cbo << "clientpath"sv << Entry.ClientFilePath; + + Cbo.EndObject(); + } + + Cbo.EndArray(); + + CbObject Manifest = Cbo.Save(); + + // Persist manifest + + const std::string ProjectUri = fmt::format("/prj/{}", m_ProjectName); + const std::string ProjectOplogUri = fmt::format("/prj/{}/oplog/{}", m_ProjectName, m_OplogName); + + HttpClient Client(m_HostName); + + // Ensure project exists + + if (HttpClient::Response ProjectResponse = Client.Get(ProjectUri); !ProjectResponse) + { + // Create project + + CbObjectWriter Project; + + Project << "root" << m_RootPath; + + if (auto NewProjectResponse = Client.Post(ProjectUri, Project.Save()); !NewProjectResponse) + { + // TODO: include details + throw std::runtime_error("failed to create project"); + } + } + + // Ensure oplog exists + + if (HttpClient::Response OplogResponse = Client.Get(ProjectOplogUri); !OplogResponse) + { + // Create oplog + + CbObjectWriter Oplog; + + if (auto NewOplogResponse = Client.Post(ProjectOplogUri, Oplog.Save()); !NewOplogResponse) + { + // TODO: include details + throw std::runtime_error("failed to create oplog"); + } + } + + // Append manifest + + const std::string Uri = fmt::format("/prj/{}/oplog/{}/new", m_ProjectName, m_OplogName); + + HttpClient::Response HttpResponse = Client.Post(Uri, Manifest); + + if (!HttpResponse) + { + ZEN_CONSOLE("error: failed to append manifest!"); + + return 1; + } + + ZEN_CONSOLE("ok serving files now"); + +#if ZEN_PLATFORM_WINDOWS + _getch(); // TEMPORARY HACK +#endif + + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/serve_cmd.h b/src/zen/cmds/serve_cmd.h new file mode 100644 index 000000000..007038d84 --- /dev/null +++ b/src/zen/cmds/serve_cmd.h @@ -0,0 +1,28 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +/** File serving + */ +class ServeCommand : public ZenCmdBase +{ +public: + ServeCommand(); + ~ServeCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"serve", "Serve files from a tree"}; + std::string m_HostName; + std::string m_ProjectName; + std::string m_OplogName; + std::string m_RootPath; +}; + +} // namespace zen diff --git a/src/zen/cmds/status.cpp b/src/zen/cmds/status.cpp deleted file mode 100644 index 1afe191b7..000000000 --- a/src/zen/cmds/status.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "status.h" - -#include -#include -#include -#include -#include - -namespace zen { - -StatusCommand::StatusCommand() -{ -} - -StatusCommand::~StatusCommand() = default; - -int -StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions, argc, argv); - - ZenServerState State; - if (!State.InitializeReadOnly()) - { - ZEN_CONSOLE("no Zen state found"); - - return 0; - } - - ZEN_CONSOLE("{:>5} {:>6} {:>24}", "port", "pid", "session"); - State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { - StringBuilder<25> SessionStringBuilder; - Entry.GetSessionId().ToString(SessionStringBuilder); - ZEN_CONSOLE("{:>5} {:>6} {:>24}", Entry.EffectiveListenPort.load(), Entry.Pid.load(), SessionStringBuilder); - }); - - return 0; -} - -} // namespace zen diff --git a/src/zen/cmds/status.h b/src/zen/cmds/status.h deleted file mode 100644 index 98f72e651..000000000 --- a/src/zen/cmds/status.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -class StatusCommand : public ZenCmdBase -{ -public: - StatusCommand(); - ~StatusCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"status", "Show zen status"}; -}; - -} // namespace zen diff --git a/src/zen/cmds/status_cmd.cpp b/src/zen/cmds/status_cmd.cpp new file mode 100644 index 000000000..cc936835a --- /dev/null +++ b/src/zen/cmds/status_cmd.cpp @@ -0,0 +1,42 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "status_cmd.h" + +#include +#include +#include +#include +#include + +namespace zen { + +StatusCommand::StatusCommand() +{ +} + +StatusCommand::~StatusCommand() = default; + +int +StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions, argc, argv); + + ZenServerState State; + if (!State.InitializeReadOnly()) + { + ZEN_CONSOLE("no Zen state found"); + + return 0; + } + + ZEN_CONSOLE("{:>5} {:>6} {:>24}", "port", "pid", "session"); + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + StringBuilder<25> SessionStringBuilder; + Entry.GetSessionId().ToString(SessionStringBuilder); + ZEN_CONSOLE("{:>5} {:>6} {:>24}", Entry.EffectiveListenPort.load(), Entry.Pid.load(), SessionStringBuilder); + }); + + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/status_cmd.h b/src/zen/cmds/status_cmd.h new file mode 100644 index 000000000..98f72e651 --- /dev/null +++ b/src/zen/cmds/status_cmd.h @@ -0,0 +1,22 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +class StatusCommand : public ZenCmdBase +{ +public: + StatusCommand(); + ~StatusCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"status", "Show zen status"}; +}; + +} // namespace zen diff --git a/src/zen/cmds/top.cpp b/src/zen/cmds/top.cpp deleted file mode 100644 index 6aed6fe1b..000000000 --- a/src/zen/cmds/top.cpp +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "top.h" - -#include -#include -#include -#include - -#include - -////////////////////////////////////////////////////////////////////////// - -namespace zen { - -TopCommand::TopCommand() -{ -} - -TopCommand::~TopCommand() = default; - -int -TopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions, argc, argv); - - ZenServerState State; - if (!State.InitializeReadOnly()) - { - ZEN_CONSOLE("no Zen state found"); - - return 0; - } - - int n = 0; - const int HeaderPeriod = 20; - - for (;;) - { - if ((n++ % HeaderPeriod) == 0) - { - ZEN_CONSOLE("{:>5} {:>6} {:>24}", "port", "pid", "session"); - } - - State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { - StringBuilder<25> SessionStringBuilder; - Entry.GetSessionId().ToString(SessionStringBuilder); - ZEN_CONSOLE("{:>5} {:>6} {:>24}", Entry.EffectiveListenPort.load(), Entry.Pid.load(), SessionStringBuilder); - }); - - zen::Sleep(1000); - - if (!State.IsReadOnly()) - { - State.Sweep(); - } - } - - return 0; -} - -////////////////////////////////////////////////////////////////////////// - -PsCommand::PsCommand() -{ -} - -PsCommand::~PsCommand() = default; - -int -PsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions, argc, argv); - - ZenServerState State; - if (!State.InitializeReadOnly()) - { - ZEN_CONSOLE("no Zen state found"); - - return 0; - } - - State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { - ZEN_CONSOLE("Port {} : pid {}", Entry.EffectiveListenPort.load(), Entry.Pid.load()); - }); - - return 0; -} - -} // namespace zen diff --git a/src/zen/cmds/top.h b/src/zen/cmds/top.h deleted file mode 100644 index 83410587b..000000000 --- a/src/zen/cmds/top.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -class TopCommand : public ZenCmdBase -{ -public: - TopCommand(); - ~TopCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"top", "Show dev UI"}; -}; - -class PsCommand : public ZenCmdBase -{ -public: - PsCommand(); - ~PsCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"ps", "Enumerate running Zen server instances"}; -}; - -} // namespace zen diff --git a/src/zen/cmds/top_cmd.cpp b/src/zen/cmds/top_cmd.cpp new file mode 100644 index 000000000..568ee76c9 --- /dev/null +++ b/src/zen/cmds/top_cmd.cpp @@ -0,0 +1,90 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "top_cmd.h" + +#include +#include +#include +#include + +#include + +////////////////////////////////////////////////////////////////////////// + +namespace zen { + +TopCommand::TopCommand() +{ +} + +TopCommand::~TopCommand() = default; + +int +TopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions, argc, argv); + + ZenServerState State; + if (!State.InitializeReadOnly()) + { + ZEN_CONSOLE("no Zen state found"); + + return 0; + } + + int n = 0; + const int HeaderPeriod = 20; + + for (;;) + { + if ((n++ % HeaderPeriod) == 0) + { + ZEN_CONSOLE("{:>5} {:>6} {:>24}", "port", "pid", "session"); + } + + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + StringBuilder<25> SessionStringBuilder; + Entry.GetSessionId().ToString(SessionStringBuilder); + ZEN_CONSOLE("{:>5} {:>6} {:>24}", Entry.EffectiveListenPort.load(), Entry.Pid.load(), SessionStringBuilder); + }); + + zen::Sleep(1000); + + if (!State.IsReadOnly()) + { + State.Sweep(); + } + } + + return 0; +} + +////////////////////////////////////////////////////////////////////////// + +PsCommand::PsCommand() +{ +} + +PsCommand::~PsCommand() = default; + +int +PsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions, argc, argv); + + ZenServerState State; + if (!State.InitializeReadOnly()) + { + ZEN_CONSOLE("no Zen state found"); + + return 0; + } + + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + ZEN_CONSOLE("Port {} : pid {}", Entry.EffectiveListenPort.load(), Entry.Pid.load()); + }); + + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/top_cmd.h b/src/zen/cmds/top_cmd.h new file mode 100644 index 000000000..83410587b --- /dev/null +++ b/src/zen/cmds/top_cmd.h @@ -0,0 +1,35 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +class TopCommand : public ZenCmdBase +{ +public: + TopCommand(); + ~TopCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"top", "Show dev UI"}; +}; + +class PsCommand : public ZenCmdBase +{ +public: + PsCommand(); + ~PsCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"ps", "Enumerate running Zen server instances"}; +}; + +} // namespace zen diff --git a/src/zen/cmds/trace.cpp b/src/zen/cmds/trace.cpp deleted file mode 100644 index f8968a680..000000000 --- a/src/zen/cmds/trace.cpp +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "trace.h" -#include -#include -#include - -using namespace std::literals; - -namespace zen { - -TraceCommand::TraceCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_option("", "s", "stop", "Stop tracing", cxxopts::value(m_Stop)->default_value("false"), ""); - m_Options.add_option("", "", "host", "Start tracing to host", cxxopts::value(m_TraceHost), ""); - m_Options.add_option("", "", "file", "Start tracing to file", cxxopts::value(m_TraceFile), ""); -} - -TraceCommand::~TraceCommand() = default; - -int -TraceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - zen::HttpClient Http(m_HostName); - - if (m_Stop) - { - if (zen::HttpClient::Response Response = Http.Post("/admin/trace/stop"sv)) - { - ZEN_CONSOLE("OK: {}", Response.ToText()); - return 0; - } - else - { - ZEN_ERROR("trace stop failed: {}", Response.AsText()); - return 1; - } - } - - std::string StartArg; - if (!m_TraceHost.empty()) - { - StartArg = fmt::format("host={}", m_TraceHost); - } - else if (!m_TraceFile.empty()) - { - StartArg = fmt::format("file={}", m_TraceFile); - } - - if (!StartArg.empty()) - { - if (zen::HttpClient::Response Response = Http.Post(fmt::format("/admin/trace/start?{}"sv, StartArg))) - { - ZEN_CONSOLE("OK: {}", Response.ToText()); - return 0; - } - else - { - ZEN_ERROR("trace start failed: {}", Response.AsText()); - return 1; - } - } - - if (zen::HttpClient::Response Response = Http.Get("/admin/trace"sv)) - { - ZEN_CONSOLE("OK: {}", Response.ToText()); - return 0; - } - else - { - ZEN_ERROR("trace status failed: {}", Response.AsText()); - } - - return 1; -} - -} // namespace zen diff --git a/src/zen/cmds/trace.h b/src/zen/cmds/trace.h deleted file mode 100644 index 7b2d15fb1..000000000 --- a/src/zen/cmds/trace.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -/** Scrub storage - */ -class TraceCommand : public ZenCmdBase -{ -public: - TraceCommand(); - ~TraceCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"trace", "Control zen realtime tracing"}; - std::string m_HostName; - bool m_Stop; - std::string m_TraceHost; - std::string m_TraceFile; -}; - -} // namespace zen diff --git a/src/zen/cmds/trace_cmd.cpp b/src/zen/cmds/trace_cmd.cpp new file mode 100644 index 000000000..fee4dd6bc --- /dev/null +++ b/src/zen/cmds/trace_cmd.cpp @@ -0,0 +1,93 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "trace_cmd.h" +#include +#include +#include + +using namespace std::literals; + +namespace zen { + +TraceCommand::TraceCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", "s", "stop", "Stop tracing", cxxopts::value(m_Stop)->default_value("false"), ""); + m_Options.add_option("", "", "host", "Start tracing to host", cxxopts::value(m_TraceHost), ""); + m_Options.add_option("", "", "file", "Start tracing to file", cxxopts::value(m_TraceFile), ""); +} + +TraceCommand::~TraceCommand() = default; + +int +TraceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + zen::HttpClient Http(m_HostName); + + if (m_Stop) + { + if (zen::HttpClient::Response Response = Http.Post("/admin/trace/stop"sv)) + { + ZEN_CONSOLE("OK: {}", Response.ToText()); + return 0; + } + else + { + ZEN_ERROR("trace stop failed: {}", Response.AsText()); + return 1; + } + } + + std::string StartArg; + if (!m_TraceHost.empty()) + { + StartArg = fmt::format("host={}", m_TraceHost); + } + else if (!m_TraceFile.empty()) + { + StartArg = fmt::format("file={}", m_TraceFile); + } + + if (!StartArg.empty()) + { + if (zen::HttpClient::Response Response = Http.Post(fmt::format("/admin/trace/start?{}"sv, StartArg))) + { + ZEN_CONSOLE("OK: {}", Response.ToText()); + return 0; + } + else + { + ZEN_ERROR("trace start failed: {}", Response.AsText()); + return 1; + } + } + + if (zen::HttpClient::Response Response = Http.Get("/admin/trace"sv)) + { + ZEN_CONSOLE("OK: {}", Response.ToText()); + return 0; + } + else + { + ZEN_ERROR("trace status failed: {}", Response.AsText()); + } + + return 1; +} + +} // namespace zen diff --git a/src/zen/cmds/trace_cmd.h b/src/zen/cmds/trace_cmd.h new file mode 100644 index 000000000..7b2d15fb1 --- /dev/null +++ b/src/zen/cmds/trace_cmd.h @@ -0,0 +1,28 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +/** Scrub storage + */ +class TraceCommand : public ZenCmdBase +{ +public: + TraceCommand(); + ~TraceCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"trace", "Control zen realtime tracing"}; + std::string m_HostName; + bool m_Stop; + std::string m_TraceHost; + std::string m_TraceFile; +}; + +} // namespace zen diff --git a/src/zen/cmds/up.cpp b/src/zen/cmds/up.cpp deleted file mode 100644 index d1ae0794a..000000000 --- a/src/zen/cmds/up.cpp +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "up.h" - -#include -#include -#include - -#include - -namespace zen { - -UpCommand::UpCommand() -{ - m_Options.add_option("lifetime", - "", - "owner-pid", - "Specify owning process id", - cxxopts::value(m_OwnerPid)->default_value("0"), - ""); - m_Options.add_options()("config", "Path to Lua config file", cxxopts::value(m_ConfigFile)); -} - -UpCommand::~UpCommand() = default; - -int -UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions, argc, argv); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - { - ZenServerState State; - if (State.InitializeReadOnly()) - { - ZEN_CONSOLE("Zen server already running"); - return 0; - } - } - - std::filesystem::path ExePath = zen::GetRunningExecutablePath(); - ZenServerEnvironment ServerEnvironment; - ServerEnvironment.Initialize(ExePath.parent_path()); - ZenServerInstance Server(ServerEnvironment); - if (m_OwnerPid != 0) - { - Server.SetOwnerPid(m_OwnerPid); - } - std::string AdditionalArguments; - if (!m_ConfigFile.empty()) - { - AdditionalArguments = fmt::format("--config {}", m_ConfigFile); - } - Server.SpawnServer(0, AdditionalArguments); - - int Timeout = 10000; - - if (!Server.WaitUntilReady(Timeout)) - { - ZEN_ERROR("zen server launch failed (timed out)"); - } - else - { - ZEN_CONSOLE("zen server up"); - } - - return 0; -} - -////////////////////////////////////////////////////////////////////////// - -AttachCommand::AttachCommand() -{ - m_Options.add_option("", "p", "port", "Host port", cxxopts::value(m_Port)->default_value("1337"), ""); - m_Options.add_option("lifetime", "", "owner-pid", "Specify owning process id", cxxopts::value(m_OwnerPid), ""); -} - -AttachCommand::~AttachCommand() = default; - -int -AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions, argc, argv); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - ZenServerState Instance; - Instance.Initialize(); - ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port); - if (!Entry) - { - ZEN_WARN("no zen server instance to add sponsor process to"); - return 1; - } - - if (!Entry->AddSponsorProcess(m_OwnerPid)) - { - ZEN_WARN("unable to add sponsor process to running zen server instance"); - return 1; - } - - ZEN_CONSOLE("added sponsor process {} to running instance {} on port {}", m_OwnerPid, Entry->Pid.load(), m_Port); - return 0; -} - -////////////////////////////////////////////////////////////////////////// - -DownCommand::DownCommand() -{ - m_Options.add_option("", "p", "port", "Host port", cxxopts::value(m_Port)->default_value("1337"), ""); -} - -DownCommand::~DownCommand() = default; - -int -DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - // Discover executing instances - - ZenServerState Instance; - Instance.Initialize(); - ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port); - - if (!Entry) - { - ZEN_WARN("no zen server to bring down"); - - return 0; - } - - try - { - std::filesystem::path ExePath = zen::GetRunningExecutablePath(); - - ZenServerEnvironment ServerEnvironment; - ServerEnvironment.Initialize(ExePath.parent_path()); - ZenServerInstance Server(ServerEnvironment); - Server.AttachToRunningServer(m_Port); - - ZEN_CONSOLE("attached to server on port {}, requesting shutdown", m_Port); - - Server.Shutdown(); - - ZEN_CONSOLE("shutdown complete"); - - return 0; - } - catch (std::exception& Ex) - { - ZEN_DEBUG("Exception caught when requesting shutdown: {}", Ex.what()); - } - - // Since we cannot obtain a handle to the process we are unable to block on the process - // handle to determine when the server has shut down. Thus we signal that we would like - // a shutdown via the shutdown flag and then exit. - - ZEN_CONSOLE("requesting shutdown of server on port {}", m_Port); - Entry->SignalShutdownRequest(); - - return 0; -} - -} // namespace zen diff --git a/src/zen/cmds/up.h b/src/zen/cmds/up.h deleted file mode 100644 index 510cc865e..000000000 --- a/src/zen/cmds/up.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -class UpCommand : public ZenCmdBase -{ -public: - UpCommand(); - ~UpCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"up", "Bring up zen service"}; - uint16_t m_Port = 0; - int m_OwnerPid = 0; - std::string m_ConfigFile; -}; - -class AttachCommand : public ZenCmdBase -{ -public: - AttachCommand(); - ~AttachCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"}; - uint16_t m_Port; - int m_OwnerPid; -}; - -class DownCommand : public ZenCmdBase -{ -public: - DownCommand(); - ~DownCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"down", "Bring down zen service"}; - uint16_t m_Port; -}; - -} // namespace zen diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp new file mode 100644 index 000000000..b07fb6ec8 --- /dev/null +++ b/src/zen/cmds/up_cmd.cpp @@ -0,0 +1,176 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "up_cmd.h" + +#include +#include +#include + +#include + +namespace zen { + +UpCommand::UpCommand() +{ + m_Options.add_option("lifetime", + "", + "owner-pid", + "Specify owning process id", + cxxopts::value(m_OwnerPid)->default_value("0"), + ""); + m_Options.add_options()("config", "Path to Lua config file", cxxopts::value(m_ConfigFile)); +} + +UpCommand::~UpCommand() = default; + +int +UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions, argc, argv); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + { + ZenServerState State; + if (State.InitializeReadOnly()) + { + ZEN_CONSOLE("Zen server already running"); + return 0; + } + } + + std::filesystem::path ExePath = zen::GetRunningExecutablePath(); + ZenServerEnvironment ServerEnvironment; + ServerEnvironment.Initialize(ExePath.parent_path()); + ZenServerInstance Server(ServerEnvironment); + if (m_OwnerPid != 0) + { + Server.SetOwnerPid(m_OwnerPid); + } + std::string AdditionalArguments; + if (!m_ConfigFile.empty()) + { + AdditionalArguments = fmt::format("--config {}", m_ConfigFile); + } + Server.SpawnServer(0, AdditionalArguments); + + int Timeout = 10000; + + if (!Server.WaitUntilReady(Timeout)) + { + ZEN_ERROR("zen server launch failed (timed out)"); + } + else + { + ZEN_CONSOLE("zen server up"); + } + + return 0; +} + +////////////////////////////////////////////////////////////////////////// + +AttachCommand::AttachCommand() +{ + m_Options.add_option("", "p", "port", "Host port", cxxopts::value(m_Port)->default_value("1337"), ""); + m_Options.add_option("lifetime", "", "owner-pid", "Specify owning process id", cxxopts::value(m_OwnerPid), ""); +} + +AttachCommand::~AttachCommand() = default; + +int +AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions, argc, argv); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + ZenServerState Instance; + Instance.Initialize(); + ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port); + if (!Entry) + { + ZEN_WARN("no zen server instance to add sponsor process to"); + return 1; + } + + if (!Entry->AddSponsorProcess(m_OwnerPid)) + { + ZEN_WARN("unable to add sponsor process to running zen server instance"); + return 1; + } + + ZEN_CONSOLE("added sponsor process {} to running instance {} on port {}", m_OwnerPid, Entry->Pid.load(), m_Port); + return 0; +} + +////////////////////////////////////////////////////////////////////////// + +DownCommand::DownCommand() +{ + m_Options.add_option("", "p", "port", "Host port", cxxopts::value(m_Port)->default_value("1337"), ""); +} + +DownCommand::~DownCommand() = default; + +int +DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + // Discover executing instances + + ZenServerState Instance; + Instance.Initialize(); + ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port); + + if (!Entry) + { + ZEN_WARN("no zen server to bring down"); + + return 0; + } + + try + { + std::filesystem::path ExePath = zen::GetRunningExecutablePath(); + + ZenServerEnvironment ServerEnvironment; + ServerEnvironment.Initialize(ExePath.parent_path()); + ZenServerInstance Server(ServerEnvironment); + Server.AttachToRunningServer(m_Port); + + ZEN_CONSOLE("attached to server on port {}, requesting shutdown", m_Port); + + Server.Shutdown(); + + ZEN_CONSOLE("shutdown complete"); + + return 0; + } + catch (std::exception& Ex) + { + ZEN_DEBUG("Exception caught when requesting shutdown: {}", Ex.what()); + } + + // Since we cannot obtain a handle to the process we are unable to block on the process + // handle to determine when the server has shut down. Thus we signal that we would like + // a shutdown via the shutdown flag and then exit. + + ZEN_CONSOLE("requesting shutdown of server on port {}", m_Port); + Entry->SignalShutdownRequest(); + + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h new file mode 100644 index 000000000..510cc865e --- /dev/null +++ b/src/zen/cmds/up_cmd.h @@ -0,0 +1,54 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +class UpCommand : public ZenCmdBase +{ +public: + UpCommand(); + ~UpCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"up", "Bring up zen service"}; + uint16_t m_Port = 0; + int m_OwnerPid = 0; + std::string m_ConfigFile; +}; + +class AttachCommand : public ZenCmdBase +{ +public: + AttachCommand(); + ~AttachCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"}; + uint16_t m_Port; + int m_OwnerPid; +}; + +class DownCommand : public ZenCmdBase +{ +public: + DownCommand(); + ~DownCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"down", "Bring down zen service"}; + uint16_t m_Port; +}; + +} // namespace zen diff --git a/src/zen/cmds/version.cpp b/src/zen/cmds/version.cpp deleted file mode 100644 index ba83b527d..000000000 --- a/src/zen/cmds/version.cpp +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "version.h" - -#include -#include -#include -#include -#include -#include - -#include - -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END - -namespace zen { - -VersionCommand::VersionCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName), "[hosturl]"); - m_Options.add_option("", "d", "detailed", "Detailed Version", cxxopts::value(m_DetailedVersion), "[detailedversion]"); - m_Options.parse_positional({"hosturl"}); -} - -VersionCommand::~VersionCommand() = default; - -int -VersionCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - if (!ParseOptions(argc, argv)) - { - return 0; - } - - std::string Version; - - if (m_HostName.empty()) - { - if (m_DetailedVersion) - { - Version = ZEN_CFG_VERSION_BUILD_STRING_FULL; - } - else - { - Version = ZEN_CFG_VERSION; - } - } - else - { - const std::string UrlBase = fmt::format("{}/health", m_HostName); - cpr::Session Session; - std::string VersionRequest = fmt::format("{}/version{}", UrlBase, m_DetailedVersion ? "?detailed=true" : ""); - Session.SetUrl(VersionRequest); - cpr::Response Response = Session.Get(); - if (!zen::IsHttpSuccessCode(Response.status_code)) - { - if (Response.status_code) - { - ZEN_ERROR("{} failed: {}: {} ({})", VersionRequest, Response.status_code, Response.reason, Response.text); - } - else - { - ZEN_ERROR("{} failed: {}", VersionRequest, Response.error.message); - } - - return 1; - } - Version = Response.text; - } - - zen::ConsoleLog().info("{}", Version); - - return 0; -} -} // namespace zen diff --git a/src/zen/cmds/version.h b/src/zen/cmds/version.h deleted file mode 100644 index 0e37e91a0..000000000 --- a/src/zen/cmds/version.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -namespace zen { - -class VersionCommand : public ZenCmdBase -{ -public: - VersionCommand(); - ~VersionCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"version", "Get zen service version"}; - std::string m_HostName; - bool m_DetailedVersion; -}; - -} // namespace zen diff --git a/src/zen/cmds/version_cmd.cpp b/src/zen/cmds/version_cmd.cpp new file mode 100644 index 000000000..bd31862b4 --- /dev/null +++ b/src/zen/cmds/version_cmd.cpp @@ -0,0 +1,79 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "version_cmd.h" + +#include +#include +#include +#include +#include +#include + +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +VersionCommand::VersionCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName), "[hosturl]"); + m_Options.add_option("", "d", "detailed", "Detailed Version", cxxopts::value(m_DetailedVersion), "[detailedversion]"); + m_Options.parse_positional({"hosturl"}); +} + +VersionCommand::~VersionCommand() = default; + +int +VersionCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + if (!ParseOptions(argc, argv)) + { + return 0; + } + + std::string Version; + + if (m_HostName.empty()) + { + if (m_DetailedVersion) + { + Version = ZEN_CFG_VERSION_BUILD_STRING_FULL; + } + else + { + Version = ZEN_CFG_VERSION; + } + } + else + { + const std::string UrlBase = fmt::format("{}/health", m_HostName); + cpr::Session Session; + std::string VersionRequest = fmt::format("{}/version{}", UrlBase, m_DetailedVersion ? "?detailed=true" : ""); + Session.SetUrl(VersionRequest); + cpr::Response Response = Session.Get(); + if (!zen::IsHttpSuccessCode(Response.status_code)) + { + if (Response.status_code) + { + ZEN_ERROR("{} failed: {}: {} ({})", VersionRequest, Response.status_code, Response.reason, Response.text); + } + else + { + ZEN_ERROR("{} failed: {}", VersionRequest, Response.error.message); + } + + return 1; + } + Version = Response.text; + } + + zen::ConsoleLog().info("{}", Version); + + return 0; +} +} // namespace zen diff --git a/src/zen/cmds/version_cmd.h b/src/zen/cmds/version_cmd.h new file mode 100644 index 000000000..0e37e91a0 --- /dev/null +++ b/src/zen/cmds/version_cmd.h @@ -0,0 +1,24 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +class VersionCommand : public ZenCmdBase +{ +public: + VersionCommand(); + ~VersionCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"version", "Get zen service version"}; + std::string m_HostName; + bool m_DetailedVersion; +}; + +} // namespace zen diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 8449c7a43..23dce850f 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -5,22 +5,21 @@ #include "zen.h" -#include "cmds/bench.h" -#include "cmds/cache.h" -#include "cmds/copy.h" -#include "cmds/dedup.h" -#include "cmds/hash.h" -#include "cmds/jobs.h" -#include "cmds/print.h" -#include "cmds/projectstore.h" -#include "cmds/rpcreplay.h" -#include "cmds/scrub.h" -#include "cmds/serve.h" -#include "cmds/status.h" -#include "cmds/top.h" -#include "cmds/trace.h" -#include "cmds/up.h" -#include "cmds/version.h" +#include "cmds/admin_cmd.h" +#include "cmds/bench_cmd.h" +#include "cmds/cache_cmd.h" +#include "cmds/copy_cmd.h" +#include "cmds/dedup_cmd.h" +#include "cmds/hash_cmd.h" +#include "cmds/print_cmd.h" +#include "cmds/projectstore_cmd.h" +#include "cmds/rpcreplay_cmd.h" +#include "cmds/serve_cmd.h" +#include "cmds/status_cmd.h" +#include "cmds/top_cmd.h" +#include "cmds/trace_cmd.h" +#include "cmds/up_cmd.h" +#include "cmds/version_cmd.h" #include "cmds/vfs_cmd.h" #include -- cgit v1.2.3