diff options
| author | Stefan Boberg <[email protected]> | 2023-09-26 09:54:00 +0200 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2023-09-26 09:54:00 +0200 |
| commit | 6b23bf09acd11f50ec224297ee69bef15cad39ee (patch) | |
| tree | 8b2bcfe89eb5ffd71cae323dc62c4881024aa876 /src | |
| parent | sort commands for cleaner merges (diff) | |
| parent | 0.2.24 (diff) | |
| download | zen-6b23bf09acd11f50ec224297ee69bef15cad39ee.tar.xz zen-6b23bf09acd11f50ec224297ee69bef15cad39ee.zip | |
Merge branch 'main' of https://github.com/EpicGames/zen
Diffstat (limited to 'src')
| -rw-r--r-- | src/zen/cmds/admin_cmd.cpp (renamed from src/zen/cmds/scrub.cpp) | 173 | ||||
| -rw-r--r-- | src/zen/cmds/admin_cmd.h (renamed from src/zen/cmds/scrub.h) | 37 | ||||
| -rw-r--r-- | src/zen/cmds/bench_cmd.cpp (renamed from src/zen/cmds/bench.cpp) | 2 | ||||
| -rw-r--r-- | src/zen/cmds/bench_cmd.h (renamed from src/zen/cmds/bench.h) | 0 | ||||
| -rw-r--r-- | src/zen/cmds/cache_cmd.cpp (renamed from src/zen/cmds/cache.cpp) | 2 | ||||
| -rw-r--r-- | src/zen/cmds/cache_cmd.h (renamed from src/zen/cmds/cache.h) | 0 | ||||
| -rw-r--r-- | src/zen/cmds/copy_cmd.cpp (renamed from src/zen/cmds/copy.cpp) | 2 | ||||
| -rw-r--r-- | src/zen/cmds/copy_cmd.h (renamed from src/zen/cmds/copy.h) | 0 | ||||
| -rw-r--r-- | src/zen/cmds/dedup_cmd.cpp (renamed from src/zen/cmds/dedup.cpp) | 2 | ||||
| -rw-r--r-- | src/zen/cmds/dedup_cmd.h (renamed from src/zen/cmds/dedup.h) | 0 | ||||
| -rw-r--r-- | src/zen/cmds/hash_cmd.cpp (renamed from src/zen/cmds/hash.cpp) | 2 | ||||
| -rw-r--r-- | src/zen/cmds/hash_cmd.h (renamed from src/zen/cmds/hash.h) | 0 | ||||
| -rw-r--r-- | src/zen/cmds/jobs.cpp | 82 | ||||
| -rw-r--r-- | src/zen/cmds/print_cmd.cpp (renamed from src/zen/cmds/print.cpp) | 2 | ||||
| -rw-r--r-- | src/zen/cmds/print_cmd.h (renamed from src/zen/cmds/print.h) | 0 | ||||
| -rw-r--r-- | src/zen/cmds/projectstore_cmd.cpp (renamed from src/zen/cmds/projectstore.cpp) | 2 | ||||
| -rw-r--r-- | src/zen/cmds/projectstore_cmd.h (renamed from src/zen/cmds/projectstore.h) | 0 | ||||
| -rw-r--r-- | src/zen/cmds/rpcreplay_cmd.cpp (renamed from src/zen/cmds/rpcreplay.cpp) | 2 | ||||
| -rw-r--r-- | src/zen/cmds/rpcreplay_cmd.h (renamed from src/zen/cmds/rpcreplay.h) | 0 | ||||
| -rw-r--r-- | src/zen/cmds/serve_cmd.cpp (renamed from src/zen/cmds/serve.cpp) | 2 | ||||
| -rw-r--r-- | src/zen/cmds/serve_cmd.h (renamed from src/zen/cmds/serve.h) | 0 | ||||
| -rw-r--r-- | src/zen/cmds/status_cmd.cpp (renamed from src/zen/cmds/status.cpp) | 2 | ||||
| -rw-r--r-- | src/zen/cmds/status_cmd.h (renamed from src/zen/cmds/status.h) | 0 | ||||
| -rw-r--r-- | src/zen/cmds/top_cmd.cpp (renamed from src/zen/cmds/top.cpp) | 2 | ||||
| -rw-r--r-- | src/zen/cmds/top_cmd.h (renamed from src/zen/cmds/top.h) | 0 | ||||
| -rw-r--r-- | src/zen/cmds/trace_cmd.cpp | 93 | ||||
| -rw-r--r-- | src/zen/cmds/trace_cmd.h (renamed from src/zen/cmds/jobs.h) | 17 | ||||
| -rw-r--r-- | src/zen/cmds/up_cmd.cpp (renamed from src/zen/cmds/up.cpp) | 2 | ||||
| -rw-r--r-- | src/zen/cmds/up_cmd.h (renamed from src/zen/cmds/up.h) | 0 | ||||
| -rw-r--r-- | src/zen/cmds/version_cmd.cpp (renamed from src/zen/cmds/version.cpp) | 2 | ||||
| -rw-r--r-- | src/zen/cmds/version_cmd.h (renamed from src/zen/cmds/version.h) | 0 | ||||
| -rw-r--r-- | src/zen/zen.cpp | 34 | ||||
| -rw-r--r-- | src/zencore/include/zencore/trace.h | 5 | ||||
| -rw-r--r-- | src/zencore/trace.cpp | 74 | ||||
| -rw-r--r-- | src/zenhttp/httpclient.cpp | 4 | ||||
| -rw-r--r-- | src/zenhttp/httpsys.cpp | 34 | ||||
| -rw-r--r-- | src/zenhttp/include/zenhttp/httpclient.h | 2 | ||||
| -rw-r--r-- | src/zenserver/admin/admin.cpp | 150 | ||||
| -rw-r--r-- | src/zenserver/admin/admin.h | 11 | ||||
| -rw-r--r-- | src/zenserver/cache/httpstructuredcache.cpp | 8 | ||||
| -rw-r--r-- | src/zenserver/cache/structuredcachestore.cpp | 41 | ||||
| -rw-r--r-- | src/zenserver/cache/structuredcachestore.h | 11 | ||||
| -rw-r--r-- | src/zenserver/zenserver.cpp | 100 |
43 files changed, 700 insertions, 202 deletions
diff --git a/src/zen/cmds/scrub.cpp b/src/zen/cmds/admin_cmd.cpp index 4b47082a0..b48207bec 100644 --- a/src/zen/cmds/scrub.cpp +++ b/src/zen/cmds/admin_cmd.cpp @@ -1,7 +1,8 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "scrub.h" +#include "admin_cmd.h" #include <zencore/logging.h> +#include <zenhttp/formatters.h> #include <zenhttp/httpclient.h> #include <zenhttp/httpcommon.h> @@ -198,4 +199,174 @@ GcStatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 1; } +//////////////////////////////////////////// + +JobCommand::JobCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); + m_Options.add_option("", "j", "jobid", "Job id", cxxopts::value(m_JobId), "<jobid>"); + m_Options.add_option("", "c", "cancel", "Cancel job id", cxxopts::value(m_Cancel), "<cancel>"); +} + +JobCommand::~JobCommand() = default; + +int +JobCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + using namespace std::literals; + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + HttpClient Http(m_HostName); + + if (m_Cancel) + { + if (m_JobId == 0) + { + ZEN_ERROR("Job id must be given"); + return 1; + } + } + std::string Url = m_JobId != 0 ? fmt::format("/admin/jobs/{}", m_JobId) : "/admin/jobs"; + + if (m_Cancel) + { + if (HttpClient::Response Result = Http.Delete(Url, HttpClient::Accept(ZenContentType::kJSON))) + { + ZEN_CONSOLE("{}", Result); + } + else + { + Result.ThrowError("failed cancelling job"sv); + return 1; + } + } + else if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) + { + ZEN_CONSOLE("{}", Result.AsText()); + } + else + { + Result.ThrowError("failed fetching job info"sv); + return 1; + } + + return 0; +} + +//////////////////////////////////////////// + +LoggingCommand::LoggingCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); + m_Options.add_option("", "", "cache-write-log", "Enable cache write logging", cxxopts::value(m_CacheWriteLog), "<enable/disable>"); + m_Options.add_option("", "", "cache-access-log", "Enable cache access logging", cxxopts::value(m_CacheAccessLog), "<enable/disable>"); + m_Options + .add_option("", "", "set-log-level", "Set zenserver log level", cxxopts::value(m_SetLogLevel), "<trace/debug/info/warning/error>"); +} + +LoggingCommand::~LoggingCommand() = default; + +int +LoggingCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + using namespace std::literals; + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + HttpClient Http(m_HostName); + + HttpClient::KeyValueMap Parameters; + + if (!m_CacheWriteLog.empty()) + { + if (m_CacheWriteLog == "enable") + { + (*Parameters)["cacheenablewritelog"] = "true"; + } + else if (m_CacheWriteLog == "disable") + { + (*Parameters)["cacheenablewritelog"] = "false"; + } + else + { + ZEN_ERROR("Invalid value for parameter 'cache-write-log'. Use 'enable' or 'disable'"); + return 1; + } + } + + if (!m_CacheAccessLog.empty()) + { + if (m_CacheAccessLog == "enable") + { + (*Parameters)["cacheenableaccesslog"] = "true"; + } + else if (m_CacheAccessLog == "disable") + { + (*Parameters)["cacheenableaccesslog"] = "false"; + } + else + { + ZEN_ERROR("Invalid value for parameter 'cache-access-log'. Use 'enable' or 'disable'"); + return 1; + } + } + + if (!m_SetLogLevel.empty()) + { + (*Parameters)["loglevel"] = m_SetLogLevel; + } + + if ((*Parameters).empty()) + { + if (HttpClient::Response Result = Http.Get("/admin/logs", HttpClient::Accept(ZenContentType::kJSON))) + { + ZEN_CONSOLE("{}", Result.AsText()); + } + else + { + Result.ThrowError("failed fetching log info"sv); + return 1; + } + return 0; + } + if (HttpClient::Response Result = Http.Post("/admin/logs", HttpClient::KeyValueMap{}, Parameters)) + { + ZEN_CONSOLE("{}", Result.AsText()); + } + else + { + Result.ThrowError("failed setting log info"sv); + return 1; + } + + return 0; +} + } // namespace zen diff --git a/src/zen/cmds/scrub.h b/src/zen/cmds/admin_cmd.h index ee8b4fdbb..873c230d9 100644 --- a/src/zen/cmds/scrub.h +++ b/src/zen/cmds/admin_cmd.h @@ -55,4 +55,41 @@ private: std::string m_HostName; }; +//////////////////////////////////////////// + +class JobCommand : public ZenCmdBase +{ +public: + JobCommand(); + ~JobCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"jobs", "Show/cancel zen background jobs"}; + std::string m_HostName; + std::uint64_t m_JobId = 0; + bool m_Cancel = 0; +}; + +//////////////////////////////////////////// + +class LoggingCommand : public ZenCmdBase +{ +public: + LoggingCommand(); + ~LoggingCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"logs", "Show/control zen logging"}; + std::string m_HostName; + std::string m_CacheWriteLog; + std::string m_CacheAccessLog; + std::string m_SetLogLevel; +}; + } // namespace zen diff --git a/src/zen/cmds/bench.cpp b/src/zen/cmds/bench_cmd.cpp index a2986ce16..06b8967a3 100644 --- a/src/zen/cmds/bench.cpp +++ b/src/zen/cmds/bench_cmd.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "bench.h" +#include "bench_cmd.h" #include <zencore/except.h> #include <zencore/filesystem.h> diff --git a/src/zen/cmds/bench.h b/src/zen/cmds/bench_cmd.h index 8a8bd4a7c..8a8bd4a7c 100644 --- a/src/zen/cmds/bench.h +++ b/src/zen/cmds/bench_cmd.h diff --git a/src/zen/cmds/cache.cpp b/src/zen/cmds/cache_cmd.cpp index 15c24f9ee..1bf6ee60e 100644 --- a/src/zen/cmds/cache.cpp +++ b/src/zen/cmds/cache_cmd.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "cache.h" +#include "cache_cmd.h" #include <zencore/except.h> #include <zencore/filesystem.h> diff --git a/src/zen/cmds/cache.h b/src/zen/cmds/cache_cmd.h index 1f368bdec..1f368bdec 100644 --- a/src/zen/cmds/cache.h +++ b/src/zen/cmds/cache_cmd.h diff --git a/src/zen/cmds/copy.cpp b/src/zen/cmds/copy_cmd.cpp index 6fff973ba..9f689e5bb 100644 --- a/src/zen/cmds/copy.cpp +++ b/src/zen/cmds/copy_cmd.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "copy.h" +#include "copy_cmd.h" #include <zencore/filesystem.h> #include <zencore/fmtutils.h> diff --git a/src/zen/cmds/copy.h b/src/zen/cmds/copy_cmd.h index 549114160..549114160 100644 --- a/src/zen/cmds/copy.h +++ b/src/zen/cmds/copy_cmd.h diff --git a/src/zen/cmds/dedup.cpp b/src/zen/cmds/dedup_cmd.cpp index b48fb8c2d..d496cf404 100644 --- a/src/zen/cmds/dedup.cpp +++ b/src/zen/cmds/dedup_cmd.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "dedup.h" +#include "dedup_cmd.h" #include <zencore/blake3.h> #include <zencore/filesystem.h> diff --git a/src/zen/cmds/dedup.h b/src/zen/cmds/dedup_cmd.h index 6318704f5..6318704f5 100644 --- a/src/zen/cmds/dedup.h +++ b/src/zen/cmds/dedup_cmd.h diff --git a/src/zen/cmds/hash.cpp b/src/zen/cmds/hash_cmd.cpp index cc59ed46e..d1f7a1975 100644 --- a/src/zen/cmds/hash.cpp +++ b/src/zen/cmds/hash_cmd.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "hash.h" +#include "hash_cmd.h" #include <zencore/blake3.h> #include <zencore/logging.h> diff --git a/src/zen/cmds/hash.h b/src/zen/cmds/hash_cmd.h index e5ee071e9..e5ee071e9 100644 --- a/src/zen/cmds/hash.h +++ b/src/zen/cmds/hash_cmd.h diff --git a/src/zen/cmds/jobs.cpp b/src/zen/cmds/jobs.cpp deleted file mode 100644 index 137c321af..000000000 --- a/src/zen/cmds/jobs.cpp +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "jobs.h" - -#include <zencore/fmtutils.h> -#include <zencore/logging.h> -#include <zencore/string.h> -#include <zencore/uid.h> -#include <zenhttp/formatters.h> -#include <zenhttp/httpclient.h> - -namespace zen { - -//////////////////////////////////////////// - -JobCommand::JobCommand() -{ - m_Options.add_options()("h,help", "Print help"); - m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); - m_Options.add_option("", "j", "jobid", "Job id", cxxopts::value(m_JobId), "<jobid>"); - m_Options.add_option("", "c", "cancel", "Cancel job id", cxxopts::value(m_Cancel), "<cancel>"); -} - -JobCommand::~JobCommand() = default; - -int -JobCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - ZEN_UNUSED(GlobalOptions); - - using namespace std::literals; - - if (!ParseOptions(argc, argv)) - { - return 0; - } - - m_HostName = ResolveTargetHostSpec(m_HostName); - - if (m_HostName.empty()) - { - throw OptionParseException("unable to resolve server specification"); - } - - HttpClient Http(m_HostName); - - if (m_Cancel) - { - if (m_JobId == 0) - { - ZEN_ERROR("Job id must be given"); - return 1; - } - } - std::string Url = m_JobId != 0 ? fmt::format("/admin/jobs/{}", m_JobId) : "/admin/jobs"; - - if (m_Cancel) - { - if (HttpClient::Response Result = Http.Delete(Url, HttpClient::Accept(ZenContentType::kJSON))) - { - ZEN_CONSOLE("{}", Result); - } - else - { - Result.ThrowError("failed cancelling job"sv); - return 1; - } - } - else if (HttpClient::Response Result = Http.Get(Url, HttpClient::Accept(ZenContentType::kJSON))) - { - ZEN_CONSOLE("{}", Result.AsText()); - } - else - { - Result.ThrowError("failed fetching job info"sv); - return 1; - } - - return 0; -} - -} // namespace zen diff --git a/src/zen/cmds/print.cpp b/src/zen/cmds/print_cmd.cpp index a3a9bb3cc..acffb2002 100644 --- a/src/zen/cmds/print.cpp +++ b/src/zen/cmds/print_cmd.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "print.h" +#include "print_cmd.h" #include <zencore/compactbinarypackage.h> #include <zencore/compactbinaryvalidation.h> diff --git a/src/zen/cmds/print.h b/src/zen/cmds/print_cmd.h index 09d91830a..09d91830a 100644 --- a/src/zen/cmds/print.h +++ b/src/zen/cmds/print_cmd.h diff --git a/src/zen/cmds/projectstore.cpp b/src/zen/cmds/projectstore_cmd.cpp index edeff7d85..5795b3190 100644 --- a/src/zen/cmds/projectstore.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "projectstore.h" +#include "projectstore_cmd.h" #include <zencore/compactbinarybuilder.h> #include <zencore/filesystem.h> diff --git a/src/zen/cmds/projectstore.h b/src/zen/cmds/projectstore_cmd.h index fd1590423..fd1590423 100644 --- a/src/zen/cmds/projectstore.h +++ b/src/zen/cmds/projectstore_cmd.h diff --git a/src/zen/cmds/rpcreplay.cpp b/src/zen/cmds/rpcreplay_cmd.cpp index 349025791..9e43280e1 100644 --- a/src/zen/cmds/rpcreplay.cpp +++ b/src/zen/cmds/rpcreplay_cmd.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "rpcreplay.h" +#include "rpcreplay_cmd.h" #include <zencore/compactbinarybuilder.h> #include <zencore/filesystem.h> diff --git a/src/zen/cmds/rpcreplay.h b/src/zen/cmds/rpcreplay_cmd.h index 742e5ec5b..742e5ec5b 100644 --- a/src/zen/cmds/rpcreplay.h +++ b/src/zen/cmds/rpcreplay_cmd.h diff --git a/src/zen/cmds/serve.cpp b/src/zen/cmds/serve_cmd.cpp index 72afc105d..c8117774b 100644 --- a/src/zen/cmds/serve.cpp +++ b/src/zen/cmds/serve_cmd.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "serve.h" +#include "serve_cmd.h" #include <zencore/blake3.h> #include <zencore/compactbinarybuilder.h> diff --git a/src/zen/cmds/serve.h b/src/zen/cmds/serve_cmd.h index 007038d84..007038d84 100644 --- a/src/zen/cmds/serve.h +++ b/src/zen/cmds/serve_cmd.h diff --git a/src/zen/cmds/status.cpp b/src/zen/cmds/status_cmd.cpp index 1afe191b7..cc936835a 100644 --- a/src/zen/cmds/status.cpp +++ b/src/zen/cmds/status_cmd.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "status.h" +#include "status_cmd.h" #include <zencore/fmtutils.h> #include <zencore/logging.h> diff --git a/src/zen/cmds/status.h b/src/zen/cmds/status_cmd.h index 98f72e651..98f72e651 100644 --- a/src/zen/cmds/status.h +++ b/src/zen/cmds/status_cmd.h diff --git a/src/zen/cmds/top.cpp b/src/zen/cmds/top_cmd.cpp index 6aed6fe1b..568ee76c9 100644 --- a/src/zen/cmds/top.cpp +++ b/src/zen/cmds/top_cmd.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "top.h" +#include "top_cmd.h" #include <zencore/fmtutils.h> #include <zencore/logging.h> diff --git a/src/zen/cmds/top.h b/src/zen/cmds/top_cmd.h index 83410587b..83410587b 100644 --- a/src/zen/cmds/top.h +++ b/src/zen/cmds/top_cmd.h diff --git a/src/zen/cmds/trace_cmd.cpp b/src/zen/cmds/trace_cmd.cpp new file mode 100644 index 000000000..fee4dd6bc --- /dev/null +++ b/src/zen/cmds/trace_cmd.cpp @@ -0,0 +1,93 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "trace_cmd.h" +#include <zencore/logging.h> +#include <zenhttp/httpclient.h> +#include <zenhttp/httpcommon.h> + +using namespace std::literals; + +namespace zen { + +TraceCommand::TraceCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); + m_Options.add_option("", "s", "stop", "Stop tracing", cxxopts::value(m_Stop)->default_value("false"), "<stop>"); + m_Options.add_option("", "", "host", "Start tracing to host", cxxopts::value(m_TraceHost), "<hostip>"); + m_Options.add_option("", "", "file", "Start tracing to file", cxxopts::value(m_TraceFile), "<filepath>"); +} + +TraceCommand::~TraceCommand() = default; + +int +TraceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_HostName.empty()) + { + throw OptionParseException("unable to resolve server specification"); + } + + zen::HttpClient Http(m_HostName); + + if (m_Stop) + { + if (zen::HttpClient::Response Response = Http.Post("/admin/trace/stop"sv)) + { + ZEN_CONSOLE("OK: {}", Response.ToText()); + return 0; + } + else + { + ZEN_ERROR("trace stop failed: {}", Response.AsText()); + return 1; + } + } + + std::string StartArg; + if (!m_TraceHost.empty()) + { + StartArg = fmt::format("host={}", m_TraceHost); + } + else if (!m_TraceFile.empty()) + { + StartArg = fmt::format("file={}", m_TraceFile); + } + + if (!StartArg.empty()) + { + if (zen::HttpClient::Response Response = Http.Post(fmt::format("/admin/trace/start?{}"sv, StartArg))) + { + ZEN_CONSOLE("OK: {}", Response.ToText()); + return 0; + } + else + { + ZEN_ERROR("trace start failed: {}", Response.AsText()); + return 1; + } + } + + if (zen::HttpClient::Response Response = Http.Get("/admin/trace"sv)) + { + ZEN_CONSOLE("OK: {}", Response.ToText()); + return 0; + } + else + { + ZEN_ERROR("trace status failed: {}", Response.AsText()); + } + + return 1; +} + +} // namespace zen diff --git a/src/zen/cmds/jobs.h b/src/zen/cmds/trace_cmd.h index 2c523f24a..7b2d15fb1 100644 --- a/src/zen/cmds/jobs.h +++ b/src/zen/cmds/trace_cmd.h @@ -6,22 +6,23 @@ namespace zen { -//////////////////////////////////////////// - -class JobCommand : public ZenCmdBase +/** Scrub storage + */ +class TraceCommand : public ZenCmdBase { public: - JobCommand(); - ~JobCommand(); + TraceCommand(); + ~TraceCommand(); virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"jobs", "Show/cancel zen background jobs"}; + cxxopts::Options m_Options{"trace", "Control zen realtime tracing"}; std::string m_HostName; - std::uint64_t m_JobId = 0; - bool m_Cancel = 0; + bool m_Stop; + std::string m_TraceHost; + std::string m_TraceFile; }; } // namespace zen diff --git a/src/zen/cmds/up.cpp b/src/zen/cmds/up_cmd.cpp index d1ae0794a..b07fb6ec8 100644 --- a/src/zen/cmds/up.cpp +++ b/src/zen/cmds/up_cmd.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "up.h" +#include "up_cmd.h" #include <zencore/filesystem.h> #include <zencore/logging.h> diff --git a/src/zen/cmds/up.h b/src/zen/cmds/up_cmd.h index 510cc865e..510cc865e 100644 --- a/src/zen/cmds/up.h +++ b/src/zen/cmds/up_cmd.h diff --git a/src/zen/cmds/version.cpp b/src/zen/cmds/version_cmd.cpp index ba83b527d..bd31862b4 100644 --- a/src/zen/cmds/version.cpp +++ b/src/zen/cmds/version_cmd.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "version.h" +#include "version_cmd.h" #include <zencore/config.h> #include <zencore/filesystem.h> diff --git a/src/zen/cmds/version.h b/src/zen/cmds/version_cmd.h index 0e37e91a0..0e37e91a0 100644 --- a/src/zen/cmds/version.h +++ b/src/zen/cmds/version_cmd.h diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index ede8b0415..949a5f4ac 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -5,21 +5,21 @@ #include "zen.h" -#include "cmds/bench.h" -#include "cmds/cache.h" -#include "cmds/copy.h" -#include "cmds/dedup.h" -#include "cmds/hash.h" -#include "cmds/jobs.h" -#include "cmds/print.h" -#include "cmds/projectstore.h" -#include "cmds/rpcreplay.h" -#include "cmds/scrub.h" -#include "cmds/serve.h" -#include "cmds/status.h" -#include "cmds/top.h" -#include "cmds/up.h" -#include "cmds/version.h" +#include "cmds/admin_cmd.h" +#include "cmds/bench_cmd.h" +#include "cmds/cache_cmd.h" +#include "cmds/copy_cmd.h" +#include "cmds/dedup_cmd.h" +#include "cmds/hash_cmd.h" +#include "cmds/print_cmd.h" +#include "cmds/projectstore_cmd.h" +#include "cmds/rpcreplay_cmd.h" +#include "cmds/serve_cmd.h" +#include "cmds/status_cmd.h" +#include "cmds/top_cmd.h" +#include "cmds/trace_cmd.h" +#include "cmds/up_cmd.h" +#include "cmds/version_cmd.h" #include "cmds/vfs_cmd.h" #include <zencore/filesystem.h> @@ -228,7 +228,9 @@ main(int argc, char** argv) ServeCommand ServeCmd; SnapshotOplogCommand SnapshotOplogCmd; StatusCommand StatusCmd; + LoggingCommand LoggingCmd; TopCommand TopCmd; + TraceCommand TraceCmd; UpCommand UpCmd; VersionCommand VersionCmd; VfsCommand VfsCmd; @@ -276,8 +278,10 @@ main(int argc, char** argv) {"scrub", &ScrubCmd, "Scrub zen storage (verify data integrity)"}, {"serve", &ServeCmd, "Serve files from a directory"}, {"status", &StatusCmd, "Show zen status"}, + {"logs", &LoggingCmd, "Show/control zen logging"}, {"jobs", &JobCmd, "Show/cancel zen background jobs"}, {"top", &TopCmd, "Monitor zen server activity"}, + {"trace", &TraceCmd, "Control zen realtime tracing"}, {"up", &UpCmd, "Bring zen server up"}, {"attach", &AttachCmd, "Add a sponsor process to a running zen service"}, {"version", &VersionCmd, "Get zen server version"}, diff --git a/src/zencore/include/zencore/trace.h b/src/zencore/include/zencore/trace.h index 3bc2f7f02..665df5808 100644 --- a/src/zencore/include/zencore/trace.h +++ b/src/zencore/include/zencore/trace.h @@ -25,8 +25,11 @@ enum class TraceType None }; -void TraceInit(const char* HostOrPath, TraceType Type); +void TraceInit(); void TraceShutdown(); +bool IsTracing(); +void TraceStart(const char* HostOrPath, TraceType Type); +bool TraceStop(); #else diff --git a/src/zencore/trace.cpp b/src/zencore/trace.cpp index 6da5d28d6..d71ca0984 100644 --- a/src/zencore/trace.cpp +++ b/src/zencore/trace.cpp @@ -9,10 +9,48 @@ # include <zencore/trace.h> void -TraceInit(const char* HostOrPath, TraceType Type) +TraceInit() { - bool EnableEvents = true; + static std::atomic_bool gInited = false; + bool Expected = false; + if (!gInited.compare_exchange_strong(Expected, true)) + { + return; + } + + trace::FInitializeDesc Desc = { + .bUseImportantCache = true, + }; + trace::Initialize(Desc); + trace::ThreadRegister("main", /* system id */ 0, /* sort id */ 0); + trace::DescribeSession("zenserver", +# if ZEN_BUILD_DEBUG + trace::Build::Debug, +# else + trace::Build::Development, +# endif + "", + ZEN_CFG_VERSION_BUILD_STRING); +} + +void +TraceShutdown() +{ + (void)TraceStop(); + trace::Shutdown(); +} + +bool +IsTracing() +{ + return trace::IsTracing(); +} + +void +TraceStart(const char* HostOrPath, TraceType Type) +{ + TraceInit(); switch (Type) { case TraceType::Network: @@ -24,34 +62,20 @@ TraceInit(const char* HostOrPath, TraceType Type) break; case TraceType::None: - EnableEvents = false; break; } - - trace::FInitializeDesc Desc = { - .bUseImportantCache = false, - }; - trace::Initialize(Desc); - - if (EnableEvents) - { - trace::ToggleChannel("cpu", true); - trace::ThreadRegister("main", /* system id */ 0, /* sort id */ 0); - trace::DescribeSession("zenserver", -# if ZEN_BUILD_DEBUG - trace::Build::Debug, -# else - trace::Build::Development, -# endif - "", - ZEN_CFG_VERSION_BUILD_STRING); - } + trace::ToggleChannel("cpu", true); } -void -TraceShutdown() +bool +TraceStop() { - trace::Shutdown(); + trace::ToggleChannel("cpu", false); + if (trace::Stop()) + { + return true; + } + return false; } #endif // ZEN_WITH_TRACE diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 5ae5f78d6..caefce5f4 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -587,11 +587,11 @@ HttpClient::Delete(std::string_view Url, const KeyValueMap& AdditionalHeader) } HttpClient::Response -HttpClient::Post(std::string_view Url, const KeyValueMap& AdditionalHeader) +HttpClient::Post(std::string_view Url, const KeyValueMap& AdditionalHeader, const KeyValueMap& Parameters) { ZEN_TRACE_CPU("HttpClient::PostNoPayload"); - Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}); + Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, Parameters); return CommonResponse(Sess.Post()); } diff --git a/src/zenhttp/httpsys.cpp b/src/zenhttp/httpsys.cpp index 3bcaa5861..f95a31914 100644 --- a/src/zenhttp/httpsys.cpp +++ b/src/zenhttp/httpsys.cpp @@ -877,8 +877,11 @@ HttpSysServer::InitializeServer(int BasePort) HTTP_BINDING_INFO HttpBindingInfo = {{0}, 0}; + WideStringBuilder<64> QueueName; + QueueName << "zenserver_" << EffectivePort; + Result = HttpCreateRequestQueue(HTTPAPI_VERSION_2, - /* Name */ nullptr, + /* Name */ QueueName.c_str(), /* SecurityAttributes */ nullptr, /* Flags */ 0, &m_RequestQueueHandle); @@ -902,6 +905,20 @@ HttpSysServer::InitializeServer(int BasePort) return EffectivePort; } + // Configure rejection method. Default is to drop the connection, it's better if we + // return an explicit error code when the queue cannot accept more requests + + { + HTTP_503_RESPONSE_VERBOSITY VerbosityInformation = Http503ResponseVerbosityLimited; + + Result = HttpSetRequestQueueProperty(m_RequestQueueHandle, + HttpServer503VerbosityProperty, + &VerbosityInformation, + sizeof VerbosityInformation, + 0, + 0); + } + // Create I/O completion port std::error_code ErrorCode; @@ -918,6 +935,21 @@ HttpSysServer::InitializeServer(int BasePort) ZEN_INFO("Started http.sys server at '{}'", WideToUtf8(m_BaseUris.front())); } + // This is not available in all Windows SDK versions so for now we can't use recently + // released functionality. We should investigate how to get more recent SDK releases + // into the build + +# if 0 + if (HttpIsFeatureSupported(/* HttpFeatureHttp3 */ (HTTP_FEATURE_ID) 4)) + { + ZEN_DEBUG("HTTP3 is available"); + } + else + { + ZEN_DEBUG("HTTP3 is NOT available"); + } +# endif + return EffectivePort; } diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h index 2c81f4a23..044a3162e 100644 --- a/src/zenhttp/include/zenhttp/httpclient.h +++ b/src/zenhttp/include/zenhttp/httpclient.h @@ -128,7 +128,7 @@ public: [[nodiscard]] Response Get(std::string_view Url, const KeyValueMap& AdditionalHeader = {}, const KeyValueMap& Parameters = {}); [[nodiscard]] Response Head(std::string_view Url, const KeyValueMap& AdditionalHeader = {}); [[nodiscard]] Response Delete(std::string_view Url, const KeyValueMap& AdditionalHeader = {}); - [[nodiscard]] Response Post(std::string_view Url, const KeyValueMap& AdditionalHeader = {}); + [[nodiscard]] Response Post(std::string_view Url, const KeyValueMap& AdditionalHeader = {}, const KeyValueMap& Parameters = {}); [[nodiscard]] Response Post(std::string_view Url, const IoBuffer& Payload, const KeyValueMap& AdditionalHeader = {}); [[nodiscard]] Response Post(std::string_view Url, CbObject Payload, const KeyValueMap& AdditionalHeader = {}); [[nodiscard]] Response Post(std::string_view Url, CbPackage Payload, const KeyValueMap& AdditionalHeader = {}); diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp index cef2ba403..083086f50 100644 --- a/src/zenserver/admin/admin.cpp +++ b/src/zenserver/admin/admin.cpp @@ -5,15 +5,29 @@ #include <zencore/compactbinarybuilder.h> #include <zencore/jobqueue.h> #include <zencore/string.h> +#if ZEN_WITH_TRACE +# include <zencore/trace.h> +#endif // ZEN_WITH_TRACE + #include <zenstore/gc.h> +#include "cache/structuredcachestore.h" #include <chrono> +ZEN_THIRD_PARTY_INCLUDES_START +#include <spdlog/spdlog.h> +ZEN_THIRD_PARTY_INCLUDES_END + namespace zen { -HttpAdminService::HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue) +HttpAdminService::HttpAdminService(GcScheduler& Scheduler, + JobQueue& BackgroundJobQueue, + ZenCacheStore& CacheStore, + const LogPaths& LogPaths) : m_GcScheduler(Scheduler) , m_BackgroundJobQueue(BackgroundJobQueue) +, m_CacheStore(CacheStore) +, m_LogPaths(LogPaths) { using namespace std::literals; @@ -248,6 +262,140 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJ Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); }, HttpVerb::kPost); +#if ZEN_WITH_TRACE + m_Router.RegisterRoute( + "trace", + [this](HttpRouterRequest& Req) { + bool Enabled = IsTracing(); + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Enabled ? "enabled" : "disabled"); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "trace/start", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + TraceType Type = TraceType::None; + std::string HostOrPath; + if (auto Param = Params.GetValue("file"); Param.empty() == false) + { + Type = TraceType::File; + HostOrPath = Param; + } + if (auto Param = Params.GetValue("host"); Param.empty() == false) + { + Type = TraceType::Network; + HostOrPath = Param; + } + if (Type == TraceType::None) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Invalid trace type, use `file` or `host`"sv); + } + if (IsTracing()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Tracing is already enabled"sv); + } + TraceStart(HostOrPath.c_str(), Type); + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing started"); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "trace/stop", + [this](HttpRouterRequest& Req) { + if (!IsTracing()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Tracing is not enabled"sv); + } + if (TraceStop()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing stopped"); + } + else + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError, + HttpContentType::kText, + "Failed stopping trace"); + } + }, + HttpVerb::kPost); +#endif // ZEN_WITH_TRACE + + m_Router.RegisterRoute( + "logs", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + spdlog::string_view_t LogLevel = spdlog::level::to_string_view(spdlog::get_level()); + Obj.AddString("loglevel", std::string_view(LogLevel.data(), LogLevel.size())); + Obj.AddString("Logfile", PathToUtf8(m_LogPaths.AbsLogPath)); + Obj.BeginObject("cache"); + { + const ZenCacheStore::Configuration& CacheConfig = m_CacheStore.GetConfiguration(); + Obj.AddString("Logfile", PathToUtf8(m_LogPaths.CacheLogPath)); + Obj.AddBool("Write", CacheConfig.Logging.EnableWriteLog); + Obj.AddBool("Access", CacheConfig.Logging.EnableAccessLog); + } + Obj.EndObject(); + Obj.BeginObject("http"); + { + Obj.AddString("Logfile", PathToUtf8(m_LogPaths.HttpLogPath)); + } + Obj.EndObject(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "logs", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + bool SetCacheLogConfig = false; + ExtendableStringBuilder<256> StringBuilder; + ZenCacheStore::Configuration::LogConfig LoggingConfig = m_CacheStore.GetConfiguration().Logging; + if (std::string Param(Params.GetValue("cacheenablewritelog")); Param.empty() == false) + { + LoggingConfig.EnableWriteLog = StrCaseCompare(Param.c_str(), "true") == 0; + SetCacheLogConfig = true; + } + if (std::string Param(Params.GetValue("cacheenableaccesslog")); Param.empty() == false) + { + LoggingConfig.EnableAccessLog = StrCaseCompare(Param.c_str(), "true") == 0; + SetCacheLogConfig = true; + } + if (SetCacheLogConfig) + { + m_CacheStore.SetLoggingConfig(LoggingConfig); + StringBuilder.Append(fmt::format("cache write log: {}, cache access log: {}", + LoggingConfig.EnableWriteLog ? "true" : "false", + LoggingConfig.EnableAccessLog ? "true" : "false")); + } + if (std::string Param(Params.GetValue("loglevel")); Param.empty() == false) + { + spdlog::level::level_enum NewLevel = spdlog::level::from_str(Param); + spdlog::string_view_t LogLevel = spdlog::level::to_string_view(NewLevel); + if (LogLevel != Param) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid log level '{}'", Param)); + } + spdlog::set_level(NewLevel); + if (StringBuilder.Size() > 0) + { + StringBuilder.Append(", "); + } + StringBuilder.Append("loglevel: "); + StringBuilder.Append(Param); + } + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, StringBuilder.ToView()); + }, + HttpVerb::kPost); } HttpAdminService::~HttpAdminService() diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h index 3152f87ab..12a47f29e 100644 --- a/src/zenserver/admin/admin.h +++ b/src/zenserver/admin/admin.h @@ -9,11 +9,18 @@ namespace zen { class GcScheduler; class JobQueue; +class ZenCacheStore; class HttpAdminService : public zen::HttpService { public: - HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue); + struct LogPaths + { + std::filesystem::path AbsLogPath; + std::filesystem::path HttpLogPath; + std::filesystem::path CacheLogPath; + }; + HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue, ZenCacheStore& CacheStore, const LogPaths& LogPaths); ~HttpAdminService(); virtual const char* BaseUri() const override; @@ -23,6 +30,8 @@ private: HttpRequestRouter m_Router; GcScheduler& m_GcScheduler; JobQueue& m_BackgroundJobQueue; + ZenCacheStore& m_CacheStore; + LogPaths m_LogPaths; }; } // namespace zen diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp index 32e8c1f98..11ac81dcb 100644 --- a/src/zenserver/cache/httpstructuredcache.cpp +++ b/src/zenserver/cache/httpstructuredcache.cpp @@ -717,8 +717,12 @@ HttpStructuredCacheService::HandleCacheRequest(HttpServerRequest& Request) BasePathString << Info.Config.BasePath.u8string(); ResponseWriter.AddString("BasePath"sv, BasePathString.ToView()); ResponseWriter.AddBool("AllowAutomaticCreationOfNamespaces", Info.Config.AllowAutomaticCreationOfNamespaces); - ResponseWriter.AddBool("EnableWriteLog", Info.Config.EnableWriteLog); - ResponseWriter.AddBool("EnableAccessLog", Info.Config.EnableAccessLog); + ResponseWriter.BeginObject("Logging"); + { + ResponseWriter.AddBool("EnableWriteLog", Info.Config.Logging.EnableWriteLog); + ResponseWriter.AddBool("EnableAccessLog", Info.Config.Logging.EnableAccessLog); + } + ResponseWriter.EndObject(); } ResponseWriter.EndObject(); diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp index 4499b05f7..809df1a94 100644 --- a/src/zenserver/cache/structuredcachestore.cpp +++ b/src/zenserver/cache/structuredcachestore.cpp @@ -238,10 +238,7 @@ ZenCacheStore::ZenCacheStore(GcManager& Gc, const Configuration& Configuration, , m_Configuration(Configuration) , m_ExitLogging(false) { - if (m_Configuration.EnableAccessLog || m_Configuration.EnableWriteLog) - { - m_AsyncLoggingThread = std::thread(&ZenCacheStore::LogWorker, this); - } + SetLoggingConfig(m_Configuration.Logging); CreateDirectories(m_Configuration.BasePath); ZEN_INFO("Initializing at '{}'", m_Configuration.BasePath); @@ -281,12 +278,7 @@ ZenCacheStore::ZenCacheStore(GcManager& Gc, const Configuration& Configuration, ZenCacheStore::~ZenCacheStore() { - m_ExitLogging.store(true); - m_LogEvent.Set(); - if (m_AsyncLoggingThread.joinable()) - { - m_AsyncLoggingThread.join(); - } + SetLoggingConfig({.EnableWriteLog = false, .EnableAccessLog = false}); m_Namespaces.clear(); } @@ -381,7 +373,7 @@ ZenCacheStore::Get(const CacheRequestContext& Context, { bool Result = Store->Get(Bucket, HashKey, OutValue); - if (m_Configuration.EnableAccessLog) + if (m_AccessLogEnabled) { ZEN_TRACE_CPU("Z$::Get::AccessLog"); bool Signal = false; @@ -421,7 +413,7 @@ ZenCacheStore::Put(const CacheRequestContext& Context, { ZEN_TRACE_CPU("Z$::Put"); - if (m_Configuration.EnableWriteLog) + if (m_WriteLogEnabled) { ZEN_TRACE_CPU("Z$::Get::WriteLog"); bool Signal = false; @@ -616,6 +608,31 @@ ZenCacheStore::StorageSize() const return Size; } +void +ZenCacheStore::SetLoggingConfig(const Configuration::LogConfig& Loggingconfig) +{ + if (!Loggingconfig.EnableAccessLog && !Loggingconfig.EnableWriteLog) + { + m_AccessLogEnabled.store(false); + m_WriteLogEnabled.store(false); + m_ExitLogging.store(true); + m_LogEvent.Set(); + if (m_AsyncLoggingThread.joinable()) + { + m_AsyncLoggingThread.join(); + } + m_Configuration.Logging = Loggingconfig; + return; + } + if (!m_AccessLogEnabled.load() && !m_WriteLogEnabled.load()) + { + m_AsyncLoggingThread = std::thread(&ZenCacheStore::LogWorker, this); + } + m_WriteLogEnabled.store(Loggingconfig.EnableWriteLog); + m_AccessLogEnabled.store(Loggingconfig.EnableAccessLog); + m_Configuration.Logging = Loggingconfig; +} + ZenCacheStore::Info ZenCacheStore::GetInfo() const { diff --git a/src/zenserver/cache/structuredcachestore.h b/src/zenserver/cache/structuredcachestore.h index 239efe68f..e7b64babe 100644 --- a/src/zenserver/cache/structuredcachestore.h +++ b/src/zenserver/cache/structuredcachestore.h @@ -122,8 +122,11 @@ public: { std::filesystem::path BasePath; bool AllowAutomaticCreationOfNamespaces = false; - bool EnableWriteLog = true; - bool EnableAccessLog = true; + struct LogConfig + { + bool EnableWriteLog = true; + bool EnableAccessLog = true; + } Logging; }; struct Info @@ -159,6 +162,8 @@ public: GcStorageSize StorageSize() const; + Configuration GetConfiguration() const { return m_Configuration; } + void SetLoggingConfig(const Configuration::LogConfig& Loggingconfig); Info GetInfo() const; std::optional<ZenCacheNamespace::Info> GetNamespaceInfo(std::string_view Namespace); std::optional<ZenCacheNamespace::BucketInfo> GetBucketInfo(std::string_view Namespace, std::string_view Bucket); @@ -201,6 +206,8 @@ private: std::atomic_bool m_ExitLogging; Event m_LogEvent; std::thread m_AsyncLoggingThread; + std::atomic_bool m_WriteLogEnabled; + std::atomic_bool m_AccessLogEnabled; }; void z$_forcelink(); diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 921e3038d..40a797c21 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -161,13 +161,31 @@ namespace utils { void sink_it_(const spdlog::details::log_msg& msg) override { - std::string Message = fmt::format("{}\n{}({}) [{}]", msg.payload, msg.source.filename, msg.source.line, msg.source.funcname); - sentry_value_t event = sentry_value_new_message_event( - /* level */ MapToSentryLevel[msg.level], - /* logger */ nullptr, - /* message */ Message.c_str()); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); + try + { + std::string Message = + fmt::format("{}\n{}({}) [{}]", msg.payload, msg.source.filename, msg.source.line, msg.source.funcname); + sentry_value_t event = sentry_value_new_message_event( + /* level */ MapToSentryLevel[msg.level], + /* logger */ nullptr, + /* message */ Message.c_str()); + sentry_event_value_add_stacktrace(event, NULL, 0); + sentry_capture_event(event); + } + catch (std::exception&) + { + // If our logging with Message formatting fails we do a non-allocating version and just post the msg.payload raw + char TmpBuffer[256]; + size_t MaxCopy = Min<size_t>(msg.payload.size(), size_t(255)); + memcpy(TmpBuffer, msg.payload.data(), MaxCopy); + TmpBuffer[MaxCopy] = '\0'; + sentry_value_t event = sentry_value_new_message_event( + /* level */ SENTRY_LEVEL_ERROR, + /* logger */ nullptr, + /* message */ TmpBuffer); + sentry_event_value_add_stacktrace(event, NULL, 0); + sentry_capture_event(event); + } } void flush_() override {} }; @@ -181,13 +199,26 @@ namespace utils { const char* FunctionName, const char* Msg) { - std::string Message = fmt::format("ASSERT {}:({}) [{}]\n\"{}\"", Filename, LineNumber, FunctionName, Msg); - sentry_value_t event = sentry_value_new_message_event( - /* level */ SENTRY_LEVEL_ERROR, - /* logger */ nullptr, - /* message */ Message.c_str()); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); + try + { + std::string Message = fmt::format("ASSERT {}:({}) [{}]\n\"{}\"", Filename, LineNumber, FunctionName, Msg); + sentry_value_t event = sentry_value_new_message_event( + /* level */ SENTRY_LEVEL_ERROR, + /* logger */ nullptr, + /* message */ Message.c_str()); + sentry_event_value_add_stacktrace(event, NULL, 0); + sentry_capture_event(event); + } + catch (std::exception&) + { + // If our logging with Message formatting fails we do a non-allocating version and just post the Msg raw + sentry_value_t event = sentry_value_new_message_event( + /* level */ SENTRY_LEVEL_ERROR, + /* logger */ nullptr, + /* message */ Msg); + sentry_event_value_add_stacktrace(event, NULL, 0); + sentry_capture_event(event); + } } AssertImpl* PrevAssertImpl; }; @@ -433,7 +464,13 @@ public: m_GcScheduler.Initialize(GcConfig); // Create and register admin interface last to make sure all is properly initialized - m_AdminService = std::make_unique<HttpAdminService>(m_GcScheduler, *m_JobQueue); + m_AdminService = + std::make_unique<HttpAdminService>(m_GcScheduler, + *m_JobQueue, + *m_CacheStore, + HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, + .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", + .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}); m_Http->RegisterService(*m_AdminService); return EffectiveBasePort; @@ -905,12 +942,13 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) using namespace std::literals; ZEN_INFO("instantiating structured cache service"); - m_CacheStore = new ZenCacheStore(m_GcManager, - ZenCacheStore::Configuration{.BasePath = m_DataRoot / "cache", - .AllowAutomaticCreationOfNamespaces = true, - .EnableWriteLog = ServerOptions.StructuredCacheWriteLogEnabled, - .EnableAccessLog = ServerOptions.StructuredCacheAccessLogEnabled}, - m_GcManager.GetDiskWriteBlocker()); + m_CacheStore = + new ZenCacheStore(m_GcManager, + ZenCacheStore::Configuration{.BasePath = m_DataRoot / "cache", + .AllowAutomaticCreationOfNamespaces = true, + .Logging = {.EnableWriteLog = ServerOptions.StructuredCacheWriteLogEnabled, + .EnableAccessLog = ServerOptions.StructuredCacheAccessLogEnabled}}, + m_GcManager.GetDiskWriteBlocker()); const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig; @@ -1136,7 +1174,7 @@ ZenEntryPoint::Run() sentry_options_set_dsn(SentryOptions, "https://[email protected]/5919284"); sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str()); sentry_options_set_logger(SentryOptions, SentryLogFunction, this); - std::string SentryAttachmentPath = m_ServerOptions.AbsLogFile.string(); + std::string SentryAttachmentPath = PathToUtf8(m_ServerOptions.AbsLogFile); if (SentryAttachmentPath.starts_with("\\\\?\\")) { SentryAttachmentPath = SentryAttachmentPath.substr(4); @@ -1186,8 +1224,8 @@ ZenEntryPoint::Run() auto _ = zen::MakeGuard([&SentryAssert, SentryErrorCode] { if (SentryErrorCode == 0) { - SentryAssert.reset(); zen::logging::SetErrorLog(std::shared_ptr<spdlog::logger>()); + SentryAssert.reset(); sentry_close(); } }); @@ -1420,14 +1458,6 @@ test_main(int argc, char** argv) } #endif -#if ZEN_WITH_TRACE -static void -StopTrace() -{ - TraceShutdown(); -} -#endif // ZEN_WITH_TRACE - int main(int argc, char* argv[]) { @@ -1466,17 +1496,17 @@ main(int argc, char* argv[]) #if ZEN_WITH_TRACE if (ServerOptions.TraceHost.size()) { - TraceInit(ServerOptions.TraceHost.c_str(), TraceType::Network); + TraceStart(ServerOptions.TraceHost.c_str(), TraceType::Network); } else if (ServerOptions.TraceFile.size()) { - TraceInit(ServerOptions.TraceFile.c_str(), TraceType::File); + TraceStart(ServerOptions.TraceFile.c_str(), TraceType::File); } else { - TraceInit(nullptr, TraceType::None); + TraceInit(); } - atexit(StopTrace); + atexit(TraceShutdown); #endif // ZEN_WITH_TRACE #if ZEN_PLATFORM_WINDOWS |