aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/zen/cmds/info_cmd.cpp51
-rw-r--r--src/zen/cmds/info_cmd.h24
-rw-r--r--src/zen/zen.cpp3
-rw-r--r--src/zenserver/admin/admin.cpp129
-rw-r--r--src/zenserver/admin/admin.h29
-rw-r--r--src/zenserver/config.cpp75
-rw-r--r--src/zenserver/config.h4
-rw-r--r--src/zenserver/zenserver.cpp52
8 files changed, 336 insertions, 31 deletions
diff --git a/src/zen/cmds/info_cmd.cpp b/src/zen/cmds/info_cmd.cpp
new file mode 100644
index 000000000..aec8ca46b
--- /dev/null
+++ b/src/zen/cmds/info_cmd.cpp
@@ -0,0 +1,51 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "info_cmd.h"
+
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/string.h>
+#include <zenhttp/httpclient.h>
+
+using namespace std::literals;
+
+namespace zen {
+
+InfoCommand::InfoCommand()
+{
+ m_Options.add_options()("h,help", "Print help");
+ m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+}
+
+InfoCommand::~InfoCommand()
+{
+}
+
+int
+InfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+ ZEN_UNUSED(GlobalOptions);
+
+ if (!ParseOptions(argc, argv))
+ {
+ return 0;
+ }
+
+ m_HostName = ResolveTargetHostSpec(m_HostName);
+
+ if (m_HostName.empty())
+ {
+ throw OptionParseException("unable to resolve server specification");
+ }
+
+ HttpClient Http(m_HostName);
+
+ if (HttpClient::Response Result = Http.Get("/admin/info", HttpClient::Accept(ZenContentType::kJSON)))
+ {
+ ZEN_CONSOLE("{}", Result.AsText());
+ }
+
+ return 0;
+}
+
+} // namespace zen
diff --git a/src/zen/cmds/info_cmd.h b/src/zen/cmds/info_cmd.h
new file mode 100644
index 000000000..9723a075b
--- /dev/null
+++ b/src/zen/cmds/info_cmd.h
@@ -0,0 +1,24 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "../zen.h"
+
+namespace zen {
+
+class InfoCommand : public ZenCmdBase
+{
+public:
+ InfoCommand();
+ ~InfoCommand();
+
+ virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+ virtual cxxopts::Options& Options() override { return m_Options; }
+ // virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
+
+private:
+ cxxopts::Options m_Options{"info", "Show high level zen store information"};
+ std::string m_HostName;
+};
+
+} // namespace zen
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index c949008ff..10d2f5593 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -10,6 +10,7 @@
#include "cmds/cache_cmd.h"
#include "cmds/copy_cmd.h"
#include "cmds/dedup_cmd.h"
+#include "cmds/info_cmd.h"
#include "cmds/print_cmd.h"
#include "cmds/projectstore_cmd.h"
#include "cmds/rpcreplay_cmd.h"
@@ -273,6 +274,7 @@ main(int argc, char** argv)
GcStatusCommand GcStatusCmd;
GcStopCommand GcStopCmd;
ImportOplogCommand ImportOplogCmd;
+ InfoCommand InfoCmd;
JobCommand JobCmd;
OplogMirrorCommand OplogMirrorCmd;
PrintCommand PrintCmd;
@@ -316,6 +318,7 @@ main(int argc, char** argv)
{"gc-status", &GcStatusCmd, "Garbage collect zen storage status check"},
{"gc-stop", &GcStopCmd, "Request cancel of running garbage collection in zen storage"},
{"gc", &GcCmd, "Garbage collect zen storage"},
+ {"info", &InfoCmd, "Show high level Zen server information"},
{"jobs", &JobCmd, "Show/cancel zen background jobs"},
{"logs", &LoggingCmd, "Show/control zen logging"},
{"oplog-create", &CreateOplogCmd, "Create a project oplog"},
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index c2df847ad..cc1ffdcdc 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -3,6 +3,7 @@
#include "admin.h"
#include <zencore/compactbinarybuilder.h>
+#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/jobqueue.h>
#include <zencore/logging.h>
@@ -20,24 +21,86 @@
#include <zenstore/gc.h>
#include "cache/structuredcachestore.h"
+#include "config.h"
#include "projectstore/projectstore.h"
#include <chrono>
namespace zen {
-HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
- JobQueue& BackgroundJobQueue,
- ZenCacheStore* CacheStore,
- CidStore* CidStore,
- ProjectStore* ProjectStore,
- const LogPaths& LogPaths)
+struct DirStats
+{
+ uint64_t FileCount = 0;
+ uint64_t DirCount = 0;
+ uint64_t ByteCount = 0;
+};
+
+DirStats
+GetStatsForDirectory(std::filesystem::path Dir)
+{
+ if (!std::filesystem::exists(Dir))
+ return {};
+
+ FileSystemTraversal Traversal;
+
+ struct StatsTraversal : public FileSystemTraversal::TreeVisitor
+ {
+ virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+ {
+ ZEN_UNUSED(Parent, File);
+ ++TotalFileCount;
+ TotalBytes += FileSize;
+ }
+ virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override
+ {
+ ++TotalDirCount;
+ return true;
+ }
+
+ uint64_t TotalBytes = 0;
+ uint64_t TotalFileCount = 0;
+ uint64_t TotalDirCount = 0;
+
+ DirStats GetStats() { return {.FileCount = TotalFileCount, .DirCount = TotalDirCount, .ByteCount = TotalBytes}; }
+ };
+
+ StatsTraversal DirTraverser;
+ Traversal.TraverseFileSystem(Dir, DirTraverser);
+
+ return DirTraverser.GetStats();
+}
+
+struct StateDiskStats
+{
+ DirStats CacheStats;
+ DirStats CasStats;
+ DirStats ProjectStats;
+};
+
+StateDiskStats
+GetStatsForStateDirectory(std::filesystem::path StateDir)
+{
+ StateDiskStats Stats;
+ Stats.CacheStats = GetStatsForDirectory(StateDir / "cache");
+ Stats.CasStats = GetStatsForDirectory(StateDir / "cas");
+ Stats.ProjectStats = GetStatsForDirectory(StateDir / "projects");
+ return Stats;
+}
+
+HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
+ JobQueue& BackgroundJobQueue,
+ ZenCacheStore* CacheStore,
+ CidStore* CidStore,
+ ProjectStore* ProjectStore,
+ const LogPaths& LogPaths,
+ const ZenServerOptions& ServerOptions)
: m_GcScheduler(Scheduler)
, m_BackgroundJobQueue(BackgroundJobQueue)
, m_CacheStore(CacheStore)
, m_CidStore(CidStore)
, m_ProjectStore(ProjectStore)
, m_LogPaths(LogPaths)
+, m_ServerOptions(ServerOptions)
{
using namespace std::literals;
@@ -509,6 +572,60 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
#endif // ZEN_WITH_TRACE
m_Router.RegisterRoute(
+ "info",
+ [this](HttpRouterRequest& Req) {
+ CbObjectWriter Obj;
+
+ Obj << "root" << m_ServerOptions.SystemRootDir.generic_wstring();
+ Obj << "install" << (m_ServerOptions.SystemRootDir / "Install").generic_wstring();
+
+ Obj.BeginObject("primary");
+ Obj << "data" << m_ServerOptions.DataDir.generic_wstring();
+
+ try
+ {
+ auto Stats = GetStatsForStateDirectory(m_ServerOptions.DataDir);
+
+ auto EmitStats = [&](std::string_view Tag, const DirStats& Stats) {
+ Obj.BeginObject(Tag);
+ Obj << "bytes" << Stats.ByteCount;
+ Obj << "files" << Stats.FileCount;
+ Obj << "dirs" << Stats.DirCount;
+ Obj.EndObject();
+ };
+
+ EmitStats("cache", Stats.CacheStats);
+ EmitStats("cas", Stats.CasStats);
+ EmitStats("project", Stats.ProjectStats);
+ }
+ catch (std::exception& Ex)
+ {
+ ZEN_WARN("exception in disk stats gathering for '{}': {}", m_ServerOptions.DataDir, Ex.what());
+ }
+ Obj.EndObject();
+
+ try
+ {
+ std::vector<CbObject> Manifests = ReadAllCentralManifests(m_ServerOptions.SystemRootDir);
+
+ Obj.BeginArray("known");
+
+ for (const auto& Manifest : Manifests)
+ {
+ Obj.AddObject(Manifest);
+ }
+
+ Obj.EndArray();
+ }
+ catch (std::exception& Ex)
+ {
+ ZEN_WARN("exception in state gathering for '{}': {}", m_ServerOptions.SystemRootDir, Ex.what());
+ }
+ Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
+ },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
"logs",
[this](HttpRouterRequest& Req) {
CbObjectWriter Obj;
diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h
index 9d8bdfe50..563c4f536 100644
--- a/src/zenserver/admin/admin.h
+++ b/src/zenserver/admin/admin.h
@@ -12,6 +12,7 @@ class JobQueue;
class ZenCacheStore;
class CidStore;
class ProjectStore;
+struct ZenServerOptions;
class HttpAdminService : public zen::HttpService
{
@@ -22,25 +23,27 @@ public:
std::filesystem::path HttpLogPath;
std::filesystem::path CacheLogPath;
};
- HttpAdminService(GcScheduler& Scheduler,
- JobQueue& BackgroundJobQueue,
- ZenCacheStore* CacheStore,
- CidStore* CidStore,
- ProjectStore* ProjectStore,
- const LogPaths& LogPaths);
+ HttpAdminService(GcScheduler& Scheduler,
+ JobQueue& BackgroundJobQueue,
+ ZenCacheStore* CacheStore,
+ CidStore* CidStore,
+ ProjectStore* ProjectStore,
+ const LogPaths& LogPaths,
+ const ZenServerOptions& ServerOptions);
~HttpAdminService();
virtual const char* BaseUri() const override;
virtual void HandleRequest(zen::HttpServerRequest& Request) override;
private:
- HttpRequestRouter m_Router;
- GcScheduler& m_GcScheduler;
- JobQueue& m_BackgroundJobQueue;
- ZenCacheStore* m_CacheStore;
- CidStore* m_CidStore;
- ProjectStore* m_ProjectStore;
- LogPaths m_LogPaths;
+ HttpRequestRouter m_Router;
+ GcScheduler& m_GcScheduler;
+ JobQueue& m_BackgroundJobQueue;
+ ZenCacheStore* m_CacheStore;
+ CidStore* m_CidStore;
+ ProjectStore* m_ProjectStore;
+ LogPaths m_LogPaths;
+ const ZenServerOptions& m_ServerOptions;
};
} // namespace zen
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 5f2c3351e..e3286bfb8 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -5,6 +5,8 @@
#include "config/luaconfig.h"
#include "diag/logging.h"
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryvalidation.h>
#include <zencore/crypto.h>
#include <zencore/except.h>
#include <zencore/fmtutils.h>
@@ -41,7 +43,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
namespace zen {
std::filesystem::path
-PickDefaultStateDirectory()
+PickDefaultSystemRootDirectory()
{
// Pick sensible default
PWSTR ProgramDataDir = nullptr;
@@ -50,7 +52,7 @@ PickDefaultStateDirectory()
if (SUCCEEDED(hRes))
{
std::filesystem::path FinalPath(ProgramDataDir);
- FinalPath /= L"Epic\\Zen\\Data";
+ FinalPath /= L"Epic\\Zen";
::CoTaskMemFree(ProgramDataDir);
return FinalPath;
@@ -66,7 +68,7 @@ PickDefaultStateDirectory()
namespace zen {
std::filesystem::path
-PickDefaultStateDirectory()
+PickDefaultSystemRootDirectory()
{
int UserId = getuid();
const passwd* Passwd = getpwuid(UserId);
@@ -79,6 +81,62 @@ PickDefaultStateDirectory()
namespace zen {
+std::filesystem::path
+PickDefaultStateDirectory(std::filesystem::path SystemRoot)
+{
+ if (SystemRoot.empty())
+ return SystemRoot;
+
+ return SystemRoot / "Data";
+}
+
+void
+EmitCentralManifest(const std::filesystem::path& SystemRoot, Oid Identifier, CbObject Manifest, std::filesystem::path ManifestPath)
+{
+ CbObjectWriter Cbo;
+ Cbo << "path" << ManifestPath.generic_wstring();
+ Cbo << "manifest" << Manifest;
+
+ const std::filesystem::path StatesPath = SystemRoot / "States";
+
+ CreateDirectories(StatesPath);
+ WriteFile(StatesPath / fmt::format("{}", Identifier), Cbo.Save().GetBuffer().AsIoBuffer());
+}
+
+std::vector<CbObject>
+ReadAllCentralManifests(const std::filesystem::path& SystemRoot)
+{
+ std::vector<CbObject> Manifests;
+
+ DirectoryContent Content;
+ GetDirectoryContent(SystemRoot / "States", DirectoryContent::IncludeFilesFlag, Content);
+
+ for (std::filesystem::path& File : Content.Files)
+ {
+ try
+ {
+ FileContents FileData = ReadFile(File);
+ IoBuffer DataBuffer = FileData.Flatten();
+ CbValidateError ValidateError = ValidateCompactBinary(DataBuffer, CbValidateMode::All);
+
+ if (ValidateError == CbValidateError::None)
+ {
+ Manifests.push_back(LoadCompactBinaryObject(DataBuffer));
+ }
+ else
+ {
+ ZEN_WARN("failed to load manifest '{}': {}", File, ToString(ValidateError));
+ }
+ }
+ catch (std::exception& Ex)
+ {
+ ZEN_WARN("failed to load manifest '{}': {}", File, Ex.what());
+ }
+ }
+
+ return Manifests;
+}
+
void
ValidateOptions(ZenServerOptions& ServerOptions)
{
@@ -343,6 +401,7 @@ ParseConfigFile(const std::filesystem::path& Path,
LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv);
LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.NoSentry, "no-sentry"sv);
LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryAllowPII, "sentry-allow-personal-info"sv);
+ LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv);
LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv);
LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv);
LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv);
@@ -503,6 +562,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
// stream operator to convert argv value into the options type. std::fs::path
// expects paths in streams to be quoted but argv paths are unquoted. By
// going into a std::string first, paths with whitespace parse correctly.
+ std::string SystemRootDir;
std::string DataDir;
std::string ContentDir;
std::string AbsLogFile;
@@ -525,6 +585,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
options.add_options()("help", "Show command line help");
options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(ServerOptions.IsTest)->default_value("false"));
options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::string>(DataDir));
+ options.add_options()("system-dir", "Specify system root", cxxopts::value<std::string>(SystemRootDir));
options.add_options()("snapshot-dir",
"Specify a snapshot of server state to mirror into the persistence root at startup",
cxxopts::value<std::string>(BaseSnapshotDir));
@@ -975,6 +1036,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
}
logging::RefreshLogLevels();
+ ServerOptions.SystemRootDir = MakeSafePath(SystemRootDir);
ServerOptions.DataDir = MakeSafePath(DataDir);
ServerOptions.BaseSnapshotDir = MakeSafePath(BaseSnapshotDir);
ServerOptions.ContentDir = MakeSafePath(ContentDir);
@@ -1022,9 +1084,14 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
throw;
}
+ if (ServerOptions.SystemRootDir.empty())
+ {
+ ServerOptions.SystemRootDir = PickDefaultSystemRootDirectory();
+ }
+
if (ServerOptions.DataDir.empty())
{
- ServerOptions.DataDir = PickDefaultStateDirectory();
+ ServerOptions.DataDir = PickDefaultStateDirectory(ServerOptions.SystemRootDir);
}
if (ServerOptions.AbsLogFile.empty())
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index cd2d92523..b5314b600 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -128,6 +128,7 @@ struct ZenServerOptions
zen::HttpServerConfig HttpServerConfig;
ZenStructuredCacheConfig StructuredCacheConfig;
ZenStatsConfig StatsConfig;
+ std::filesystem::path SystemRootDir; // System root directory (used for machine level config)
std::filesystem::path DataDir; // Root directory for state (used for testing)
std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental)
std::filesystem::path AbsLogFile; // Absolute path to main log file
@@ -162,4 +163,7 @@ struct ZenServerOptions
void ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions);
+void EmitCentralManifest(const std::filesystem::path& SystemRoot, Oid Identifier, CbObject Manifest, std::filesystem::path ManifestPath);
+std::vector<CbObject> ReadAllCentralManifests(const std::filesystem::path& SystemRoot);
+
} // namespace zen
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 336f715f4..f80f95f8e 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -305,7 +305,8 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
m_ProjectStore,
HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile,
.HttpLogPath = ServerOptions.DataDir / "logs" / "http.log",
- .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"});
+ .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"},
+ ServerOptions);
m_Http->RegisterService(*m_AdminService);
return EffectiveBasePort;
@@ -329,6 +330,8 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
bool UpdateManifest = false;
std::filesystem::path ManifestPath = m_DataRoot / "root_manifest";
+ Oid StateId = Oid::Zero;
+ DateTime CreatedWhen{0};
if (!WipeState)
{
@@ -365,6 +368,8 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
m_RootManifest = LoadCompactBinaryObject(Manifest);
const int32_t ManifestVersion = m_RootManifest["schema_version"].AsInt32(0);
+ StateId = m_RootManifest["state_id"].AsObjectId();
+ CreatedWhen = m_RootManifest["created"].AsDateTime();
if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION)
{
@@ -391,6 +396,20 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
}
}
+ if (StateId == Oid::Zero)
+ {
+ StateId = Oid::NewOid();
+ UpdateManifest = true;
+ }
+
+ const DateTime Now = DateTime::Now();
+
+ if (CreatedWhen.GetTicks() == 0)
+ {
+ CreatedWhen = Now;
+ UpdateManifest = true;
+ }
+
// Handle any state wipe
if (WipeState)
@@ -418,19 +437,36 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
UpdateManifest = true;
}
- if (UpdateManifest)
- {
- // Write new manifest
-
- const DateTime Now = DateTime::Now();
+ // Write manifest
+ {
CbObjectWriter Cbo;
- Cbo << "schema_version" << ZEN_CFG_SCHEMA_VERSION << "created" << Now << "updated" << Now << "state_id" << Oid::NewOid();
+ Cbo << "schema_version" << ZEN_CFG_SCHEMA_VERSION << "created" << CreatedWhen << "updated" << Now << "state_id" << StateId;
m_RootManifest = Cbo.Save();
- WriteFile(ManifestPath, m_RootManifest.GetBuffer().AsIoBuffer());
+ if (UpdateManifest)
+ {
+ IoBuffer ManifestBuffer = m_RootManifest.GetBuffer().AsIoBuffer();
+
+ WriteFile(ManifestPath, ManifestBuffer);
+ }
+
+ if (!ServerOptions.IsTest)
+ {
+ try
+ {
+ EmitCentralManifest(ServerOptions.SystemRootDir, StateId, m_RootManifest, ManifestPath);
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_WARN("Unable to emit central manifest: ", Ex.what());
+ }
+ }
}
+
+ // Write state marker
+
{
std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker";
static const std::string_view StateMarkerContent = "deleting this file will cause " ZEN_APP_NAME " to exit"sv;
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-24 05:56:41 +0000