Adding an info command to display a top-level summary of disk space etc (#602)

this also adds a central, shared folder for storing information which may be found by any instance on the host. The directory is currently located alongside the default install and state directory.

Initially this is used to store a collection of known `root_manifest` locations and a copy of the latest manifest version which allow us to find all known locations where zen state is present.

8 files changed, 336 insertions, 31 deletions

diff --git a/src/zen/cmds/info_cmd.cpp b/src/zen/cmds/info_cmd.cpp
new file mode 100644
index 000000000..aec8ca46b
--- /dev/null
+++ b/src/zen/cmds/info_cmd.cpp
@@ -0,0 +1,51 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "info_cmd.h"
+
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/string.h>
+#include <zenhttp/httpclient.h>
+
+using namespace std::literals;
+
+namespace zen {
+
+InfoCommand::InfoCommand()
+{
+	m_Options.add_options()("h,help", "Print help");
+	m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+}
+
+InfoCommand::~InfoCommand()
+{
+}
+
+int
+InfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+	ZEN_UNUSED(GlobalOptions);
+
+	if (!ParseOptions(argc, argv))
+	{
+		return 0;
+	}
+
+	m_HostName = ResolveTargetHostSpec(m_HostName);
+
+	if (m_HostName.empty())
+	{
+		throw OptionParseException("unable to resolve server specification");
+	}
+
+	HttpClient Http(m_HostName);
+
+	if (HttpClient::Response Result = Http.Get("/admin/info", HttpClient::Accept(ZenContentType::kJSON)))
+	{
+		ZEN_CONSOLE("{}", Result.AsText());
+	}
+
+	return 0;
+}
+
+}  // namespace zen
diff --git a/src/zen/cmds/info_cmd.h b/src/zen/cmds/info_cmd.h
new file mode 100644
index 000000000..9723a075b
--- /dev/null
+++ b/src/zen/cmds/info_cmd.h
@@ -0,0 +1,24 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "../zen.h"
+
+namespace zen {
+
+class InfoCommand : public ZenCmdBase
+{
+public:
+	InfoCommand();
+	~InfoCommand();
+
+	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+	virtual cxxopts::Options& Options() override { return m_Options; }
+	// virtual ZenCmdCategory&	  CommandCategory() const override { return g_UtilitiesCategory; }
+
+private:
+	cxxopts::Options m_Options{"info", "Show high level zen store information"};
+	std::string		 m_HostName;
+};
+
+}  // namespace zen
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index c949008ff..10d2f5593 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -10,6 +10,7 @@
 #include "cmds/cache_cmd.h"
 #include "cmds/copy_cmd.h"
 #include "cmds/dedup_cmd.h"
+#include "cmds/info_cmd.h"
 #include "cmds/print_cmd.h"
 #include "cmds/projectstore_cmd.h"
 #include "cmds/rpcreplay_cmd.h"
@@ -273,6 +274,7 @@ main(int argc, char** argv)
 	GcStatusCommand			 GcStatusCmd;
 	GcStopCommand			 GcStopCmd;
 	ImportOplogCommand		 ImportOplogCmd;
+	InfoCommand				 InfoCmd;
 	JobCommand				 JobCmd;
 	OplogMirrorCommand		 OplogMirrorCmd;
 	PrintCommand			 PrintCmd;
@@ -316,6 +318,7 @@ main(int argc, char** argv)
 		{"gc-status",			&GcStatusCmd,			"Garbage collect zen storage status check"},
 		{"gc-stop",				&GcStopCmd,				"Request cancel of running garbage collection in zen storage"},
 		{"gc",					&GcCmd,					"Garbage collect zen storage"},
+		{"info",				&InfoCmd,				"Show high level Zen server information"},
 		{"jobs",				&JobCmd,				"Show/cancel zen background jobs"},
 		{"logs",				&LoggingCmd,			"Show/control zen logging"},
 		{"oplog-create",		&CreateOplogCmd,		"Create a project oplog"},
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index c2df847ad..cc1ffdcdc 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -3,6 +3,7 @@
 #include "admin.h"
 
 #include <zencore/compactbinarybuilder.h>
+#include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/jobqueue.h>
 #include <zencore/logging.h>
@@ -20,24 +21,86 @@
 #include <zenstore/gc.h>
 
 #include "cache/structuredcachestore.h"
+#include "config.h"
 #include "projectstore/projectstore.h"
 
 #include <chrono>
 
 namespace zen {
 
-HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
-								   JobQueue&	   BackgroundJobQueue,
-								   ZenCacheStore*  CacheStore,
-								   CidStore*	   CidStore,
-								   ProjectStore*   ProjectStore,
-								   const LogPaths& LogPaths)
+struct DirStats
+{
+	uint64_t FileCount = 0;
+	uint64_t DirCount  = 0;
+	uint64_t ByteCount = 0;
+};
+
+DirStats
+GetStatsForDirectory(std::filesystem::path Dir)
+{
+	if (!std::filesystem::exists(Dir))
+		return {};
+
+	FileSystemTraversal Traversal;
+
+	struct StatsTraversal : public FileSystemTraversal::TreeVisitor
+	{
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+		{
+			ZEN_UNUSED(Parent, File);
+			++TotalFileCount;
+			TotalBytes += FileSize;
+		}
+		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override
+		{
+			++TotalDirCount;
+			return true;
+		}
+
+		uint64_t TotalBytes		= 0;
+		uint64_t TotalFileCount = 0;
+		uint64_t TotalDirCount	= 0;
+
+		DirStats GetStats() { return {.FileCount = TotalFileCount, .DirCount = TotalDirCount, .ByteCount = TotalBytes}; }
+	};
+
+	StatsTraversal DirTraverser;
+	Traversal.TraverseFileSystem(Dir, DirTraverser);
+
+	return DirTraverser.GetStats();
+}
+
+struct StateDiskStats
+{
+	DirStats CacheStats;
+	DirStats CasStats;
+	DirStats ProjectStats;
+};
+
+StateDiskStats
+GetStatsForStateDirectory(std::filesystem::path StateDir)
+{
+	StateDiskStats Stats;
+	Stats.CacheStats   = GetStatsForDirectory(StateDir / "cache");
+	Stats.CasStats	   = GetStatsForDirectory(StateDir / "cas");
+	Stats.ProjectStats = GetStatsForDirectory(StateDir / "projects");
+	return Stats;
+}
+
+HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
+								   JobQueue&			   BackgroundJobQueue,
+								   ZenCacheStore*		   CacheStore,
+								   CidStore*			   CidStore,
+								   ProjectStore*		   ProjectStore,
+								   const LogPaths&		   LogPaths,
+								   const ZenServerOptions& ServerOptions)
 : m_GcScheduler(Scheduler)
 , m_BackgroundJobQueue(BackgroundJobQueue)
 , m_CacheStore(CacheStore)
 , m_CidStore(CidStore)
 , m_ProjectStore(ProjectStore)
 , m_LogPaths(LogPaths)
+, m_ServerOptions(ServerOptions)
 {
 	using namespace std::literals;
 
@@ -509,6 +572,60 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 #endif	// ZEN_WITH_TRACE
 
 	m_Router.RegisterRoute(
+		"info",
+		[this](HttpRouterRequest& Req) {
+			CbObjectWriter Obj;
+
+			Obj << "root" << m_ServerOptions.SystemRootDir.generic_wstring();
+			Obj << "install" << (m_ServerOptions.SystemRootDir / "Install").generic_wstring();
+
+			Obj.BeginObject("primary");
+			Obj << "data" << m_ServerOptions.DataDir.generic_wstring();
+
+			try
+			{
+				auto Stats = GetStatsForStateDirectory(m_ServerOptions.DataDir);
+
+				auto EmitStats = [&](std::string_view Tag, const DirStats& Stats) {
+					Obj.BeginObject(Tag);
+					Obj << "bytes" << Stats.ByteCount;
+					Obj << "files" << Stats.FileCount;
+					Obj << "dirs" << Stats.DirCount;
+					Obj.EndObject();
+				};
+
+				EmitStats("cache", Stats.CacheStats);
+				EmitStats("cas", Stats.CasStats);
+				EmitStats("project", Stats.ProjectStats);
+			}
+			catch (std::exception& Ex)
+			{
+				ZEN_WARN("exception in disk stats gathering for '{}': {}", m_ServerOptions.DataDir, Ex.what());
+			}
+			Obj.EndObject();
+
+			try
+			{
+				std::vector<CbObject> Manifests = ReadAllCentralManifests(m_ServerOptions.SystemRootDir);
+
+				Obj.BeginArray("known");
+
+				for (const auto& Manifest : Manifests)
+				{
+					Obj.AddObject(Manifest);
+				}
+
+				Obj.EndArray();
+			}
+			catch (std::exception& Ex)
+			{
+				ZEN_WARN("exception in state gathering for '{}': {}", m_ServerOptions.SystemRootDir, Ex.what());
+			}
+			Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
+		},
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
 		"logs",
 		[this](HttpRouterRequest& Req) {
 			CbObjectWriter Obj;
diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h
index 9d8bdfe50..563c4f536 100644
--- a/src/zenserver/admin/admin.h
+++ b/src/zenserver/admin/admin.h
@@ -12,6 +12,7 @@ class JobQueue;
 class ZenCacheStore;
 class CidStore;
 class ProjectStore;
+struct ZenServerOptions;
 
 class HttpAdminService : public zen::HttpService
 {
@@ -22,25 +23,27 @@ public:
 		std::filesystem::path HttpLogPath;
 		std::filesystem::path CacheLogPath;
 	};
-	HttpAdminService(GcScheduler&	 Scheduler,
-					 JobQueue&		 BackgroundJobQueue,
-					 ZenCacheStore*	 CacheStore,
-					 CidStore*		 CidStore,
-					 ProjectStore*	 ProjectStore,
-					 const LogPaths& LogPaths);
+	HttpAdminService(GcScheduler&			 Scheduler,
+					 JobQueue&				 BackgroundJobQueue,
+					 ZenCacheStore*			 CacheStore,
+					 CidStore*				 CidStore,
+					 ProjectStore*			 ProjectStore,
+					 const LogPaths&		 LogPaths,
+					 const ZenServerOptions& ServerOptions);
 	~HttpAdminService();
 
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
 
 private:
-	HttpRequestRouter m_Router;
-	GcScheduler&	  m_GcScheduler;
-	JobQueue&		  m_BackgroundJobQueue;
-	ZenCacheStore*	  m_CacheStore;
-	CidStore*		  m_CidStore;
-	ProjectStore*	  m_ProjectStore;
-	LogPaths		  m_LogPaths;
+	HttpRequestRouter		m_Router;
+	GcScheduler&			m_GcScheduler;
+	JobQueue&				m_BackgroundJobQueue;
+	ZenCacheStore*			m_CacheStore;
+	CidStore*				m_CidStore;
+	ProjectStore*			m_ProjectStore;
+	LogPaths				m_LogPaths;
+	const ZenServerOptions& m_ServerOptions;
 };
 
 }  // namespace zen
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 5f2c3351e..e3286bfb8 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -5,6 +5,8 @@
 #include "config/luaconfig.h"
 #include "diag/logging.h"
 
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryvalidation.h>
 #include <zencore/crypto.h>
 #include <zencore/except.h>
 #include <zencore/fmtutils.h>
@@ -41,7 +43,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 namespace zen {
 
 std::filesystem::path
-PickDefaultStateDirectory()
+PickDefaultSystemRootDirectory()
 {
 	// Pick sensible default
 	PWSTR	ProgramDataDir = nullptr;
@@ -50,7 +52,7 @@ PickDefaultStateDirectory()
 	if (SUCCEEDED(hRes))
 	{
 		std::filesystem::path FinalPath(ProgramDataDir);
-		FinalPath /= L"Epic\\Zen\\Data";
+		FinalPath /= L"Epic\\Zen";
 		::CoTaskMemFree(ProgramDataDir);
 
 		return FinalPath;
@@ -66,7 +68,7 @@ PickDefaultStateDirectory()
 namespace zen {
 
 std::filesystem::path
-PickDefaultStateDirectory()
+PickDefaultSystemRootDirectory()
 {
 	int			  UserId = getuid();
 	const passwd* Passwd = getpwuid(UserId);
@@ -79,6 +81,62 @@ PickDefaultStateDirectory()
 
 namespace zen {
 
+std::filesystem::path
+PickDefaultStateDirectory(std::filesystem::path SystemRoot)
+{
+	if (SystemRoot.empty())
+		return SystemRoot;
+
+	return SystemRoot / "Data";
+}
+
+void
+EmitCentralManifest(const std::filesystem::path& SystemRoot, Oid Identifier, CbObject Manifest, std::filesystem::path ManifestPath)
+{
+	CbObjectWriter Cbo;
+	Cbo << "path" << ManifestPath.generic_wstring();
+	Cbo << "manifest" << Manifest;
+
+	const std::filesystem::path StatesPath = SystemRoot / "States";
+
+	CreateDirectories(StatesPath);
+	WriteFile(StatesPath / fmt::format("{}", Identifier), Cbo.Save().GetBuffer().AsIoBuffer());
+}
+
+std::vector<CbObject>
+ReadAllCentralManifests(const std::filesystem::path& SystemRoot)
+{
+	std::vector<CbObject> Manifests;
+
+	DirectoryContent Content;
+	GetDirectoryContent(SystemRoot / "States", DirectoryContent::IncludeFilesFlag, Content);
+
+	for (std::filesystem::path& File : Content.Files)
+	{
+		try
+		{
+			FileContents	FileData	  = ReadFile(File);
+			IoBuffer		DataBuffer	  = FileData.Flatten();
+			CbValidateError ValidateError = ValidateCompactBinary(DataBuffer, CbValidateMode::All);
+
+			if (ValidateError == CbValidateError::None)
+			{
+				Manifests.push_back(LoadCompactBinaryObject(DataBuffer));
+			}
+			else
+			{
+				ZEN_WARN("failed to load manifest '{}': {}", File, ToString(ValidateError));
+			}
+		}
+		catch (std::exception& Ex)
+		{
+			ZEN_WARN("failed to load manifest '{}': {}", File, Ex.what());
+		}
+	}
+
+	return Manifests;
+}
+
 void
 ValidateOptions(ZenServerOptions& ServerOptions)
 {
@@ -343,6 +401,7 @@ ParseConfigFile(const std::filesystem::path& Path,
 	LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv);
 	LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.NoSentry, "no-sentry"sv);
 	LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryAllowPII, "sentry-allow-personal-info"sv);
+	LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv);
 	LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv);
 	LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv);
 	LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv);
@@ -503,6 +562,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 	// stream operator to convert argv value into the options type. std::fs::path
 	// expects paths in streams to be quoted but argv paths are unquoted. By
 	// going into a std::string first, paths with whitespace parse correctly.
+	std::string SystemRootDir;
 	std::string DataDir;
 	std::string ContentDir;
 	std::string AbsLogFile;
@@ -525,6 +585,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 	options.add_options()("help", "Show command line help");
 	options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(ServerOptions.IsTest)->default_value("false"));
 	options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::string>(DataDir));
+	options.add_options()("system-dir", "Specify system root", cxxopts::value<std::string>(SystemRootDir));
 	options.add_options()("snapshot-dir",
 						  "Specify a snapshot of server state to mirror into the persistence root at startup",
 						  cxxopts::value<std::string>(BaseSnapshotDir));
@@ -975,6 +1036,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 		}
 		logging::RefreshLogLevels();
 
+		ServerOptions.SystemRootDir					  = MakeSafePath(SystemRootDir);
 		ServerOptions.DataDir						  = MakeSafePath(DataDir);
 		ServerOptions.BaseSnapshotDir				  = MakeSafePath(BaseSnapshotDir);
 		ServerOptions.ContentDir					  = MakeSafePath(ContentDir);
@@ -1022,9 +1084,14 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 		throw;
 	}
 
+	if (ServerOptions.SystemRootDir.empty())
+	{
+		ServerOptions.SystemRootDir = PickDefaultSystemRootDirectory();
+	}
+
 	if (ServerOptions.DataDir.empty())
 	{
-		ServerOptions.DataDir = PickDefaultStateDirectory();
+		ServerOptions.DataDir = PickDefaultStateDirectory(ServerOptions.SystemRootDir);
 	}
 
 	if (ServerOptions.AbsLogFile.empty())
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index cd2d92523..b5314b600 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -128,6 +128,7 @@ struct ZenServerOptions
 	zen::HttpServerConfig	 HttpServerConfig;
 	ZenStructuredCacheConfig StructuredCacheConfig;
 	ZenStatsConfig			 StatsConfig;
+	std::filesystem::path	 SystemRootDir;				  // System root directory (used for machine level config)
 	std::filesystem::path	 DataDir;					  // Root directory for state (used for testing)
 	std::filesystem::path	 ContentDir;				  // Root directory for serving frontend content (experimental)
 	std::filesystem::path	 AbsLogFile;				  // Absolute path to main log file
@@ -162,4 +163,7 @@ struct ZenServerOptions
 
 void ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions);
 
+void EmitCentralManifest(const std::filesystem::path& SystemRoot, Oid Identifier, CbObject Manifest, std::filesystem::path ManifestPath);
+std::vector<CbObject> ReadAllCentralManifests(const std::filesystem::path& SystemRoot);
+
 }  // namespace zen
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 336f715f4..f80f95f8e 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -305,7 +305,8 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 										   m_ProjectStore,
 										   HttpAdminService::LogPaths{.AbsLogPath	= ServerOptions.AbsLogFile,
 																	  .HttpLogPath	= ServerOptions.DataDir / "logs" / "http.log",
-																	  .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"});
+																	  .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"},
+										   ServerOptions);
 	m_Http->RegisterService(*m_AdminService);
 
 	return EffectiveBasePort;
@@ -329,6 +330,8 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
 
 	bool				  UpdateManifest = false;
 	std::filesystem::path ManifestPath	 = m_DataRoot / "root_manifest";
+	Oid					  StateId		 = Oid::Zero;
+	DateTime			  CreatedWhen{0};
 
 	if (!WipeState)
 	{
@@ -365,6 +368,8 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
 				m_RootManifest = LoadCompactBinaryObject(Manifest);
 
 				const int32_t ManifestVersion = m_RootManifest["schema_version"].AsInt32(0);
+				StateId						  = m_RootManifest["state_id"].AsObjectId();
+				CreatedWhen					  = m_RootManifest["created"].AsDateTime();
 
 				if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION)
 				{
@@ -391,6 +396,20 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
 		}
 	}
 
+	if (StateId == Oid::Zero)
+	{
+		StateId		   = Oid::NewOid();
+		UpdateManifest = true;
+	}
+
+	const DateTime Now = DateTime::Now();
+
+	if (CreatedWhen.GetTicks() == 0)
+	{
+		CreatedWhen	   = Now;
+		UpdateManifest = true;
+	}
+
 	// Handle any state wipe
 
 	if (WipeState)
@@ -418,19 +437,36 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
 		UpdateManifest = true;
 	}
 
-	if (UpdateManifest)
-	{
-		// Write new manifest
-
-		const DateTime Now = DateTime::Now();
+	// Write manifest
 
+	{
 		CbObjectWriter Cbo;
-		Cbo << "schema_version" << ZEN_CFG_SCHEMA_VERSION << "created" << Now << "updated" << Now << "state_id" << Oid::NewOid();
+		Cbo << "schema_version" << ZEN_CFG_SCHEMA_VERSION << "created" << CreatedWhen << "updated" << Now << "state_id" << StateId;
 
 		m_RootManifest = Cbo.Save();
 
-		WriteFile(ManifestPath, m_RootManifest.GetBuffer().AsIoBuffer());
+		if (UpdateManifest)
+		{
+			IoBuffer ManifestBuffer = m_RootManifest.GetBuffer().AsIoBuffer();
+
+			WriteFile(ManifestPath, ManifestBuffer);
+		}
+
+		if (!ServerOptions.IsTest)
+		{
+			try
+			{
+				EmitCentralManifest(ServerOptions.SystemRootDir, StateId, m_RootManifest, ManifestPath);
+			}
+			catch (const std::exception& Ex)
+			{
+				ZEN_WARN("Unable to emit central manifest: ", Ex.what());
+			}
+		}
 	}
+
+	// Write state marker
+
 	{
 		std::filesystem::path		  StateMarkerPath	 = m_DataRoot / "state_marker";
 		static const std::string_view StateMarkerContent = "deleting this file will cause " ZEN_APP_NAME " to exit"sv;