Merge remote-tracking branch 'origin/main' into de/zen-service-command

1 files changed, 144 insertions, 28 deletions

diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index f64bc96a8..a5fd693f2 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -7,6 +7,7 @@
 
 #include "cmds/admin_cmd.h"
 #include "cmds/bench_cmd.h"
+#include "cmds/builds_cmd.h"
 #include "cmds/cache_cmd.h"
 #include "cmds/copy_cmd.h"
 #include "cmds/dedup_cmd.h"
@@ -25,14 +26,25 @@
 #include "cmds/vfs_cmd.h"
 #include "cmds/workspaces_cmd.h"
 
+#include <zencore/callstack.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
+#include <zencore/process.h>
 #include <zencore/scopeguard.h>
 #include <zencore/string.h>
+#include <zencore/trace.h>
+#include <zencore/windows.h>
 #include <zenhttp/httpcommon.h>
+#include <zenutil/logging.h>
 #include <zenutil/zenserverprocess.h>
 
+#include <zencore/memory/fmalloc.h>
+#include <zencore/memory/llm.h>
+#include <zencore/memory/memory.h>
+#include <zencore/memory/memorytrace.h>
+#include <zencore/memory/newdelete.h>
+
 #if ZEN_WITH_TESTS
 #	define ZEN_TEST_WITH_RUNNER 1
 #	include <zencore/testing.h>
@@ -47,6 +59,10 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 #include <zencore/memory/newdelete.h>
 
+#ifndef ZEN_PLATFORM_WINDOWS
+#	include <unistd.h>
+#endif
+
 //////////////////////////////////////////////////////////////////////////
 
 namespace zen {
@@ -252,7 +268,24 @@ ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec)
 	return ResolveTargetHostSpec(InHostSpec, /* out */ Dummy);
 }
 
-ProgressBar::ProgressBar(bool PlainProgress) : m_PlainProgress(PlainProgress), m_LastUpdateMS(m_SW.GetElapsedTimeMs() - 10000)
+static bool
+IsStdoutTty()
+{
+#if ZEN_PLATFORM_WINDOWS
+	static HANDLE hStdOut	= ::GetStdHandle(STD_OUTPUT_HANDLE);
+	DWORD		  dwMode	= 0;
+	static bool	  IsConsole = ::GetConsoleMode(hStdOut, &dwMode);
+	return IsConsole;
+#else
+	return isatty(fileno(stdout));
+#endif
+}
+
+ProgressBar::ProgressBar(bool PlainProgress, bool ShowDetails)
+: m_StdoutIsTty(IsStdoutTty())
+, m_PlainProgress(PlainProgress || !m_StdoutIsTty)
+, m_ShowDetails(ShowDetails)
+, m_LastUpdateMS(m_SW.GetElapsedTimeMs() - 10000)
 {
 }
 
@@ -260,7 +293,7 @@ ProgressBar::~ProgressBar()
 {
 	try
 	{
-		Finish();
+		ForceLinebreak();
 	}
 	catch (const std::exception& Ex)
 	{
@@ -271,6 +304,7 @@ ProgressBar::~ProgressBar()
 void
 ProgressBar::UpdateState(const State& NewState, bool DoLinebreak)
 {
+	ZEN_ASSERT(NewState.TotalCount >= NewState.RemainingCount);
 	if (DoLinebreak == false && m_State == NewState)
 	{
 		return;
@@ -289,7 +323,8 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak)
 
 	if (m_PlainProgress)
 	{
-		ZEN_CONSOLE("{} {}% ({})", NewState.Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS));
+		std::string Details = (m_ShowDetails && !NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : "";
+		ZEN_CONSOLE("{} {}% ({}){}", NewState.Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), Details);
 	}
 	else
 	{
@@ -309,7 +344,27 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak)
 										 ETA,
 										 NewState.Details.empty() ? "" : fmt::format(". {}", NewState.Details));
 		std::string::size_type EraseLength = m_LastOutputLength > Output.length() ? (m_LastOutputLength - Output.length()) : 0;
-		printf("%s%s%s", Output.c_str(), std::string(EraseLength, ' ').c_str(), DoLinebreak ? "\n" : "");
+
+		ExtendableStringBuilder<128> LineToPrint;
+		LineToPrint << Output << std::string(EraseLength, ' ');
+		if (DoLinebreak)
+			LineToPrint << "\n";
+
+#if ZEN_PLATFORM_WINDOWS
+		static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE);
+
+		if (m_StdoutIsTty)
+		{
+			WriteConsoleA(hStdOut, LineToPrint.c_str(), (DWORD)LineToPrint.Size(), 0, 0);
+		}
+		else
+		{
+			::WriteFile(hStdOut, (LPCVOID)LineToPrint.c_str(), (DWORD)LineToPrint.Size(), 0, 0);
+		}
+#else
+		fwrite(LineToPrint.c_str(), 1, LineToPrint.Size(), stdout);
+#endif
+
 		m_LastOutputLength = DoLinebreak ? 0 : Output.length();
 		m_State			   = NewState;
 	}
@@ -328,7 +383,7 @@ ProgressBar::ForceLinebreak()
 void
 ProgressBar::Finish()
 {
-	if (m_LastOutputLength > 0 && m_State.RemainingCount > 0)
+	if (m_LastOutputLength > 0)
 	{
 		State NewState			= m_State;
 		NewState.RemainingCount = 0;
@@ -361,30 +416,29 @@ ProgressBar::HasActiveTask() const
 int
 main(int argc, char** argv)
 {
-	using namespace zen;
-	using namespace std::literals;
-
-	zen::logging::InitializeLogging();
-
-	// Set output mode to handle virtual terminal sequences
-	zen::logging::EnableVTMode();
-	std::set_terminate([]() { ZEN_CRITICAL("Program exited abnormally via std::terminate()"); });
-
-	LoggerRef DefaultLogger = zen::logging::Default();
-	auto&	  Sinks			= DefaultLogger.SpdLogger->sinks();
-
-	Sinks.clear();
-	auto ConsoleSink = std::make_shared<spdlog::sinks::ansicolor_stdout_sink_mt>();
-	Sinks.push_back(ConsoleSink);
-
-	zen::MaximizeOpenFileCount();
+	std::vector<std::string> Args;
+#if ZEN_PLATFORM_WINDOWS
+	LPWSTR		RawCommandLine = GetCommandLine();
+	std::string CommandLine	   = zen::WideToUtf8(RawCommandLine);
+	Args					   = zen::ParseCommandLine(CommandLine);
+#else
+	Args.reserve(argc);
+	for (int I = 0; I < argc; I++)
+	{
+		Args.push_back(std::string(argv[I]));
+	}
+#endif
+	std::vector<char*> RawArgs = zen::StripCommandlineQuotes(Args);
 
-	//////////////////////////////////////////////////////////////////////////
+	argc = gsl::narrow<int>(RawArgs.size());
+	argv = RawArgs.data();
 
-	auto _ = zen::MakeGuard([] { spdlog::shutdown(); });
+	using namespace zen;
+	using namespace std::literals;
 
 	AttachCommand			 AttachCmd;
 	BenchCommand			 BenchCmd;
+	BuildsCommand			 BuildsCmd;
 	CacheDetailsCommand		 CacheDetailsCmd;
 	CacheGetCommand			 CacheGetCmd;
 	CacheGenerateCommand	 CacheGenerateCmd;
@@ -441,6 +495,7 @@ main(int argc, char** argv)
 		// clang-format off
 		{"attach",								&AttachCmd,					"Add a sponsor process to a running zen service"},
 		{"bench",								&BenchCmd,					"Utility command for benchmarking"},
+		{BuildsCommand::Name,				    &BuildsCmd,				    BuildsCommand::Description},
 		{"cache-details",						&CacheDetailsCmd,			"Details on cache"},
 		{"cache-info",							&CacheInfoCmd,				"Info on cache, namespace or bucket"},
 		{CacheGetCommand::Name,					&CacheGetCmd,				CacheGetCommand::Description},
@@ -582,15 +637,41 @@ main(int argc, char** argv)
 	GlobalOptions.PassthroughArgs		 = PassthroughArgs;
 	GlobalOptions.PassthroughArgV		 = PassthroughArgV;
 
+	std::string MemoryOptions;
+
 	std::string SubCommand = "<None>";
 
 	cxxopts::Options Options("zen", "Zen management tool");
 
 	Options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(GlobalOptions.IsDebug));
 	Options.add_options()("v, verbose", "Enable verbose logging", cxxopts::value<bool>(GlobalOptions.IsVerbose));
+	Options.add_options()("malloc", "Configure memory allocator subsystem", cxxopts::value(MemoryOptions)->default_value("mimalloc"));
 	Options.add_options()("help", "Show command line help");
 	Options.add_options()("c, command", "Sub command", cxxopts::value<std::string>(SubCommand));
 
+#if ZEN_WITH_TRACE
+	std::string TraceChannels;
+	std::string TraceHost;
+	std::string TraceFile;
+
+	Options.add_option("ue-trace",
+					   "",
+					   "trace",
+					   "Specify which trace channels should be enabled",
+					   cxxopts::value<std::string>(TraceChannels)->default_value(""),
+					   "");
+
+	Options.add_option("ue-trace",
+					   "",
+					   "tracehost",
+					   "Hostname to send the trace to",
+					   cxxopts::value<std::string>(TraceHost)->default_value(""),
+					   "");
+
+	Options
+		.add_option("ue-trace", "", "tracefile", "Path to write a trace to", cxxopts::value<std::string>(TraceFile)->default_value(""), "");
+#endif	// ZEN_WITH_TRACE
+
 	Options.parse_positional({"command"});
 
 	const bool IsNullInvoke = (argc == 1);	// If no arguments are passed we want to print usage information
@@ -632,14 +713,49 @@ main(int argc, char** argv)
 			exit(0);
 		}
 
-		if (GlobalOptions.IsDebug)
+		zen::LoggingOptions LogOptions;
+		LogOptions.IsDebug	  = GlobalOptions.IsDebug;
+		LogOptions.IsVerbose  = GlobalOptions.IsVerbose;
+		LogOptions.AllowAsync = false;
+		zen::InitializeLogging(LogOptions);
+
+		std::set_terminate([]() {
+			void*			 Frames[8];
+			uint32_t		 FrameCount = GetCallstack(2, 8, Frames);
+			CallstackFrames* Callstack	= CreateCallstack(FrameCount, Frames);
+			ZEN_CRITICAL("Program exited abnormally via std::terminate()\n{}", CallstackToString(Callstack, "  "));
+			FreeCallstack(Callstack);
+		});
+
+		zen::MaximizeOpenFileCount();
+
+		//////////////////////////////////////////////////////////////////////////
+
+		auto _ = zen::MakeGuard([] { zen::ShutdownLogging(); });
+
+#if ZEN_WITH_TRACE
+		if (TraceHost.size())
+		{
+			TraceStart("zen", TraceHost.c_str(), TraceType::Network);
+		}
+		else if (TraceFile.size())
 		{
-			logging::SetLogLevel(logging::level::Debug);
+			TraceStart("zen", TraceFile.c_str(), TraceType::File);
 		}
-		if (GlobalOptions.IsVerbose)
+		else
 		{
-			logging::SetLogLevel(logging::level::Trace);
+			TraceInit("zen");
 		}
+#endif	// ZEN_WITH_TRACE
+
+#if ZEN_WITH_MEMTRACK
+		FMalloc* TraceMalloc = MemoryTrace_Create(GMalloc);
+		if (TraceMalloc != GMalloc)
+		{
+			GMalloc = TraceMalloc;
+			MemoryTrace_Initialize();
+		}
+#endif
 
 		for (const CommandInfo& CmdInfo : Commands)
 		{