Merged from upstream

1 files changed, 173 insertions, 110 deletions

diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index f9b4d5677..18c59636d 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -1,5 +1,6 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
+#include <zencore/compactbinarybuilder.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/iobuffer.h>
@@ -31,6 +32,10 @@
 #include <set>
 #include <unordered_map>
 
+#if !defined(BUILD_VERSION)
+#	define BUILD_VERSION ("dev-build")
+#endif
+
 //////////////////////////////////////////////////////////////////////////
 // We don't have any doctest code in this file but this is needed to bring
 // in some shared code into the executable
@@ -83,6 +88,8 @@
 #include "diag/diagsvcs.h"
 #include "experimental/frontend.h"
 #include "experimental/usnjournal.h"
+#include "monitoring/httpstats.h"
+#include "monitoring/httpstatus.h"
 #include "projectstore.h"
 #include "testing/httptest.h"
 #include "testing/launch.h"
@@ -96,17 +103,16 @@
 
 namespace zen {
 
-class ZenServer
-{
-	ZenServerState::ZenServerEntry* m_ServerEntry = nullptr;
+using namespace std::literals;
 
+class ZenServer : public IHttpStatusProvider
+{
 public:
 	void Initialize(ZenServiceConfig& ServiceConfig, int BasePort, int ParentPid, ZenServerState::ZenServerEntry* ServerEntry)
 	{
-		m_ServerEntry = ServerEntry;
 		using namespace fmt::literals;
-		ZEN_INFO(ZEN_APP_NAME " initializing");
 
+		m_ServerEntry			 = ServerEntry;
 		m_DebugOptionForcedCrash = ServiceConfig.ShouldCrash;
 
 		if (ParentPid)
@@ -140,6 +146,15 @@ public:
 
 		// Ok so now we're configured, let's kick things off
 
+		m_Http = zen::CreateHttpServer();
+		m_Http->Initialize(BasePort);
+		m_Http->RegisterService(m_HealthService);
+		m_Http->RegisterService(m_StatsService);
+		m_Http->RegisterService(m_StatusService);
+		m_StatusService.RegisterHandler("status", *this);
+
+		// Initialize storage and services
+
 		ZEN_INFO("initializing storage");
 
 		zen::CasStoreConfiguration Config;
@@ -167,95 +182,7 @@ public:
 
 		if (ServiceConfig.StructuredCacheEnabled)
 		{
-			using namespace std::literals;
-			auto ValueOrDefault = [](std::string_view Value, std::string_view Default) { return Value.empty() ? Default : Value; };
-
-			ZEN_INFO("instantiating structured cache service");
-			m_CacheStore = std::make_unique<ZenCacheStore>(*m_CasStore, m_DataRoot / "cache");
-
-			std::unique_ptr<zen::UpstreamCache> UpstreamCache;
-			if (ServiceConfig.UpstreamCacheConfig.CachePolicy != UpstreamCachePolicy::Disabled)
-			{
-				const ZenUpstreamCacheConfig& UpstreamConfig = ServiceConfig.UpstreamCacheConfig;
-
-				zen::UpstreamCacheOptions UpstreamOptions;
-				UpstreamOptions.ReadUpstream =
-					(uint8_t(ServiceConfig.UpstreamCacheConfig.CachePolicy) & uint8_t(UpstreamCachePolicy::Read)) != 0;
-				UpstreamOptions.WriteUpstream =
-					(uint8_t(ServiceConfig.UpstreamCacheConfig.CachePolicy) & uint8_t(UpstreamCachePolicy::Write)) != 0;
-
-				if (UpstreamConfig.UpstreamThreadCount < 32)
-				{
-					UpstreamOptions.ThreadCount = static_cast<uint32_t>(UpstreamConfig.UpstreamThreadCount);
-				}
-
-				UpstreamOptions.StatsEnabled = UpstreamConfig.StatsEnabled;
-
-				UpstreamCache = zen::MakeUpstreamCache(UpstreamOptions, *m_CacheStore, *m_CidStore);
-
-				if (!UpstreamConfig.ZenConfig.Urls.empty())
-				{
-					std::unique_ptr<zen::UpstreamEndpoint> ZenEndpoint = zen::MakeZenUpstreamEndpoint(UpstreamConfig.ZenConfig.Urls);
-					UpstreamCache->RegisterEndpoint(std::move(ZenEndpoint));
-				}
-
-				{
-					zen::CloudCacheClientOptions Options;
-					if (UpstreamConfig.JupiterConfig.UseProductionSettings)
-					{
-						Options = zen::CloudCacheClientOptions{
-							.ServiceUrl			= "https://jupiter.devtools.epicgames.com"sv,
-							.DdcNamespace		= "ue.ddc"sv,
-							.BlobStoreNamespace = "ue.ddc"sv,
-							.OAuthProvider		= "https://epicgames.okta.com/oauth2/auso645ojjWVdRI3d0x7/v1/token"sv,
-							.OAuthClientId		= "0oao91lrhqPiAlaGD0x7"sv,
-							.OAuthSecret		= "-GBWjjenhCgOwhxL5yBKNJECVIoDPH0MK4RDuN7d"sv,
-							.UseLegacyDdc		= false};
-					}
-					else if (UpstreamConfig.JupiterConfig.UseDevelopmentSettings)
-					{
-						Options = zen::CloudCacheClientOptions{
-							.ServiceUrl			= "https://jupiter.devtools-dev.epicgames.com"sv,
-							.DdcNamespace		= "ue4.ddc"sv,
-							.BlobStoreNamespace = "test.ddc"sv,
-							.OAuthProvider		= "https://epicgames.okta.com/oauth2/auso645ojjWVdRI3d0x7/v1/token"sv,
-							.OAuthClientId		= "0oao91lrhqPiAlaGD0x7"sv,
-							.OAuthSecret		= "-GBWjjenhCgOwhxL5yBKNJECVIoDPH0MK4RDuN7d"sv,
-							.UseLegacyDdc		= false};
-					}
-
-					Options.ServiceUrl		   = ValueOrDefault(UpstreamConfig.JupiterConfig.Url, Options.ServiceUrl);
-					Options.DdcNamespace	   = ValueOrDefault(UpstreamConfig.JupiterConfig.DdcNamespace, Options.DdcNamespace);
-					Options.BlobStoreNamespace = ValueOrDefault(UpstreamConfig.JupiterConfig.Namespace, Options.BlobStoreNamespace);
-					Options.OAuthProvider	   = ValueOrDefault(UpstreamConfig.JupiterConfig.OAuthProvider, Options.OAuthProvider);
-					Options.OAuthClientId	   = ValueOrDefault(UpstreamConfig.JupiterConfig.OAuthClientId, Options.OAuthClientId);
-					Options.OAuthSecret		   = ValueOrDefault(UpstreamConfig.JupiterConfig.OAuthClientSecret, Options.OAuthSecret);
-					Options.UseLegacyDdc |= UpstreamConfig.JupiterConfig.UseLegacyDdc;
-
-					if (!Options.ServiceUrl.empty())
-					{
-						std::unique_ptr<zen::UpstreamEndpoint> JupiterEndpoint = zen::MakeJupiterUpstreamEndpoint(Options);
-						UpstreamCache->RegisterEndpoint(std::move(JupiterEndpoint));
-					}
-				}
-
-				if (UpstreamCache->Initialize())
-				{
-					ZEN_INFO("upstream cache active ({})",
-							 UpstreamOptions.ReadUpstream && UpstreamOptions.WriteUpstream ? "READ|WRITE"
-							 : UpstreamOptions.ReadUpstream								   ? "READONLY"
-							 : UpstreamOptions.WriteUpstream							   ? "WRITEONLY"
-																						   : "DISABLED");
-				}
-				else
-				{
-					UpstreamCache.reset();
-					ZEN_INFO("NOT using upstream cache");
-				}
-			}
-
-			m_StructuredCacheService.reset(
-				new zen::HttpStructuredCacheService(*m_CacheStore, *m_CasStore, *m_CidStore, std::move(UpstreamCache)));
+			InitializeStructuredCache(ServiceConfig);
 		}
 		else
 		{
@@ -273,13 +200,8 @@ public:
 		}
 #endif
 
-		m_Http = zen::CreateHttpServer();
-		m_Http->Initialize(BasePort);
-		m_Http->RegisterService(m_HealthService);
-
 		m_Http->RegisterService(m_TestService);	 // NOTE: this is intentionally not limited to test mode as it's useful for diagnostics
 		m_Http->RegisterService(m_TestingService);
-
 		m_Http->RegisterService(m_AdminService);
 
 		if (m_HttpProjectService)
@@ -305,12 +227,15 @@ public:
 		}
 
 		m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot);
+
 		if (m_FrontendService)
 		{
 			m_Http->RegisterService(*m_FrontendService);
 		}
 	}
 
+	void InitializeStructuredCache(ZenServiceConfig& ServiceConfig);
+
 #if ZEN_ENABLE_MESH
 	void StartMesh(int BasePort)
 	{
@@ -351,8 +276,12 @@ public:
 
 		const bool IsInteractiveMode = zen::IsInteractiveSession() && !m_TestMode;
 
+		m_CurrentState = kRunning;
+
 		m_Http->Run(IsInteractiveMode);
 
+		m_CurrentState = kShuttingDown;
+
 		ZEN_INFO(ZEN_APP_NAME " exiting");
 
 		m_IoContext.stop();
@@ -425,6 +354,7 @@ public:
 
 	void Scrub()
 	{
+		Stopwatch Timer;
 		ZEN_INFO("Storage validation STARTING");
 
 		ScrubContext Ctx;
@@ -433,7 +363,13 @@ public:
 		m_ProjectStore->Scrub(Ctx);
 		m_StructuredCacheService->Scrub(Ctx);
 
-		ZEN_INFO("Storage validation DONE");
+		const uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
+
+		ZEN_INFO("Storage validation DONE in {}, ({} in {} chunks - {})",
+				 NiceTimeSpanMs(ElapsedTimeMs),
+				 NiceBytes(Ctx.ScrubbedBytes()),
+				 Ctx.ScrubbedChunks(),
+				 NiceByteRate(Ctx.ScrubbedBytes(), ElapsedTimeMs));
 	}
 
 	void Flush()
@@ -451,18 +387,51 @@ public:
 			m_ProjectStore->Flush();
 	}
 
+	virtual void HandleStatusRequest(HttpServerRequest& Request) override
+	{
+		CbObjectWriter Cbo;
+		Cbo << "ok" << true;
+		Cbo << "state" << ToString(m_CurrentState);
+		Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+	}
+
 private:
-	bool				  m_IsDedicatedMode = false;
-	bool				  m_TestMode		= false;
-	std::filesystem::path m_DataRoot;
-	std::filesystem::path m_ContentRoot;
-	std::jthread		  m_IoRunner;
-	asio::io_context	  m_IoContext;
-	asio::steady_timer	  m_PidCheckTimer{m_IoContext};
-	zen::ProcessMonitor	  m_ProcessMonitor;
-	zen::NamedMutex		  m_ServerMutex;
+	ZenServerState::ZenServerEntry* m_ServerEntry	  = nullptr;
+	bool							m_IsDedicatedMode = false;
+	bool							m_TestMode		  = false;
+	std::filesystem::path			m_DataRoot;
+	std::filesystem::path			m_ContentRoot;
+	std::jthread					m_IoRunner;
+	asio::io_context				m_IoContext;
+	asio::steady_timer				m_PidCheckTimer{m_IoContext};
+	zen::ProcessMonitor				m_ProcessMonitor;
+	zen::NamedMutex					m_ServerMutex;
+
+	enum ServerState
+	{
+		kInitializing,
+		kRunning,
+		kShuttingDown
+	} m_CurrentState = kInitializing;
+
+	std::string_view ToString(ServerState Value)
+	{
+		switch (Value)
+		{
+			case kInitializing:
+				return "initializing"sv;
+			case kRunning:
+				return "running"sv;
+			case kShuttingDown:
+				return "shutdown"sv;
+			default:
+				return "unknown"sv;
+		}
+	}
 
 	zen::Ref<zen::HttpServer>						 m_Http;
+	zen::HttpStatusService							 m_StatusService;
+	zen::HttpStatsService							 m_StatsService;
 	std::unique_ptr<zen::CasStore>					 m_CasStore{zen::CreateCasStore()};
 	std::unique_ptr<zen::CidStore>					 m_CidStore;
 	std::unique_ptr<zen::ZenCacheStore>				 m_CacheStore;
@@ -485,6 +454,100 @@ private:
 	bool m_DebugOptionForcedCrash = false;
 };
 
+void
+ZenServer::InitializeStructuredCache(ZenServiceConfig& ServiceConfig)
+{
+	using namespace std::literals;
+	auto ValueOrDefault = [](std::string_view Value, std::string_view Default) { return Value.empty() ? Default : Value; };
+
+	ZEN_INFO("instantiating structured cache service");
+	m_CacheStore = std::make_unique<ZenCacheStore>(*m_CasStore, m_DataRoot / "cache");
+
+	std::unique_ptr<zen::UpstreamCache> UpstreamCache;
+	if (ServiceConfig.UpstreamCacheConfig.CachePolicy != UpstreamCachePolicy::Disabled)
+	{
+		const ZenUpstreamCacheConfig& UpstreamConfig = ServiceConfig.UpstreamCacheConfig;
+
+		zen::UpstreamCacheOptions UpstreamOptions;
+		UpstreamOptions.ReadUpstream  = (uint8_t(ServiceConfig.UpstreamCacheConfig.CachePolicy) & uint8_t(UpstreamCachePolicy::Read)) != 0;
+		UpstreamOptions.WriteUpstream = (uint8_t(ServiceConfig.UpstreamCacheConfig.CachePolicy) & uint8_t(UpstreamCachePolicy::Write)) != 0;
+
+		if (UpstreamConfig.UpstreamThreadCount < 32)
+		{
+			UpstreamOptions.ThreadCount = static_cast<uint32_t>(UpstreamConfig.UpstreamThreadCount);
+		}
+
+		UpstreamOptions.StatsEnabled = UpstreamConfig.StatsEnabled;
+
+		UpstreamCache = zen::MakeUpstreamCache(UpstreamOptions, *m_CacheStore, *m_CidStore);
+
+		if (!UpstreamConfig.ZenConfig.Urls.empty())
+		{
+			std::unique_ptr<zen::UpstreamEndpoint> ZenEndpoint = zen::MakeZenUpstreamEndpoint(UpstreamConfig.ZenConfig.Urls);
+			UpstreamCache->RegisterEndpoint(std::move(ZenEndpoint));
+		}
+
+		{
+			zen::CloudCacheClientOptions Options;
+			if (UpstreamConfig.JupiterConfig.UseProductionSettings)
+			{
+				Options = zen::CloudCacheClientOptions{.ServiceUrl		   = "https://jupiter.devtools.epicgames.com"sv,
+													   .DdcNamespace	   = "ue.ddc"sv,
+													   .BlobStoreNamespace = "ue.ddc"sv,
+													   .OAuthProvider = "https://epicgames.okta.com/oauth2/auso645ojjWVdRI3d0x7/v1/token"sv,
+													   .OAuthClientId = "0oao91lrhqPiAlaGD0x7"sv,
+													   .OAuthSecret	  = "-GBWjjenhCgOwhxL5yBKNJECVIoDPH0MK4RDuN7d"sv,
+													   .UseLegacyDdc  = false};
+			}
+			else if (UpstreamConfig.JupiterConfig.UseDevelopmentSettings)
+			{
+				Options = zen::CloudCacheClientOptions{.ServiceUrl		   = "https://jupiter.devtools-dev.epicgames.com"sv,
+													   .DdcNamespace	   = "ue4.ddc"sv,
+													   .BlobStoreNamespace = "test.ddc"sv,
+													   .OAuthProvider = "https://epicgames.okta.com/oauth2/auso645ojjWVdRI3d0x7/v1/token"sv,
+													   .OAuthClientId = "0oao91lrhqPiAlaGD0x7"sv,
+													   .OAuthSecret	  = "-GBWjjenhCgOwhxL5yBKNJECVIoDPH0MK4RDuN7d"sv,
+													   .UseLegacyDdc  = false};
+			}
+
+			Options.ServiceUrl		   = ValueOrDefault(UpstreamConfig.JupiterConfig.Url, Options.ServiceUrl);
+			Options.DdcNamespace	   = ValueOrDefault(UpstreamConfig.JupiterConfig.DdcNamespace, Options.DdcNamespace);
+			Options.BlobStoreNamespace = ValueOrDefault(UpstreamConfig.JupiterConfig.Namespace, Options.BlobStoreNamespace);
+			Options.OAuthProvider	   = ValueOrDefault(UpstreamConfig.JupiterConfig.OAuthProvider, Options.OAuthProvider);
+			Options.OAuthClientId	   = ValueOrDefault(UpstreamConfig.JupiterConfig.OAuthClientId, Options.OAuthClientId);
+			Options.OAuthSecret		   = ValueOrDefault(UpstreamConfig.JupiterConfig.OAuthClientSecret, Options.OAuthSecret);
+			Options.UseLegacyDdc |= UpstreamConfig.JupiterConfig.UseLegacyDdc;
+
+			if (!Options.ServiceUrl.empty())
+			{
+				std::unique_ptr<zen::UpstreamEndpoint> JupiterEndpoint = zen::MakeJupiterUpstreamEndpoint(Options);
+				UpstreamCache->RegisterEndpoint(std::move(JupiterEndpoint));
+			}
+		}
+
+		if (UpstreamCache->Initialize())
+		{
+			ZEN_INFO("upstream cache active ({})",
+					 UpstreamOptions.ReadUpstream && UpstreamOptions.WriteUpstream ? "READ|WRITE"
+					 : UpstreamOptions.ReadUpstream								   ? "READONLY"
+					 : UpstreamOptions.WriteUpstream							   ? "WRITEONLY"
+																				   : "DISABLED");
+		}
+		else
+		{
+			UpstreamCache.reset();
+			ZEN_INFO("NOT using upstream cache");
+		}
+	}
+
+	m_StructuredCacheService.reset(new zen::HttpStructuredCacheService(*m_CacheStore,
+																	   *m_CasStore,
+																	   *m_CidStore,
+																	   m_StatsService,
+																	   m_StatusService,
+																	   std::move(UpstreamCache)));
+}
+
 }  // namespace zen
 
 class ZenWindowsService : public WindowsService
@@ -532,7 +595,7 @@ ZenWindowsService::Run()
 
 		ParseServiceConfig(GlobalOptions.DataDir, /* out */ ServiceConfig);
 
-		ZEN_INFO("zen cache server starting on port {}", GlobalOptions.BasePort);
+		ZEN_INFO(ZEN_APP_NAME " - starting on port {}, build '{}'", GlobalOptions.BasePort, BUILD_VERSION);
 
 		ZenServerState ServerState;
 		ServerState.Initialize();