Merge branch 'main' of https://github.com/EpicGames/zen

12 files changed, 209 insertions, 13 deletions

diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
index 36ecbd9a7..ee2f3c3c7 100644
--- a/zencore/include/zencore/iobuffer.h
+++ b/zencore/include/zencore/iobuffer.h
@@ -25,6 +25,7 @@ enum class ZenContentType : uint8_t
 	kCbPackageOffer		= 6,
 	kCompressedBinary	= 7,
 	kUnknownContentType = 8,
+	kHTML				= 9,
 	kCOUNT
 };
 
@@ -54,6 +55,8 @@ ToString(ZenContentType ContentType)
 			return "compressed-binary"sv;
 		case ZenContentType::kYAML:
 			return "yaml"sv;
+		case ZenContentType::kHTML:
+			return "html"sv;
 	}
 }
 
diff --git a/zenhttp/httpserver.cpp b/zenhttp/httpserver.cpp
index 8e5d61877..cfd1463ba 100644
--- a/zenhttp/httpserver.cpp
+++ b/zenhttp/httpserver.cpp
@@ -58,6 +58,9 @@ MapContentTypeToString(HttpContentType ContentType)
 
 		case HttpContentType::kYAML:
 			return "text/yaml"sv;
+
+		case HttpContentType::kHTML:
+			return "text/html"sv;
 	}
 }
 
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index a4a19881c..8ab0276c5 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -405,6 +405,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 	if (!Success)
 	{
 		ZEN_DEBUG("MISS - '{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType));
+		m_CacheStats.MissCount++;
 		return Request.WriteResponse(HttpResponseCode::NotFound);
 	}
 
@@ -449,6 +450,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 						 ValidCount,
 						 AttachmentCount);
 
+				m_CacheStats.MissCount++;
 				return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Missing attachments"sv);
 			}
 		}
@@ -467,6 +469,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 
 		IoBuffer Response(IoBuffer::Clone, MemStream.Data(), MemStream.Size());
 
+		m_CacheStats.HitCount++;
 		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, Response);
 	}
 	else
@@ -478,6 +481,12 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 				  ToString(Value.Value.GetContentType()),
 				  InUpstreamCache ? "UPSTREAM" : "LOCAL");
 
+		m_CacheStats.HitCount++;
+		if (InUpstreamCache)
+		{
+			m_CacheStats.UpstreamHitCount++;
+		}
+
 		Request.WriteResponse(HttpResponseCode::OK, Value.Value.GetContentType(), Value.Value);
 	}
 }
@@ -713,6 +722,7 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques
 	if (!Payload)
 	{
 		ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, Ref.PayloadId, ToString(Request.AcceptContentType()));
+		m_CacheStats.MissCount++;
 		return Request.WriteResponse(HttpResponseCode::NotFound);
 	}
 
@@ -724,6 +734,12 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques
 			  ToString(Payload.GetContentType()),
 			  InUpstreamCache ? "UPSTREAM" : "LOCAL");
 
+	m_CacheStats.HitCount++;
+	if (InUpstreamCache)
+	{
+		m_CacheStats.UpstreamHitCount++;
+	}
+
 	Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Payload);
 }
 
@@ -845,6 +861,17 @@ HttpStructuredCacheService::HandleStatusRequest(zen::HttpServerRequest& Request)
 	Cbo << "ok" << true;
 
 	EmitSnapshot("requests", m_HttpRequests, Cbo);
+
+	const uint64_t HitCount			= m_CacheStats.HitCount;
+	const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount;
+	const uint64_t MissCount		= m_CacheStats.MissCount;
+	const uint64_t TotalCount		= HitCount + MissCount;
+
+	Cbo.BeginObject("cache");
+	Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount) * 100.0) : 0.0);
+	Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) * 100.0 : 0.0);
+	Cbo.EndObject();
+
 	if (m_UpstreamCache)
 	{
 		Cbo.BeginObject("upstream");
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index 47fc173e9..a360878bd 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -71,6 +71,13 @@ private:
 		IoHash		PayloadId;
 	};
 
+	struct CacheStats
+	{
+		std::atomic_uint64_t HitCount{};
+		std::atomic_uint64_t UpstreamHitCount{};
+		std::atomic_uint64_t MissCount{};
+	};
+
 	[[nodiscard]] bool ValidateKeyUri(zen::HttpServerRequest& Request, CacheRef& OutRef);
 	void			   HandleCacheRecordRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy);
 	void			   HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy);
@@ -89,6 +96,7 @@ private:
 	std::unique_ptr<UpstreamCache> m_UpstreamCache;
 	uint64_t					   m_LastScrubTime = 0;
 	metrics::OperationTiming	   m_HttpRequests;
+	CacheStats					   m_CacheStats;
 };
 
 }  // namespace zen
diff --git a/zenserver/config.cpp b/zenserver/config.cpp
index 254032226..759534d58 100644
--- a/zenserver/config.cpp
+++ b/zenserver/config.cpp
@@ -90,6 +90,7 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
 	options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(GlobalOptions.IsTest)->default_value("false"));
 	options.add_options()("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(GlobalOptions.LogId));
 	options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::filesystem::path>(GlobalOptions.DataDir));
+	options.add_options()("content-dir", "Frontend content directory", cxxopts::value<std::filesystem::path>(GlobalOptions.ContentDir));
 
 	options
 		.add_option("lifetime", "", "owner-pid", "Specify owning process id", cxxopts::value<int>(GlobalOptions.OwnerPid), "<identifier>");
@@ -227,7 +228,7 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
 					   "",
 					   "upstream-stats",
 					   "Collect performance metrics for upstream endpoints",
-					   cxxopts::value<bool>(ServiceConfig.UpstreamCacheConfig.StatsEnabled)->default_value("true"),
+					   cxxopts::value<bool>(ServiceConfig.UpstreamCacheConfig.StatsEnabled)->default_value("false"),
 					   "");
 
 	try
diff --git a/zenserver/config.h b/zenserver/config.h
index 75c19d690..af1a24455 100644
--- a/zenserver/config.h
+++ b/zenserver/config.h
@@ -17,6 +17,7 @@ struct ZenServerOptions
 	bool				  UninstallService = false;	 // Flag used to initiate service uninstall (temporary)
 	std::string			  LogId;					 // Id for tagging log output
 	std::filesystem::path DataDir;					 // Root directory for state (used for testing)
+	std::filesystem::path ContentDir;				 // Root directory for serving frontend content (experimental)
 };
 
 struct ZenUpstreamJupiterConfig
diff --git a/zenserver/experimental/frontend.cpp b/zenserver/experimental/frontend.cpp
new file mode 100644
index 000000000..79fcf0a17
--- /dev/null
+++ b/zenserver/experimental/frontend.cpp
@@ -0,0 +1,119 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "frontend.h"
+
+#include <zencore/filesystem.h>
+#include <zencore/string.h>
+
+namespace zen {
+
+namespace html {
+
+	constexpr std::string_view Index = R"(
+<!DOCTYPE html>
+<html>
+<head>
+<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" integrity="sha384-F3w7mX95PdgyTmZZMECAngseQB83DfGTowi0iMjiWaeVhAn4FJkqJByhZMI3AhiU" crossorigin="anonymous">
+<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.min.js" integrity="sha384-skAcpIdS7UcVUC05LJ9Dxay8AXcDYfBJqt1CJ85S/CFujBsIzCIv+l9liuYLaMQ/" crossorigin="anonymous"></script>
+<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/font/bootstrap-icons.css">
+<style type="text/css">
+body {
+	background-color: #fafafa;
+}
+</style>
+<script type="text/javascript">
+	const getCacheStats = () => {
+		const opts = { headers: { "Accept": "application/json" } };
+		fetch("/z$", opts)
+			.then(response => {
+				if (!response.ok) {
+					throw Error(response.statusText);
+				}
+				return response.json();
+			})
+			.then(json => {
+				document.getElementById("status").innerHTML = "connected"
+				document.getElementById("stats").innerHTML = JSON.stringify(json, null, 4);
+			})
+			.catch(error => {
+				document.getElementById("status").innerHTML = "disconnected"
+				document.getElementById("stats").innerHTML = ""
+				console.log(error);
+			})
+			.finally(() => {
+				window.setTimeout(getCacheStats, 1000);
+			});
+	};
+	getCacheStats();
+</script>
+</head>
+<body>
+	<div class="container">
+		<div class="row">
+			<div class="text-center mt-5">
+				<pre>
+__________                  _________  __                           
+\____    /  ____    ____   /   _____/_/  |_   ____  _______   ____  
+  /     / _/ __ \  /    \  \_____  \ \   __\ /  _ \ \_  __ \_/ __ \ 
+ /     /_ \  ___/ |   |  \ /        \ |  |  (  <_> ) |  | \/\  ___/ 
+/_______ \ \___  >|___|  //_______  / |__|   \____/  |__|    \___  >
+        \/     \/      \/         \/                             \/ 
+				</pre>
+				<pre id="status"/>
+			</div>
+		</div>
+		<div class="row">
+			<pre class="mb-0">Z$:</pre>
+			<pre id="stats"></pre>
+		<div>
+	</div>
+</body>
+</html>
+)";
+
+}  // namespace html
+
+HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Directory(Directory)
+{
+}
+
+HttpFrontendService::~HttpFrontendService()
+{
+}
+
+const char*
+HttpFrontendService::BaseUri() const
+{
+	return "/dashboard";  // in order to use the root path we need to remove HttpAddUrlToUrlGroup in HttpSys.cpp
+}
+
+void
+HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
+{
+	using namespace std::literals;
+
+	if (m_Directory.empty())
+	{
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kHTML, html::Index);
+	}
+	else
+	{
+		std::string_view	  Uri = Request.RelativeUri();
+		std::filesystem::path RelPath{Uri.empty() ? "index.html" : Uri};
+		std::filesystem::path AbsPath = m_Directory / RelPath;
+
+		FileContents File = ReadFile(AbsPath);
+
+		if (!File.ErrorCode)
+		{
+			// TODO: Map file extension to MIME type
+			Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kHTML, File.Data[0]);
+		}
+		else
+		{
+			return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Ooops!"sv);
+		}
+	}
+}
+
+}  // namespace zen
diff --git a/zenserver/experimental/frontend.h b/zenserver/experimental/frontend.h
new file mode 100644
index 000000000..2ae20e940
--- /dev/null
+++ b/zenserver/experimental/frontend.h
@@ -0,0 +1,24 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenhttp/httpserver.h>
+
+#include <filesystem>
+
+namespace zen {
+
+class HttpFrontendService final : public zen::HttpService
+{
+public:
+	HttpFrontendService(std::filesystem::path Directory);
+	virtual ~HttpFrontendService();
+
+	virtual const char* BaseUri() const override;
+	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
+
+private:
+	std::filesystem::path m_Directory;
+};
+
+}  // namespace zen
diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp
index 58a5b1ff3..b1966e299 100644
--- a/zenserver/upstream/upstreamcache.cpp
+++ b/zenserver/upstream/upstreamcache.cpp
@@ -612,11 +612,6 @@ struct UpstreamStats
 			 const GetUpstreamCacheResult&						   Result,
 			 const std::vector<std::unique_ptr<UpstreamEndpoint>>& Endpoints)
 	{
-		if (!m_Enabled)
-		{
-			return;
-		}
-
 		UpstreamEndpointStats& Stats = Endpoint.Stats();
 
 		if (Result.Error)
@@ -634,7 +629,7 @@ struct UpstreamStats
 			Stats.MissCount++;
 		}
 
-		if (m_SampleCount++ % MaxSampleCount)
+		if (m_Enabled && m_SampleCount++ % MaxSampleCount)
 		{
 			Dump(Logger, Endpoints);
 		}
@@ -645,11 +640,6 @@ struct UpstreamStats
 			 const PutUpstreamCacheResult&						   Result,
 			 const std::vector<std::unique_ptr<UpstreamEndpoint>>& Endpoints)
 	{
-		if (!m_Enabled)
-		{
-			return;
-		}
-
 		UpstreamEndpointStats& Stats = Endpoint.Stats();
 		if (Result.Success)
 		{
@@ -662,7 +652,7 @@ struct UpstreamStats
 			Stats.ErrorCount++;
 		}
 
-		if (m_SampleCount++ % MaxSampleCount)
+		if (m_Enabled && m_SampleCount++ % MaxSampleCount)
 		{
 			Dump(Logger, Endpoints);
 		}
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index b45df9fef..db1be9dea 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -81,6 +81,7 @@
 #include "cache/structuredcachestore.h"
 #include "compute/apply.h"
 #include "diag/diagsvcs.h"
+#include "experimental/frontend.h"
 #include "experimental/usnjournal.h"
 #include "projectstore.h"
 #include "testing/httptest.h"
@@ -302,6 +303,12 @@ public:
 		{
 			m_Http->RegisterService(*m_HttpFunctionService);
 		}
+
+		m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot);
+		if (m_FrontendService)
+		{
+			m_Http->RegisterService(*m_FrontendService);
+		}
 	}
 
 #if ZEN_ENABLE_MESH
@@ -364,6 +371,7 @@ public:
 	void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; }
 	void SetTestMode(bool State) { m_TestMode = State; }
 	void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; }
+	void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; }
 
 	void EnsureIoRunner()
 	{
@@ -447,6 +455,7 @@ private:
 	bool				  m_IsDedicatedMode = false;
 	bool				  m_TestMode		= false;
 	std::filesystem::path m_DataRoot;
+	std::filesystem::path m_ContentRoot;
 	std::jthread		  m_IoRunner;
 	asio::io_context	  m_IoContext;
 	asio::steady_timer	  m_PidCheckTimer{m_IoContext};
@@ -471,6 +480,7 @@ private:
 	zen::HttpHealthService							 m_HealthService;
 	zen::Mesh										 m_ZenMesh{m_IoContext};
 	std::unique_ptr<zen::HttpFunctionService>		 m_HttpFunctionService;
+	std::unique_ptr<zen::HttpFrontendService>		 m_FrontendService;
 
 	bool m_DebugOptionForcedCrash = false;
 };
@@ -560,6 +570,7 @@ ZenWindowsService::Run()
 
 		ZenServer Server;
 		Server.SetDataRoot(GlobalOptions.DataDir);
+		Server.SetContentRoot(GlobalOptions.ContentDir);
 		Server.SetTestMode(GlobalOptions.IsTest);
 		Server.SetDedicatedMode(GlobalOptions.IsDedicated);
 		Server.Initialize(ServiceConfig, GlobalOptions.BasePort, GlobalOptions.OwnerPid, Entry);
diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj
index bcb7ea028..335786fbf 100644
--- a/zenserver/zenserver.vcxproj
+++ b/zenserver/zenserver.vcxproj
@@ -109,6 +109,7 @@
     <ClInclude Include="compute\apply.h" />
     <ClInclude Include="config.h" />
     <ClInclude Include="diag\logging.h" />
+    <ClInclude Include="experimental\frontend.h" />
     <ClInclude Include="resource.h" />
     <ClInclude Include="sos\sos.h" />
     <ClInclude Include="testing\httptest.h" />
@@ -132,6 +133,7 @@
     <ClCompile Include="compute\apply.cpp" />
     <ClCompile Include="config.cpp" />
     <ClCompile Include="diag\logging.cpp" />
+    <ClCompile Include="experimental\frontend.cpp" />
     <ClCompile Include="projectstore.cpp" />
     <ClCompile Include="cache\cacheagent.cpp" />
     <ClCompile Include="sos\sos.cpp" />
diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters
index 6b99ca8d7..1c5b17fee 100644
--- a/zenserver/zenserver.vcxproj.filters
+++ b/zenserver/zenserver.vcxproj.filters
@@ -38,6 +38,9 @@
     <ClInclude Include="testing\httptest.h" />
     <ClInclude Include="windows\service.h" />
     <ClInclude Include="resource.h" />
+    <ClInclude Include="experimental\frontend.h">
+      <Filter>experimental</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="zenserver.cpp" />
@@ -70,6 +73,10 @@
     </ClCompile>
     <ClCompile Include="testing\httptest.cpp" />
     <ClCompile Include="windows\service.cpp" />
+    <ClCompile Include="admin\admin.cpp" />
+    <ClCompile Include="experimental\frontend.cpp">
+      <Filter>experimental</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <Filter Include="cache">