Merge branch 'main' into lm/oidctoken-exe-path

27 files changed, 1831 insertions, 111 deletions

diff --git a/src/zenserver/config/config.cpp b/src/zenserver/config/config.cpp
index 18187711b..07913e891 100644
--- a/src/zenserver/config/config.cpp
+++ b/src/zenserver/config/config.cpp
@@ -132,12 +132,14 @@ ZenServerConfiguratorBase::AddCommonConfigOptions(LuaConfig::Options& LuaOptions
 	LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv);
 	LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv);
 	LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv);
+	LuaOptions.AddOption("server.otlpendpoint"sv, ServerOptions.OtelEndpointUri, "otlp-endpoint"sv);
 	LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv);
 	LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv);
 	LuaOptions.AddOption("server.quiet"sv, ServerOptions.QuietConsole, "quiet"sv);
 	LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "noconsole"sv);
 
 	////// network
+
 	LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpConfig.ServerClass, "http"sv);
 	LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpConfig.ThreadCount, "http-threads"sv);
 	LuaOptions.AddOption("network.port"sv, ServerOptions.BasePort, "port"sv);
@@ -249,17 +251,18 @@ ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerConfi
 
 	// clang-format off
 	options.add_options("logging")
-		("abslog",			"Path to log file",									cxxopts::value<std::string>(AbsLogFile))
-		("log-id",			"Specify id for adding context to log output",		cxxopts::value<std::string>(ServerOptions.LogId))
-		("quiet",			"Configure console logger output to level WARN",	cxxopts::value<bool>(ServerOptions.QuietConsole)->default_value("false"))
-		("noconsole",		"Disable console logging",							cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false"))
-		("log-trace", 		"Change selected loggers to level TRACE", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace]))
-		("log-debug", 		"Change selected loggers to level DEBUG", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug]))
-		("log-info", 		"Change selected loggers to level INFO", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info]))
-		("log-warn", 		"Change selected loggers to level WARN", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn]))
-		("log-error", 		"Change selected loggers to level ERROR", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err]))
-		("log-critical", 	"Change selected loggers to level CRITICAL", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical]))
-		("log-off", 		"Change selected loggers to level OFF", 			cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off]))
+		("abslog",			"Path to log file",											cxxopts::value<std::string>(AbsLogFile))
+		("log-id",			"Specify id for adding context to log output",				cxxopts::value<std::string>(ServerOptions.LogId))
+		("quiet",			"Configure console logger output to level WARN",			cxxopts::value<bool>(ServerOptions.QuietConsole)->default_value("false"))
+		("noconsole",		"Disable console logging",									cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false"))
+		("log-trace", 		"Change selected loggers to level TRACE", 					cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace]))
+		("log-debug", 		"Change selected loggers to level DEBUG", 					cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug]))
+		("log-info", 		"Change selected loggers to level INFO", 					cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info]))
+		("log-warn", 		"Change selected loggers to level WARN", 					cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn]))
+		("log-error", 		"Change selected loggers to level ERROR", 					cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err]))
+		("log-critical", 	"Change selected loggers to level CRITICAL", 				cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical]))
+		("log-off", 		"Change selected loggers to level OFF", 					cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off]))
+		("otlp-endpoint", 	"OpenTelemetry endpoint URI (e.g http://localhost:4318)", 	cxxopts::value<std::string>(ServerOptions.OtelEndpointUri))
 	;
 	// clang-format on
 
diff --git a/src/zenserver/config/config.h b/src/zenserver/config/config.h
index 40639da13..7c3192a1f 100644
--- a/src/zenserver/config/config.h
+++ b/src/zenserver/config/config.h
@@ -64,6 +64,8 @@ struct ZenServerConfig
 	std::string			  ChildId;			// Id assigned by parent process (used for lifetime management)
 	std::string			  LogId;			// Id for tagging log output
 	std::string			  Loggers[zen::logging::level::LogLevelCount];
+	std::string			  OtelEndpointUri;	// OpenTelemetry endpoint URI
+
 #if ZEN_WITH_TRACE
 	bool		 HasTraceCommandlineOptions = false;
 	TraceOptions TraceCmdLineOptions;
diff --git a/src/zenserver/diag/diagsvcs.cpp b/src/zenserver/diag/diagsvcs.cpp
index 8abf6e8a3..d8d53b0e3 100644
--- a/src/zenserver/diag/diagsvcs.cpp
+++ b/src/zenserver/diag/diagsvcs.cpp
@@ -28,30 +28,6 @@ GetHealthTag()
 
 using namespace std::literals;
 
-static bool
-ReadLogFile(const std::string& Path, StringBuilderBase& Out)
-{
-	try
-	{
-		constexpr auto ReadSize	  = std::size_t{4096};
-		auto		   FileStream = std::ifstream{Path};
-
-		std::string Buf(ReadSize, '\0');
-		while (FileStream.read(&Buf[0], ReadSize))
-		{
-			Out.Append(std::string_view(&Buf[0], FileStream.gcount()));
-		}
-		Out.Append(std::string_view(&Buf[0], FileStream.gcount()));
-
-		return true;
-	}
-	catch (const std::exception&)
-	{
-		Out.Reset();
-		return false;
-	}
-}
-
 HttpHealthService::HttpHealthService()
 {
 	ZEN_MEMSCOPE(GetHealthTag());
@@ -95,10 +71,9 @@ HttpHealthService::HttpHealthService()
 				return m_HealthInfo.AbsLogPath.empty() ? m_HealthInfo.DataRoot / "logs/zenserver.log" : m_HealthInfo.AbsLogPath;
 			}();
 
-			ExtendableStringBuilder<4096> Sb;
-			if (ReadLogFile(Path.string(), Sb) && Sb.Size() > 0)
+			if (IoBuffer LogBuffer = IoBufferBuilder::MakeFromFile(Path))
 			{
-				HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Sb.ToView());
+				HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, LogBuffer);
 			}
 			else
 			{
diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp
index 80da240e8..4962b9006 100644
--- a/src/zenserver/diag/logging.cpp
+++ b/src/zenserver/diag/logging.cpp
@@ -78,12 +78,11 @@ InitializeServerLogging(const ZenServerConfig& InOptions, bool WithCacheService)
 		spdlog::register_logger(ZenClientLogger);
 	}
 
-	//
-
 #if ZEN_WITH_OTEL
-	if (false)
+	if (!InOptions.OtelEndpointUri.empty())
 	{
-		auto OtelSink = std::make_shared<zen::logging::OtelHttpProtobufSink>("http://signoz.localdomain:4318");
+		// TODO: Should sanity check that endpoint is reachable? Also, a valid URI?
+		auto OtelSink = std::make_shared<zen::logging::OtelHttpProtobufSink>(InOptions.OtelEndpointUri);
 		zen::logging::Default().SpdLogger->sinks().push_back(std::move(OtelSink));
 	}
 #endif
diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip
index f9fc8a8ef..5d33302dd 100644
--- a/src/zenserver/frontend/html.zip
+++ b/src/zenserver/frontend/html.zip
diff --git a/src/zenserver/frontend/html/pages/oplog.js b/src/zenserver/frontend/html/pages/oplog.js
index bef5bacce..879fc4c97 100644
--- a/src/zenserver/frontend/html/pages/oplog.js
+++ b/src/zenserver/frontend/html/pages/oplog.js
@@ -58,12 +58,12 @@ export class Page extends ZenPage
 	{
 		const nav = section.add_widget(Toolbar);
 		const left = nav.left();
-		left.add("|&lt;")   .on_click(() => this._on_next_prev(-10e10));
-		left.add("&lt;&lt;").on_click(() => this._on_next_prev(-10));
-		left.add("prev")    .on_click(() => this._on_next_prev( -1));
-		left.add("next")    .on_click(() => this._on_next_prev(  1));
-		left.add("&gt;&gt;").on_click(() => this._on_next_prev( 10));
-		left.add("&gt;|")   .on_click(() => this._on_next_prev( 10e10));
+		left.add("|<")  .on_click(() => this._on_next_prev(-10e10));
+		left.add("<<")  .on_click(() => this._on_next_prev(-10));
+		left.add("prev").on_click(() => this._on_next_prev( -1));
+		left.add("next").on_click(() => this._on_next_prev(  1));
+		left.add(">>")  .on_click(() => this._on_next_prev( 10));
+		left.add(">|")  .on_click(() => this._on_next_prev( 10e10));
 
 		left.sep();
 		for (var count of [10, 25, 50, 100])
diff --git a/src/zenserver/frontend/html/pages/stat.js b/src/zenserver/frontend/html/pages/stat.js
index c7902d5ed..d6c7fa8e8 100644
--- a/src/zenserver/frontend/html/pages/stat.js
+++ b/src/zenserver/frontend/html/pages/stat.js
@@ -38,7 +38,6 @@ class TemporalStat
 		var content = "";
 		for (var i = 0; i < columns.length; ++i)
 		{
-			content += "<pre>";
 			const column = columns[i];
 			for (var key in column)
 			{
@@ -51,13 +50,17 @@ class TemporalStat
 				}
 				else
 					content += friendly(value);
-				content += "\n";
+				content += "\r\n";
 			}
-			content += "</pre>";
 		}
 
 		return content;
 	}
+
+	tag()
+	{
+		return "pre";
+	}
 }
 
 ////////////////////////////////////////////////////////////////////////////////
diff --git a/src/zenserver/frontend/html/util/component.js b/src/zenserver/frontend/html/util/component.js
index 205aa038e..830c6989a 100644
--- a/src/zenserver/frontend/html/util/component.js
+++ b/src/zenserver/frontend/html/util/component.js
@@ -71,8 +71,11 @@ class ComponentDom extends ComponentBase
 
 	text(value)
 	{
+		if (value != undefined && typeof value.tag === "function")
+			this.tag(value.tag())
+
 		value = (value == undefined) ? "undefined" : value.toString();
-		this._element.innerHTML = (value != "") ? value : "";
+		this._element.textContent = (value != "") ? value : "";
 		return this;
 	}
 
diff --git a/src/zenserver/frontend/html/zen.css b/src/zenserver/frontend/html/zen.css
index c52609f52..cc53c0519 100644
--- a/src/zenserver/frontend/html/zen.css
+++ b/src/zenserver/frontend/html/zen.css
@@ -168,6 +168,7 @@ a {
 		overflow:			auto;
 		overflow-wrap:		break-word;
 		background-color:	inherit;
+		white-space: 		pre-wrap;
 	}
 }
 
diff --git a/src/zenserver/hub/README.md b/src/zenserver/hub/README.md
new file mode 100644
index 000000000..322be3649
--- /dev/null
+++ b/src/zenserver/hub/README.md
@@ -0,0 +1,28 @@
+# Zen Server Hub
+
+The Zen Server can act in a "hub" mode. In this mode, the only services offered are the basic health
+and diagnostic services alongside an API to provision and deprovision Storage server instances.
+
+## Generic Server API
+
+GET `/health` - returns an `OK!` payload when all enabled services are up and responding
+
+## Hub API
+
+GET `{moduleid}` - alphanumeric identifier to identify a dataset (typically associated with a content plug-in module)
+
+GET `/hub/status` - obtain a summary of the currently live instances
+
+GET `/hub/modules/{moduleid}` - retrieve information about a module
+
+POST `/hub/modules/{moduleid}/provision` - provision service for module
+
+POST `/hub/modules/{moduleid}/deprovision` - deprovision service for module
+
+GET `/hub/stats` - retrieve stats for service
+
+## Hub Configuration
+
+The hub service can use Consul to provide status updates
+
+The hub service can emit telemetry to an Open Telemetry collector
diff --git a/src/zenserver/hub/hubservice.cpp b/src/zenserver/hub/hubservice.cpp
new file mode 100644
index 000000000..4d9da3a57
--- /dev/null
+++ b/src/zenserver/hub/hubservice.cpp
@@ -0,0 +1,867 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "hubservice.h"
+
+#include "hydration.h"
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/scopeguard.h>
+#include <zencore/system.h>
+#include <zenutil/zenserverprocess.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <EASTL/fixed_vector.h>
+#include <asio.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#include <unordered_map>
+#include <unordered_set>
+
+namespace zen {
+
+///////////////////////////////////////////////////////////////////////////
+
+/**
+ * A timeline of events with sequence IDs and timestamps. Used to
+ * track significant events for broadcasting to listeners.
+ */
+class EventTimeline
+{
+public:
+	EventTimeline() { m_Events.reserve(1024); }
+
+	~EventTimeline() {}
+
+	EventTimeline(const EventTimeline&) = delete;
+	EventTimeline& operator=(const EventTimeline&) = delete;
+
+	void RecordEvent(std::string_view EventTag, CbObject EventMetadata)
+	{
+		const uint64_t			   SequenceId = m_NextEventId++;
+		const auto				   Now		  = std::chrono::steady_clock::now();
+		RwLock::ExclusiveLockScope _(m_Lock);
+		m_Events.emplace_back(SequenceId, EventTag, Now, std::move(EventMetadata));
+	}
+
+	struct EventRecord
+	{
+		uint64_t							  SequenceId;
+		std::string							  Tag;
+		std::chrono::steady_clock::time_point Timestamp;
+		CbObject							  EventMetadata;
+
+		EventRecord(uint64_t							  InSequenceId,
+					std::string_view					  InTag,
+					std::chrono::steady_clock::time_point InTimestamp,
+					CbObject							  InEventMetadata = CbObject())
+		: SequenceId(InSequenceId)
+		, Tag(InTag)
+		, Timestamp(InTimestamp)
+		, EventMetadata(InEventMetadata)
+		{
+		}
+	};
+
+	/**
+	 * Iterate over events that have a SequenceId greater than SinceEventId
+	 *
+	 * @param Callback A callable that takes a const EventRecord&
+	 * @param SinceEventId The SequenceId to compare against
+	 */
+	void IterateEventsSince(auto&& Callback, uint64_t SinceEventId)
+	{
+		// Hold the lock for as short a time as possible
+		eastl::fixed_vector<EventRecord, 128> EventsToProcess;
+		m_Lock.WithSharedLock([&] {
+			for (auto& Event : m_Events)
+			{
+				if (Event.SequenceId > SinceEventId)
+				{
+					EventsToProcess.push_back(Event);
+				}
+			}
+		});
+
+		// Now invoke the callback outside the lock
+		for (auto& Event : EventsToProcess)
+		{
+			Callback(Event);
+		}
+	}
+
+	/**
+	 * Trim events up to (and including) the given SequenceId. Intended
+	 * to be used for cleaning up events which are not longer interesting.
+	 *
+	 * @param UpToEventId The SequenceId up to which events should be removed
+	 */
+	void TrimEventsUpTo(uint64_t UpToEventId)
+	{
+		RwLock::ExclusiveLockScope _(m_Lock);
+		auto					   It = std::remove_if(m_Events.begin(), m_Events.end(), [UpToEventId](const EventRecord& Event) {
+			  return Event.SequenceId <= UpToEventId;
+		  });
+		m_Events.erase(It, m_Events.end());
+	}
+
+private:
+	std::atomic<uint64_t> m_NextEventId{0};
+
+	RwLock					 m_Lock;
+	std::vector<EventRecord> m_Events;
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+struct ResourceMetrics
+{
+	uint64_t DiskUsageBytes	  = 0;
+	uint64_t MemoryUsageBytes = 0;
+};
+
+/**
+ * Storage Server Instance
+ *
+ * This class manages the lifecycle of a storage server instance, and
+ * provides functions to query its state. There should be one instance
+ * per module ID.
+ */
+struct StorageServerInstance
+{
+	StorageServerInstance(ZenServerEnvironment& RunEnvironment,
+						  std::string_view		ModuleId,
+						  std::filesystem::path FileHydrationPath,
+						  std::filesystem::path HydrationTempPath);
+	~StorageServerInstance();
+
+	void Provision();
+	void Deprovision();
+
+	void Hibernate();
+	void Wake();
+
+	const ResourceMetrics& GetResourceMetrics() const { return m_ResourceMetrics; }
+
+	inline std::string_view GetModuleId() const { return m_ModuleId; }
+	inline bool				IsProvisioned() const { return m_IsProvisioned.load(); }
+
+	inline uint16_t GetBasePort() const { return m_ServerInstance.GetBasePort(); }
+
+private:
+	RwLock				  m_Lock;
+	std::string			  m_ModuleId;
+	std::atomic<bool>	  m_IsProvisioned{false};
+	std::atomic<bool>	  m_IsHibernated{false};
+	ZenServerInstance	  m_ServerInstance;
+	std::filesystem::path m_BaseDir;
+	std::filesystem::path m_TempDir;
+	std::filesystem::path m_HydrationPath;
+	ResourceMetrics		  m_ResourceMetrics;
+
+	void SpawnServerProcess();
+
+	void Hydrate();
+	void Dehydrate();
+};
+
+StorageServerInstance::StorageServerInstance(ZenServerEnvironment& RunEnvironment,
+											 std::string_view	   ModuleId,
+											 std::filesystem::path FileHydrationPath,
+											 std::filesystem::path HydrationTempPath)
+: m_ModuleId(ModuleId)
+, m_ServerInstance(RunEnvironment, ZenServerInstance::ServerMode::kStorageServer)
+, m_HydrationPath(FileHydrationPath)
+{
+	m_BaseDir = RunEnvironment.CreateChildDir(ModuleId);
+	m_TempDir = HydrationTempPath / ModuleId;
+}
+
+StorageServerInstance::~StorageServerInstance()
+{
+}
+
+void
+StorageServerInstance::SpawnServerProcess()
+{
+	ZEN_ASSERT(!m_ServerInstance.IsRunning(), "Storage server instance for module '{}' is already running", m_ModuleId);
+
+	m_ServerInstance.SetServerExecutablePath(GetRunningExecutablePath());
+	m_ServerInstance.SetDataDir(m_BaseDir);
+	const uint16_t BasePort = m_ServerInstance.SpawnServerAndWaitUntilReady();
+
+	ZEN_DEBUG("Storage server instance for module '{}' started, listening on port {}", m_ModuleId, BasePort);
+
+	m_ServerInstance.EnableShutdownOnDestroy();
+}
+
+void
+StorageServerInstance::Provision()
+{
+	RwLock::ExclusiveLockScope _(m_Lock);
+
+	if (m_IsProvisioned)
+	{
+		ZEN_WARN("Storage server instance for module '{}' is already provisioned", m_ModuleId);
+
+		return;
+	}
+
+	if (m_IsHibernated)
+	{
+		Wake();
+	}
+	else
+	{
+		ZEN_INFO("Provisioning storage server instance for module '{}', at '{}'", m_ModuleId, m_BaseDir);
+
+		Hydrate();
+
+		SpawnServerProcess();
+	}
+
+	m_IsProvisioned = true;
+}
+
+void
+StorageServerInstance::Deprovision()
+{
+	RwLock::ExclusiveLockScope _(m_Lock);
+
+	if (!m_IsProvisioned)
+	{
+		ZEN_WARN("Attempted to deprovision storage server instance for module '{}' which is not provisioned", m_ModuleId);
+
+		return;
+	}
+
+	ZEN_INFO("Deprovisioning storage server instance for module '{}'", m_ModuleId);
+
+	m_ServerInstance.Shutdown();
+
+	Dehydrate();
+
+	m_IsProvisioned = false;
+}
+
+void
+StorageServerInstance::Hibernate()
+{
+	// Signal server to shut down, but keep data around for later wake
+
+	RwLock::ExclusiveLockScope _(m_Lock);
+
+	if (!m_IsProvisioned)
+	{
+		ZEN_WARN("Attempted to hibernate storage server instance for module '{}' which is not provisioned", m_ModuleId);
+
+		return;
+	}
+
+	if (m_IsHibernated)
+	{
+		ZEN_WARN("Storage server instance for module '{}' is already hibernated", m_ModuleId);
+
+		return;
+	}
+
+	if (!m_ServerInstance.IsRunning())
+	{
+		ZEN_WARN("Attempted to hibernate storage server instance for module '{}' which is not running", m_ModuleId);
+
+		// This is an unexpected state. Should consider the instance invalid?
+
+		return;
+	}
+
+	try
+	{
+		m_ServerInstance.Shutdown();
+
+		m_IsHibernated	= true;
+		m_IsProvisioned = false;
+
+		return;
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("Failed to hibernate storage server instance for module '{}': {}", m_ModuleId, Ex.what());
+	}
+}
+
+void
+StorageServerInstance::Wake()
+{
+	// Start server in-place using existing data
+
+	RwLock::ExclusiveLockScope _(m_Lock);
+
+	if (!m_IsHibernated)
+	{
+		ZEN_WARN("Attempted to wake storage server instance for module '{}' which is not hibernated", m_ModuleId);
+
+		return;
+	}
+
+	ZEN_ASSERT(!m_ServerInstance.IsRunning(), "Storage server instance for module '{}' is already running", m_ModuleId);
+
+	try
+	{
+		SpawnServerProcess();
+		m_IsHibernated = false;
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("Failed to wake storage server instance for module '{}': {}", m_ModuleId, Ex.what());
+
+		// TODO: this instance should be marked as invalid
+	}
+}
+
+void
+StorageServerInstance::Hydrate()
+{
+	HydrationConfig Config{.ServerStateDir		= m_BaseDir,
+						   .TempDir				= m_TempDir,
+						   .ModuleId			= m_ModuleId,
+						   .TargetSpecification = WideToUtf8(m_HydrationPath.native())};
+
+	std::unique_ptr<HydrationStrategyBase> Hydrator = CreateFileHydrator();
+
+	Hydrator->Configure(Config);
+	Hydrator->Hydrate();
+}
+
+void
+StorageServerInstance::Dehydrate()
+{
+	HydrationConfig Config{.ServerStateDir		= m_BaseDir,
+						   .TempDir				= m_TempDir,
+						   .ModuleId			= m_ModuleId,
+						   .TargetSpecification = WideToUtf8(m_HydrationPath.native())};
+
+	std::unique_ptr<HydrationStrategyBase> Hydrator = CreateFileHydrator();
+
+	Hydrator->Configure(Config);
+	Hydrator->Dehydrate();
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+struct HttpHubService::Impl
+{
+	Impl(const Impl&) = delete;
+	Impl& operator=(const Impl&) = delete;
+
+	Impl();
+	~Impl();
+
+	void Initialize(std::filesystem::path HubBaseDir, std::filesystem::path ChildBaseDir)
+	{
+		m_RunEnvironment.InitializeForHub(HubBaseDir, ChildBaseDir);
+		m_FileHydrationPath = m_RunEnvironment.CreateChildDir("hydration_storage");
+		ZEN_INFO("using file hydration path: '{}'", m_FileHydrationPath);
+
+		m_HydrationTempPath = m_RunEnvironment.CreateChildDir("hydration_temp");
+		ZEN_INFO("using hydration temp path: '{}'", m_HydrationTempPath);
+
+		// This is necessary to ensure the hub assigns a distinct port range.
+		// We need to do this primarily because otherwise automated tests will
+		// fail as the test runner will create processes in the default range.
+		// We should probably make this configurable or dynamic for maximum
+		// flexibility, and to allow running multiple hubs on the same host if
+		// necessary.
+		m_RunEnvironment.SetNextPortNumber(21000);
+	}
+
+	void Cleanup()
+	{
+		RwLock::ExclusiveLockScope _(m_Lock);
+		m_Instances.clear();
+	}
+
+	struct ProvisionedInstanceInfo
+	{
+		std::string BaseUri;
+		uint16_t	Port;
+	};
+
+	/**
+	 * Provision a storage server instance for the given module ID.
+	 *
+	 * @param ModuleId The ID of the module to provision.
+	 * @param OutInfo If successful, information about the provisioned instance will be returned here.
+	 * @param OutReason If unsuccessful, the reason will be returned here.
+	 */
+	bool Provision(std::string_view ModuleId, ProvisionedInstanceInfo& OutInfo, std::string& OutReason)
+	{
+		StorageServerInstance* Instance		 = nullptr;
+		bool				   IsNewInstance = false;
+		{
+			RwLock::ExclusiveLockScope _(m_Lock);
+			if (auto It = m_Instances.find(std::string(ModuleId)); It == m_Instances.end())
+			{
+				std::string Reason;
+				if (!CanProvisionInstance(ModuleId, /* out */ Reason))
+				{
+					ZEN_WARN("Cannot provision new storage server instance for module '{}': {}", ModuleId, Reason);
+
+					OutReason = Reason;
+
+					return false;
+				}
+
+				IsNewInstance = true;
+				auto NewInstance =
+					std::make_unique<StorageServerInstance>(m_RunEnvironment, ModuleId, m_FileHydrationPath, m_HydrationTempPath);
+				Instance = NewInstance.get();
+				m_Instances.emplace(std::string(ModuleId), std::move(NewInstance));
+
+				ZEN_INFO("Created new storage server instance for module '{}'", ModuleId);
+			}
+			else
+			{
+				Instance = It->second.get();
+			}
+
+			m_ProvisioningModules.emplace(std::string(ModuleId));
+		}
+
+		ZEN_ASSERT(Instance != nullptr);
+
+		auto RemoveProvisioningModule = MakeGuard([&] {
+			RwLock::ExclusiveLockScope _(m_Lock);
+			m_ProvisioningModules.erase(std::string(ModuleId));
+		});
+
+		// NOTE: this is done while not holding the lock, as provisioning may take time
+		// and we don't want to block other operations. We track which modules are being
+		// provisioned using m_ProvisioningModules, and reject attempts to provision/deprovision
+		// those modules while in this state.
+
+		UpdateStats();
+
+		try
+		{
+			Instance->Provision();
+		}
+		catch (const std::exception& Ex)
+		{
+			ZEN_ERROR("Failed to provision storage server instance for module '{}': {}", ModuleId, Ex.what());
+			if (IsNewInstance)
+			{
+				// Clean up
+				RwLock::ExclusiveLockScope _(m_Lock);
+				m_Instances.erase(std::string(ModuleId));
+			}
+			return false;
+		}
+
+		OutInfo.Port = Instance->GetBasePort();
+
+		// TODO: base URI? Would need to know what host name / IP to use
+
+		return true;
+	}
+
+	/**
+	 * Deprovision a storage server instance for the given module ID.
+	 *
+	 * @param ModuleId The ID of the module to deprovision.
+	 * @param OutReason If unsuccessful, the reason will be returned here.
+	 * @return true if the instance was found and deprovisioned, false otherwise.
+	 */
+	bool Deprovision(const std::string& ModuleId, std::string& OutReason)
+	{
+		std::unique_ptr<StorageServerInstance> Instance;
+
+		{
+			RwLock::ExclusiveLockScope _(m_Lock);
+
+			if (auto It = m_ProvisioningModules.find(ModuleId); It != m_ProvisioningModules.end())
+			{
+				OutReason = fmt::format("Module '{}' is currently being provisioned", ModuleId);
+
+				ZEN_WARN("Attempted to deprovision module '{}' which is currently being provisioned", ModuleId);
+
+				return false;
+			}
+
+			if (auto It = m_Instances.find(ModuleId); It == m_Instances.end())
+			{
+				ZEN_WARN("Attempted to deprovision non-existent module '{}'", ModuleId);
+
+				// Not found, OutReason should be empty
+				return false;
+			}
+			else
+			{
+				Instance = std::move(It->second);
+				m_Instances.erase(It);
+				m_DeprovisioningModules.emplace(ModuleId);
+			}
+		}
+
+		// The module is deprovisioned outside the lock to avoid blocking other operations.
+		//
+		// To ensure that no new provisioning can occur while we're deprovisioning,
+		// we add the module ID to m_DeprovisioningModules and remove it once
+		// deprovisioning is complete.
+
+		auto _ = MakeGuard([&] {
+			RwLock::ExclusiveLockScope _(m_Lock);
+			m_DeprovisioningModules.erase(ModuleId);
+		});
+
+		Instance->Deprovision();
+
+		return true;
+	}
+
+	/**
+	 * Find a storage server instance for the given module ID.
+	 *
+	 * Beware that as this returns a raw pointer to the instance, the caller must ensure
+	 * that the instance is not deprovisioned while in use.
+	 *
+	 * @param ModuleId The ID of the module to find.
+	 * @param OutInstance If found, the instance will be returned here.
+	 * @return true if the instance was found, false otherwise.
+	 */
+	bool Find(std::string_view ModuleId, StorageServerInstance** OutInstance = nullptr)
+	{
+		RwLock::SharedLockScope _(m_Lock);
+		if (auto It = m_Instances.find(std::string(ModuleId)); It != m_Instances.end())
+		{
+			if (OutInstance)
+			{
+				*OutInstance = It->second.get();
+			}
+			return true;
+		}
+		else if (OutInstance)
+		{
+			*OutInstance = nullptr;
+		}
+		return false;
+	}
+
+	/**
+	 * Enumerate all storage server instances.
+	 *
+	 * @param Callback The callback to invoke for each instance. Note that you should
+	 *        not do anything heavyweight in the callback as it is invoked while holding
+	 * 		  a shared lock.
+	 */
+	void EnumerateModules(auto&& Callback)
+	{
+		RwLock::SharedLockScope _(m_Lock);
+		for (auto& It : m_Instances)
+		{
+			Callback(*It.second);
+		}
+	}
+
+	int GetInstanceCount()
+	{
+		RwLock::SharedLockScope _(m_Lock);
+		return gsl::narrow_cast<int>(m_Instances.size());
+	}
+
+	inline int GetInstanceLimit() { return m_InstanceLimit; }
+	inline int GetMaxInstanceCount() { return m_MaxInstanceCount; }
+
+private:
+	ZenServerEnvironment													m_RunEnvironment;
+	std::filesystem::path													m_FileHydrationPath;
+	std::filesystem::path													m_HydrationTempPath;
+	RwLock																	m_Lock;
+	std::unordered_map<std::string, std::unique_ptr<StorageServerInstance>> m_Instances;
+	std::unordered_set<std::string>											m_DeprovisioningModules;
+	std::unordered_set<std::string>											m_ProvisioningModules;
+	int																		m_MaxInstanceCount = 0;
+	void																	UpdateStats();
+
+	// Capacity tracking
+
+	int				m_InstanceLimit = 1000;
+	ResourceMetrics m_ResourceLimits;
+	SystemMetrics	m_HostMetrics;
+
+	void UpdateCapacityMetrics();
+	bool CanProvisionInstance(std::string_view ModuleId, std::string& OutReason);
+};
+
+HttpHubService::Impl::Impl()
+{
+	m_HostMetrics					  = zen::GetSystemMetrics();
+	m_ResourceLimits.DiskUsageBytes	  = 1000ull * 1024 * 1024 * 1024;
+	m_ResourceLimits.MemoryUsageBytes = 16ull * 1024 * 1024 * 1024;
+}
+
+HttpHubService::Impl::~Impl()
+{
+	try
+	{
+		ZEN_INFO("Hub service shutting down, deprovisioning any current instances");
+
+		m_Lock.WithExclusiveLock([this] {
+			for (auto& [ModuleId, Instance] : m_Instances)
+			{
+				Instance->Deprovision();
+			}
+			m_Instances.clear();
+		});
+	}
+	catch (const std::exception& e)
+	{
+		ZEN_WARN("Exception during hub service shutdown: {}", e.what());
+	}
+}
+
+void
+HttpHubService::Impl::UpdateCapacityMetrics()
+{
+	m_HostMetrics = zen::GetSystemMetrics();
+
+	// Update per-instance metrics
+}
+
+void
+HttpHubService::Impl::UpdateStats()
+{
+	m_Lock.WithSharedLock([this] { m_MaxInstanceCount = Max(m_MaxInstanceCount, gsl::narrow_cast<int>(m_Instances.size())); });
+}
+
+bool
+HttpHubService::Impl::CanProvisionInstance(std::string_view ModuleId, std::string& OutReason)
+{
+	if (m_DeprovisioningModules.find(std::string(ModuleId)) != m_DeprovisioningModules.end())
+	{
+		OutReason = fmt::format("module '{}' is currently being deprovisioned", ModuleId);
+
+		return false;
+	}
+
+	if (m_ProvisioningModules.find(std::string(ModuleId)) != m_ProvisioningModules.end())
+	{
+		OutReason = fmt::format("module '{}' is currently being provisioned", ModuleId);
+
+		return false;
+	}
+
+	if (gsl::narrow_cast<int>(m_Instances.size()) >= m_InstanceLimit)
+	{
+		OutReason = fmt::format("instance limit exceeded ({})", m_InstanceLimit);
+
+		return false;
+	}
+
+	// TODO: handle additional resource metrics
+
+	return true;
+}
+
+///////////////////////////////////////////////////////////////////////////
+
+HttpHubService::HttpHubService(std::filesystem::path HubBaseDir, std::filesystem::path ChildBaseDir) : m_Impl(std::make_unique<Impl>())
+{
+	using namespace std::literals;
+
+	m_Impl->Initialize(HubBaseDir, ChildBaseDir);
+
+	m_Router.AddMatcher("moduleid", [](std::string_view Str) -> bool {
+		for (const auto C : Str)
+		{
+			if (std::isalnum(C) || C == '-')
+			{
+				// fine
+			}
+			else
+			{
+				// not fine
+				return false;
+			}
+		}
+
+		return true;
+	});
+
+	m_Router.RegisterRoute(
+		"status",
+		[this](HttpRouterRequest& Req) {
+			CbObjectWriter Obj;
+			Obj.BeginArray("modules");
+			m_Impl->EnumerateModules([&Obj](StorageServerInstance& Instance) {
+				Obj.BeginObject();
+				Obj << "moduleId" << Instance.GetModuleId();
+				Obj << "provisioned" << Instance.IsProvisioned();
+				Obj.EndObject();
+			});
+			Obj.EndArray();
+			Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
+		},
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
+		"modules/{moduleid}",
+		[this](HttpRouterRequest& Req) {
+			std::string_view ModuleId = Req.GetCapture(1);
+
+			if (Req.ServerRequest().RequestVerb() == HttpVerb::kDelete)
+			{
+				HandleModuleDelete(Req.ServerRequest(), ModuleId);
+			}
+			else
+			{
+				HandleModuleGet(Req.ServerRequest(), ModuleId);
+			}
+		},
+		HttpVerb::kGet | HttpVerb::kDelete);
+
+	m_Router.RegisterRoute(
+		"modules/{moduleid}/provision",
+		[this](HttpRouterRequest& Req) {
+			std::string_view ModuleId = Req.GetCapture(1);
+
+			std::string		 FailureReason = "unknown";
+			HttpResponseCode ResponseCode  = HttpResponseCode::OK;
+
+			try
+			{
+				Impl::ProvisionedInstanceInfo Info;
+				if (m_Impl->Provision(ModuleId, /* out */ Info, /* out */ FailureReason))
+				{
+					CbObjectWriter Obj;
+					Obj << "moduleId" << ModuleId;
+					Obj << "baseUri" << Info.BaseUri;
+					Obj << "port" << Info.Port;
+					Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
+
+					return;
+				}
+				else
+				{
+					ResponseCode = HttpResponseCode::BadRequest;
+				}
+			}
+			catch (const std::exception& Ex)
+			{
+				ZEN_ERROR("Exception while provisioning module '{}': {}", ModuleId, Ex.what());
+
+				FailureReason = Ex.what();
+				ResponseCode  = HttpResponseCode::InternalServerError;
+			}
+
+			Req.ServerRequest().WriteResponse(ResponseCode, HttpContentType::kText, FailureReason);
+		},
+		HttpVerb::kPost);
+
+	m_Router.RegisterRoute(
+		"modules/{moduleid}/deprovision",
+		[this](HttpRouterRequest& Req) {
+			std::string_view ModuleId	   = Req.GetCapture(1);
+			std::string		 FailureReason = "unknown";
+
+			try
+			{
+				if (!m_Impl->Deprovision(std::string(ModuleId), /* out */ FailureReason))
+				{
+					if (FailureReason.empty())
+					{
+						return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
+					}
+					else
+					{
+						return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, FailureReason);
+					}
+				}
+
+				CbObjectWriter Obj;
+				Obj << "moduleId" << ModuleId;
+
+				return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
+			}
+			catch (const std::exception& Ex)
+			{
+				ZEN_ERROR("Exception while deprovisioning module '{}': {}", ModuleId, Ex.what());
+
+				FailureReason = Ex.what();
+			}
+
+			Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, FailureReason);
+		},
+		HttpVerb::kPost);
+
+	m_Router.RegisterRoute(
+		"stats",
+		[this](HttpRouterRequest& Req) {
+			CbObjectWriter Obj;
+			Obj << "currentInstanceCount" << m_Impl->GetInstanceCount();
+			Obj << "maxInstanceCount" << m_Impl->GetMaxInstanceCount();
+			Obj << "instanceLimit" << m_Impl->GetInstanceLimit();
+			Req.ServerRequest().WriteResponse(HttpResponseCode::OK);
+		},
+		HttpVerb::kGet);
+}
+
+HttpHubService::~HttpHubService()
+{
+}
+
+const char*
+HttpHubService::BaseUri() const
+{
+	return "/hub/";
+}
+
+void
+HttpHubService::SetNotificationEndpoint(std::string_view UpstreamNotificationEndpoint, std::string_view InstanceId)
+{
+	ZEN_UNUSED(UpstreamNotificationEndpoint, InstanceId);
+	// TODO: store these for use in notifications, on some interval/criteria which is currently TBD
+}
+
+void
+HttpHubService::HandleRequest(zen::HttpServerRequest& Request)
+{
+	m_Router.HandleRequest(Request);
+}
+
+void
+HttpHubService::HandleModuleGet(HttpServerRequest& Request, std::string_view ModuleId)
+{
+	StorageServerInstance* Instance = nullptr;
+	if (!m_Impl->Find(ModuleId, &Instance))
+	{
+		Request.WriteResponse(HttpResponseCode::NotFound);
+		return;
+	}
+
+	CbObjectWriter Obj;
+	Obj << "moduleId" << Instance->GetModuleId();
+	Obj << "provisioned" << Instance->IsProvisioned();
+	Request.WriteResponse(HttpResponseCode::OK, Obj.Save());
+}
+
+void
+HttpHubService::HandleModuleDelete(HttpServerRequest& Request, std::string_view ModuleId)
+{
+	StorageServerInstance* Instance = nullptr;
+	if (!m_Impl->Find(ModuleId, &Instance))
+	{
+		Request.WriteResponse(HttpResponseCode::NotFound);
+		return;
+	}
+
+	// TODO: deprovision and nuke all related storage
+
+	CbObjectWriter Obj;
+	Obj << "moduleId" << Instance->GetModuleId();
+	Obj << "provisioned" << Instance->IsProvisioned();
+	Request.WriteResponse(HttpResponseCode::OK, Obj.Save());
+}
+
+}  // namespace zen
diff --git a/src/zenserver/hub/hubservice.h b/src/zenserver/hub/hubservice.h
new file mode 100644
index 000000000..1a5a8c57c
--- /dev/null
+++ b/src/zenserver/hub/hubservice.h
@@ -0,0 +1,42 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenhttp/httpserver.h>
+
+#include "hydration.h"
+
+namespace zen {
+
+/** ZenServer Hub Service
+ *
+ *  Manages a set of storage servers on the behalf of external clients. For
+ *  use in UEFN content worker style scenarios.
+ *
+ */
+class HttpHubService : public zen::HttpService
+{
+public:
+	HttpHubService(std::filesystem::path HubBaseDir, std::filesystem::path ChildBaseDir);
+	~HttpHubService();
+
+	HttpHubService(const HttpHubService&) = delete;
+	HttpHubService& operator=(const HttpHubService&) = delete;
+
+	virtual const char* BaseUri() const override;
+	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
+
+	void SetNotificationEndpoint(std::string_view UpstreamNotificationEndpoint, std::string_view InstanceId);
+
+private:
+	HttpRequestRouter m_Router;
+
+	struct Impl;
+
+	std::unique_ptr<Impl> m_Impl;
+
+	void HandleModuleGet(HttpServerRequest& Request, std::string_view ModuleId);
+	void HandleModuleDelete(HttpServerRequest& Request, std::string_view ModuleId);
+};
+
+}  // namespace zen
diff --git a/src/zenserver/hub/hydration.cpp b/src/zenserver/hub/hydration.cpp
new file mode 100644
index 000000000..52c17fe1a
--- /dev/null
+++ b/src/zenserver/hub/hydration.cpp
@@ -0,0 +1,119 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "hydration.h"
+
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+
+namespace zen {
+
+///////////////////////////////////////////////////////////////////////////
+
+struct FileHydrator : public HydrationStrategyBase
+{
+	virtual void Configure(const HydrationConfig& Config) override;
+	virtual void Hydrate() override;
+	virtual void Dehydrate() override;
+
+private:
+	HydrationConfig		  m_Config;
+	std::filesystem::path m_StorageModuleRootDir;
+};
+
+void
+FileHydrator::Configure(const HydrationConfig& Config)
+{
+	m_Config = Config;
+
+	std::filesystem::path ConfigPath(Utf8ToWide(m_Config.TargetSpecification));
+
+	if (!std::filesystem::exists(ConfigPath))
+	{
+		throw std::invalid_argument(fmt::format("Target does not exist: '{}'", ConfigPath.string()));
+	}
+
+	m_StorageModuleRootDir = ConfigPath / m_Config.ModuleId;
+
+	CreateDirectories(m_StorageModuleRootDir);
+}
+
+void
+FileHydrator::Hydrate()
+{
+	ZEN_INFO("Hydrating state from '{}' to '{}'", m_StorageModuleRootDir, m_Config.ServerStateDir);
+
+	// Ensure target is clean
+	ZEN_DEBUG("Wiping server state at '{}'", m_Config.ServerStateDir);
+	const bool ForceRemoveReadOnlyFiles = true;
+	CleanDirectory(m_Config.ServerStateDir, ForceRemoveReadOnlyFiles);
+
+	bool WipeServerState = false;
+
+	try
+	{
+		ZEN_DEBUG("Copying '{}' to '{}'", m_StorageModuleRootDir, m_Config.ServerStateDir);
+		CopyTree(m_StorageModuleRootDir, m_Config.ServerStateDir, {.EnableClone = true});
+	}
+	catch (std::exception& Ex)
+	{
+		ZEN_WARN("Copy failed: {}. Will wipe any partially copied state from '{}'", Ex.what(), m_Config.ServerStateDir);
+
+		// We don't do the clean right here to avoid potentially running into double-throws
+		WipeServerState = true;
+	}
+
+	if (WipeServerState)
+	{
+		ZEN_DEBUG("Cleaning server state '{}'", m_Config.ServerStateDir);
+		CleanDirectory(m_Config.ServerStateDir, ForceRemoveReadOnlyFiles);
+	}
+
+	// Note that we leave the storage state intact until next dehydration replaces the content
+}
+
+void
+FileHydrator::Dehydrate()
+{
+	ZEN_INFO("Dehydrating state from '{}' to '{}'", m_Config.ServerStateDir, m_StorageModuleRootDir);
+
+	const std::filesystem::path TargetDir = m_StorageModuleRootDir;
+
+	// Ensure target is clean. This could be replaced with an atomic copy at a later date
+	// (i.e copy into a temporary directory name and rename it once complete)
+
+	ZEN_DEBUG("Cleaning storage root '{}'", TargetDir);
+	const bool ForceRemoveReadOnlyFiles = true;
+	CleanDirectory(TargetDir, ForceRemoveReadOnlyFiles);
+
+	bool CopySuccess = true;
+
+	try
+	{
+		ZEN_DEBUG("Copying '{}' to '{}'", m_Config.ServerStateDir, TargetDir);
+		CopyTree(m_Config.ServerStateDir, TargetDir, {.EnableClone = true});
+	}
+	catch (std::exception& Ex)
+	{
+		ZEN_WARN("Copy failed: {}. Will wipe any partially copied state from '{}'", Ex.what(), m_StorageModuleRootDir);
+
+		// We don't do the clean right here to avoid potentially running into double-throws
+		CopySuccess = false;
+	}
+
+	if (!CopySuccess)
+	{
+		ZEN_DEBUG("Removing partially copied state from '{}'", TargetDir);
+		CleanDirectory(TargetDir, ForceRemoveReadOnlyFiles);
+	}
+
+	ZEN_DEBUG("Wiping server state '{}'", m_Config.ServerStateDir);
+	CleanDirectory(m_Config.ServerStateDir, ForceRemoveReadOnlyFiles);
+}
+
+std::unique_ptr<HydrationStrategyBase>
+CreateFileHydrator()
+{
+	return std::make_unique<FileHydrator>();
+}
+
+}  // namespace zen
diff --git a/src/zenserver/hub/hydration.h b/src/zenserver/hub/hydration.h
new file mode 100644
index 000000000..f86f2accf
--- /dev/null
+++ b/src/zenserver/hub/hydration.h
@@ -0,0 +1,40 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zenhubserver.h"
+
+namespace zen {
+
+struct HydrationConfig
+{
+	// Location of server state to hydrate/dehydrate
+	std::filesystem::path ServerStateDir;
+	// Temporary directory available for use during hydration/dehydration
+	std::filesystem::path TempDir;
+	// Module ID of the server state being hydrated/dehydrated
+	std::string ModuleId;
+	// Back-end specific target specification (e.g. S3 bucket, file path, etc)
+	std::string TargetSpecification;
+};
+
+/**
+ * @brief State hydration strategy interface
+ *
+ * An instance of this interface is used to perform hydration OR
+ * dehydration of server state. It's expected to be used only once
+ * and not reused.
+ *
+ */
+struct HydrationStrategyBase
+{
+	virtual ~HydrationStrategyBase() = default;
+
+	virtual void Dehydrate()							  = 0;
+	virtual void Hydrate()								  = 0;
+	virtual void Configure(const HydrationConfig& Config) = 0;
+};
+
+std::unique_ptr<HydrationStrategyBase> CreateFileHydrator();
+
+}  // namespace zen
diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp
new file mode 100644
index 000000000..7a4ba951d
--- /dev/null
+++ b/src/zenserver/hub/zenhubserver.cpp
@@ -0,0 +1,303 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zenhubserver.h"
+#include "hubservice.h"
+
+#include <zencore/fmtutils.h>
+#include <zencore/memory/llm.h>
+#include <zencore/memory/memorytrace.h>
+#include <zencore/memory/tagtrace.h>
+#include <zencore/scopeguard.h>
+#include <zencore/sentryintegration.h>
+#include <zencore/system.h>
+#include <zencore/windows.h>
+#include <zenhttp/httpapiservice.h>
+#include <zenutil/service.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <cxxopts.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+void
+ZenHubServerConfigurator::AddCliOptions(cxxopts::Options& Options)
+{
+	Options.add_option("hub",
+					   "",
+					   "upstream-notification-endpoint",
+					   "Endpoint URL for upstream notifications",
+					   cxxopts::value<std::string>(m_ServerOptions.UpstreamNotificationEndpoint)->default_value(""),
+					   "");
+
+	Options.add_option("hub",
+					   "",
+					   "instance-id",
+					   "Instance ID for use in notifications",
+					   cxxopts::value<std::string>(m_ServerOptions.InstanceId)->default_value(""),
+					   "");
+}
+
+void
+ZenHubServerConfigurator::AddConfigOptions(LuaConfig::Options& Options)
+{
+	ZEN_UNUSED(Options);
+}
+
+void
+ZenHubServerConfigurator::ApplyOptions(cxxopts::Options& Options)
+{
+	ZEN_UNUSED(Options);
+}
+
+void
+ZenHubServerConfigurator::OnConfigFileParsed(LuaConfig::Options& LuaOptions)
+{
+	ZEN_UNUSED(LuaOptions);
+}
+
+void
+ZenHubServerConfigurator::ValidateOptions()
+{
+}
+
+///////////////////////////////////////////////////////////////////////////
+
+ZenHubServer::ZenHubServer()
+{
+}
+
+ZenHubServer::~ZenHubServer()
+{
+	Cleanup();
+}
+
+int
+ZenHubServer::Initialize(const ZenHubServerConfig& ServerConfig, ZenServerState::ZenServerEntry* ServerEntry)
+{
+	ZEN_TRACE_CPU("ZenHubServer::Initialize");
+	ZEN_MEMSCOPE(GetZenserverTag());
+
+	ZEN_INFO(ZEN_APP_NAME " initializing in HUB server mode");
+
+	const int EffectiveBasePort = ZenServerBase::Initialize(ServerConfig, ServerEntry);
+	if (EffectiveBasePort < 0)
+	{
+		return EffectiveBasePort;
+	}
+
+	// This is a workaround to make sure we can have automated tests. Without
+	// this the ranges for different child zen hub processes could overlap with
+	// the main test range.
+	ZenServerEnvironment::SetBaseChildId(1000);
+
+	m_DebugOptionForcedCrash = ServerConfig.ShouldCrash;
+
+	InitializeState(ServerConfig);
+	InitializeServices(ServerConfig);
+	RegisterServices(ServerConfig);
+
+	ZenServerBase::Finalize();
+
+	return EffectiveBasePort;
+}
+
+void
+ZenHubServer::Cleanup()
+{
+	ZEN_TRACE_CPU("ZenStorageServer::Cleanup");
+	ZEN_INFO(ZEN_APP_NAME " cleaning up");
+	try
+	{
+		m_IoContext.stop();
+		if (m_IoRunner.joinable())
+		{
+			m_IoRunner.join();
+		}
+
+		if (m_Http)
+		{
+			m_Http->Close();
+		}
+	}
+	catch (const std::exception& Ex)
+	{
+		ZEN_ERROR("exception thrown during Cleanup() in {}: '{}'", ZEN_APP_NAME, Ex.what());
+	}
+}
+
+void
+ZenHubServer::InitializeState(const ZenHubServerConfig& ServerConfig)
+{
+	ZEN_UNUSED(ServerConfig);
+}
+
+void
+ZenHubServer::InitializeServices(const ZenHubServerConfig& ServerConfig)
+{
+	ZEN_UNUSED(ServerConfig);
+
+	ZEN_INFO("instantiating API service");
+	m_ApiService = std::make_unique<zen::HttpApiService>(*m_Http);
+
+	ZEN_INFO("instantiating hub service");
+	m_HubService = std::make_unique<HttpHubService>(ServerConfig.DataDir / "hub", ServerConfig.DataDir / "servers");
+	m_HubService->SetNotificationEndpoint(ServerConfig.UpstreamNotificationEndpoint, ServerConfig.InstanceId);
+}
+
+void
+ZenHubServer::RegisterServices(const ZenHubServerConfig& ServerConfig)
+{
+	ZEN_UNUSED(ServerConfig);
+
+	if (m_HubService)
+	{
+		m_Http->RegisterService(*m_HubService);
+	}
+
+	if (m_ApiService)
+	{
+		m_Http->RegisterService(*m_ApiService);
+	}
+}
+
+void
+ZenHubServer::Run()
+{
+	if (m_ProcessMonitor.IsActive())
+	{
+		CheckOwnerPid();
+	}
+
+	if (!m_TestMode)
+	{
+		// clang-format off
+		ZEN_INFO(R"(__________                ___ ___      ___.    )" "\n"
+				 R"(\____    /____   ____    /   |   \ __ _\_ |__  )" "\n"
+				 R"(  /     // __ \ /    \  /    ~    \  |  \ __ \ )" "\n"
+				 R"( /     /\  ___/|   |  \ \    Y    /  |  / \_\ \)" "\n"
+				 R"(/_______ \___  >___|  /  \___|_  /|____/|___  /)" "\n"
+				 R"(        \/   \/     \/         \/           \/ )");
+		// clang-format on
+
+		ExtendableStringBuilder<256> BuildOptions;
+		GetBuildOptions(BuildOptions, '\n');
+		ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions);
+	}
+
+	ZEN_INFO(ZEN_APP_NAME " now running as HUB (pid: {})", GetCurrentProcessId());
+
+#if ZEN_PLATFORM_WINDOWS
+	if (zen::windows::IsRunningOnWine())
+	{
+		ZEN_INFO("detected Wine session - " ZEN_APP_NAME " is not formally tested on Wine and may therefore not work or perform well");
+	}
+#endif
+
+#if ZEN_USE_SENTRY
+	ZEN_INFO("sentry crash handler {}", m_UseSentry ? "ENABLED" : "DISABLED");
+	if (m_UseSentry)
+	{
+		SentryIntegration::ClearCaches();
+	}
+#endif
+
+	if (m_DebugOptionForcedCrash)
+	{
+		ZEN_DEBUG_BREAK();
+	}
+
+	const bool IsInteractiveMode = IsInteractiveSession();	// &&!m_TestMode;
+
+	SetNewState(kRunning);
+
+	OnReady();
+
+	m_Http->Run(IsInteractiveMode);
+
+	SetNewState(kShuttingDown);
+
+	ZEN_INFO(ZEN_APP_NAME " exiting");
+}
+
+//////////////////////////////////////////////////////////////////////////////////
+
+ZenHubServerMain::ZenHubServerMain(ZenHubServerConfig& ServerOptions) : ZenServerMain(ServerOptions), m_ServerOptions(ServerOptions)
+{
+}
+
+void
+ZenHubServerMain::DoRun(ZenServerState::ZenServerEntry* Entry)
+{
+	ZenHubServer Server;
+	Server.SetDataRoot(m_ServerOptions.DataDir);
+	Server.SetContentRoot(m_ServerOptions.ContentDir);
+	Server.SetTestMode(m_ServerOptions.IsTest);
+	Server.SetDedicatedMode(m_ServerOptions.IsDedicated);
+
+	const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry);
+	if (EffectiveBasePort == -1)
+	{
+		// Server.Initialize has already logged what the issue is - just exit with failure code here.
+		std::exit(1);
+	}
+
+	Entry->EffectiveListenPort = uint16_t(EffectiveBasePort);
+	if (EffectiveBasePort != m_ServerOptions.BasePort)
+	{
+		ZEN_INFO(ZEN_APP_NAME " - relocated to base port {}", EffectiveBasePort);
+		m_ServerOptions.BasePort = EffectiveBasePort;
+	}
+
+	std::unique_ptr<std::thread> ShutdownThread;
+	std::unique_ptr<NamedEvent>	 ShutdownEvent;
+
+	ExtendableStringBuilder<64> ShutdownEventName;
+	ShutdownEventName << "Zen_" << m_ServerOptions.BasePort << "_Shutdown";
+	ShutdownEvent.reset(new NamedEvent{ShutdownEventName});
+
+	// Monitor shutdown signals
+
+	ShutdownThread.reset(new std::thread{[&] {
+		SetCurrentThreadName("shutdown_mon");
+
+		ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}' for process {}", ShutdownEventName, zen::GetCurrentProcessId());
+
+		if (ShutdownEvent->Wait())
+		{
+			ZEN_INFO("shutdown signal for pid {} received", zen::GetCurrentProcessId());
+			Server.RequestExit(0);
+		}
+		else
+		{
+			ZEN_INFO("shutdown signal wait() failed");
+		}
+	}});
+
+	auto CleanupShutdown = MakeGuard([&ShutdownEvent, &ShutdownThread] {
+		ReportServiceStatus(ServiceStatus::Stopping);
+
+		if (ShutdownEvent)
+		{
+			ShutdownEvent->Set();
+		}
+		if (ShutdownThread && ShutdownThread->joinable())
+		{
+			ShutdownThread->join();
+		}
+	});
+
+	// If we have a parent process, establish the mechanisms we need
+	// to be able to communicate readiness with the parent
+
+	Server.SetIsReadyFunc([&] {
+		std::error_code Ec;
+		m_LockFile.Update(MakeLockData(true), Ec);
+		ReportServiceStatus(ServiceStatus::Running);
+		NotifyReady();
+	});
+
+	Server.Run();
+}
+
+}  // namespace zen
diff --git a/src/zenserver/hub/zenhubserver.h b/src/zenserver/hub/zenhubserver.h
new file mode 100644
index 000000000..ac14362f0
--- /dev/null
+++ b/src/zenserver/hub/zenhubserver.h
@@ -0,0 +1,92 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zenserver.h"
+
+namespace cxxopts {
+class Options;
+}
+namespace zen::LuaConfig {
+struct Options;
+}
+
+namespace zen {
+
+class HttpApiService;
+class HttpHubService;
+
+struct ZenHubServerConfig : public ZenServerConfig
+{
+	std::string UpstreamNotificationEndpoint;
+	std::string InstanceId;	 // For use in notifications
+};
+
+struct ZenHubServerConfigurator : public ZenServerConfiguratorBase
+{
+	ZenHubServerConfigurator(ZenHubServerConfig& ServerOptions) : ZenServerConfiguratorBase(ServerOptions), m_ServerOptions(ServerOptions)
+	{
+	}
+
+	~ZenHubServerConfigurator() = default;
+
+private:
+	virtual void AddCliOptions(cxxopts::Options& Options) override;
+	virtual void AddConfigOptions(LuaConfig::Options& Options) override;
+	virtual void ApplyOptions(cxxopts::Options& Options) override;
+	virtual void OnConfigFileParsed(LuaConfig::Options& LuaOptions) override;
+	virtual void ValidateOptions() override;
+
+	ZenHubServerConfig& m_ServerOptions;
+};
+
+class ZenHubServerMain : public ZenServerMain
+{
+public:
+	ZenHubServerMain(ZenHubServerConfig& ServerOptions);
+	virtual void DoRun(ZenServerState::ZenServerEntry* Entry) override;
+
+	ZenHubServerMain(const ZenHubServerMain&) = delete;
+	ZenHubServerMain& operator=(const ZenHubServerMain&) = delete;
+
+	typedef ZenHubServerConfig		 Config;
+	typedef ZenHubServerConfigurator Configurator;
+
+private:
+	ZenHubServerConfig& m_ServerOptions;
+};
+
+class ZenHubServer : public ZenServerBase
+{
+	ZenHubServer& operator=(ZenHubServer&&) = delete;
+	ZenHubServer(ZenHubServer&&)			= delete;
+
+public:
+	ZenHubServer();
+	~ZenHubServer();
+
+	int	 Initialize(const ZenHubServerConfig& ServerConfig, ZenServerState::ZenServerEntry* ServerEntry);
+	void Run();
+	void Cleanup();
+
+	void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; }
+	void SetTestMode(bool State) { m_TestMode = State; }
+	void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; }
+	void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; }
+
+private:
+	bool				  m_IsDedicatedMode = false;
+	bool				  m_TestMode		= false;
+	std::filesystem::path m_DataRoot;
+	std::filesystem::path m_ContentRoot;
+	bool				  m_DebugOptionForcedCrash = false;
+
+	std::unique_ptr<HttpHubService> m_HubService;
+	std::unique_ptr<HttpApiService> m_ApiService;
+
+	void InitializeState(const ZenHubServerConfig& ServerConfig);
+	void InitializeServices(const ZenHubServerConfig& ServerConfig);
+	void RegisterServices(const ZenHubServerConfig& ServerConfig);
+};
+
+}  // namespace zen
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index 34848c831..3a58d1f4a 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -19,12 +19,15 @@
 #include <zencore/thread.h>
 #include <zencore/trace.h>
 #include <zentelemetry/otlptrace.h>
+#include <zenutil/commandlineoptions.h>
 #include <zenutil/service.h>
 
 #include "diag/logging.h"
 #include "storage/storageconfig.h"
 #include "storage/zenstorageserver.h"
 
+#include "hub/zenhubserver.h"
+
 #if ZEN_PLATFORM_WINDOWS
 #	include <zencore/windows.h>
 #	include <zenutil/windows/windowsservice.h>
@@ -54,8 +57,6 @@ SignalCallbackHandler(int SigNum)
 
 namespace zen {
 
-using namespace std::literals;
-
 //////////////////////////////////////////////////////////////////////////
 
 #if ZEN_PLATFORM_WINDOWS
@@ -89,6 +90,13 @@ AppMain(int argc, char* argv[])
 {
 	using namespace std::literals;
 
+	signal(SIGINT, utils::SignalCallbackHandler);
+	signal(SIGTERM, utils::SignalCallbackHandler);
+
+#if ZEN_PLATFORM_LINUX
+	IgnoreChildSignals();
+#endif
+
 	try
 	{
 		typename Main::Config ServerOptions;
@@ -188,12 +196,12 @@ AppMain(int argc, char* argv[])
 	}
 	catch (const AssertException& AssertEx)
 	{
-		fprintf(stderr, ZEN_APP_NAME " ERROR: Caught assert exception in main: '%s'", AssertEx.FullDescription().c_str());
+		fprintf(stderr, ZEN_APP_NAME " ERROR: Caught assert exception in main: '%s'\n", AssertEx.FullDescription().c_str());
 		return 1;
 	}
 	catch (const std::exception& Ex)
 	{
-		fprintf(stderr, ZEN_APP_NAME " ERROR: Caught exception in main: '%s'", Ex.what());
+		fprintf(stderr, ZEN_APP_NAME " ERROR: Caught exception in main: '%s'\n", Ex.what());
 
 		return 1;
 	}
@@ -206,6 +214,10 @@ AppMain(int argc, char* argv[])
 int
 test_main(int argc, char** argv)
 {
+#	if ZEN_PLATFORM_WINDOWS
+	setlocale(LC_ALL, "en_us.UTF8");
+#	endif	// ZEN_PLATFORM_WINDOWS
+
 	zen::logging::InitializeLogging();
 	zen::logging::SetLogLevel(zen::logging::level::Debug);
 
@@ -218,27 +230,58 @@ test_main(int argc, char** argv)
 int
 main(int argc, char* argv[])
 {
+#if ZEN_PLATFORM_WINDOWS
+	setlocale(LC_ALL, "en_us.UTF8");
+#endif	// ZEN_PLATFORM_WINDOWS
+
+	zen::CommandLineConverter ArgConverter(argc, argv);
+
 	using namespace zen;
+	using namespace std::literals;
+
+	auto _ = zen::MakeGuard([] {
+		// Allow some time for worker threads to unravel, in an effort
+		// to prevent shutdown races in TLS object destruction
+		WaitForThreads(1000);
+	});
+
+	enum
+	{
+		kHub,
+		kStore,
+		kTest
+	} ServerMode = kStore;
 
 	if (argc >= 2)
 	{
-		if (argv[1] == "test"sv)
+		if (argv[1] == "hub"sv)
 		{
+			ServerMode = kHub;
+		}
+		else if (argv[1] == "store"sv)
+		{
+			ServerMode = kStore;
+		}
+		else if (argv[1] == "test"sv)
+		{
+			ServerMode = kTest;
+		}
+	}
+
+	switch (ServerMode)
+	{
+		case kTest:
 #if ZEN_WITH_TESTS
 			return test_main(argc, argv);
 #else
 			fprintf(stderr, "test option not available in release mode!\n");
 			exit(5);
 #endif
-		}
+			break;
+		case kHub:
+			return AppMain<ZenHubServerMain>(argc, argv);
+		default:
+		case kStore:
+			return AppMain<ZenStorageServerMain>(argc, argv);
 	}
-
-	signal(SIGINT, utils::SignalCallbackHandler);
-	signal(SIGTERM, utils::SignalCallbackHandler);
-
-#if ZEN_PLATFORM_LINUX
-	IgnoreChildSignals();
-#endif
-
-	return AppMain<ZenStorageServerMain>(argc, argv);
 }
diff --git a/src/zenserver/storage/admin/admin.cpp b/src/zenserver/storage/admin/admin.cpp
index 04f43d33a..19155e02b 100644
--- a/src/zenserver/storage/admin/admin.cpp
+++ b/src/zenserver/storage/admin/admin.cpp
@@ -121,7 +121,10 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 		},
 		HttpVerb::kGet);
 
-	m_Router.AddPattern("jobid", "([[:digit:]]+?)");
+	static constexpr AsciiSet ValidNumberCharactersSet{"0123456789"};
+
+	m_Router.AddMatcher("jobid",
+						[](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidNumberCharactersSet); });
 
 	m_Router.RegisterRoute(
 		"jobs",
@@ -539,7 +542,7 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 			const HttpServerRequest::QueryParams Params	 = HttpReq.GetQueryParams();
 
 			GcScheduler::TriggerScrubParams ScrubParams;
-			ScrubParams.MaxTimeslice = std::chrono::seconds(100);
+			ScrubParams.MaxTimeslice = std::chrono::seconds(300);
 
 			if (auto Param = Params.GetValue("skipdelete"); Param.empty() == false)
 			{
@@ -556,6 +559,14 @@ HttpAdminService::HttpAdminService(GcScheduler&			   Scheduler,
 				ScrubParams.SkipCas = (Param == "true"sv);
 			}
 
+			if (auto Param = Params.GetValue("maxtimeslice"); Param.empty() == false)
+			{
+				if (auto Value = ParseInt<uint64_t>(Param))
+				{
+					ScrubParams.MaxTimeslice = std::chrono::seconds(Value.value());
+				}
+			}
+
 			m_GcScheduler.TriggerScrub(ScrubParams);
 
 			CbObjectWriter Response;
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp
index 18fae7027..f5ba30616 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp
@@ -48,10 +48,20 @@ HttpBuildStoreService::Initialize()
 {
 	ZEN_LOG_INFO(LogBuilds, "Initializing Builds Service");
 
-	m_Router.AddPattern("namespace", "([[:alnum:]\\-_.]+)");
-	m_Router.AddPattern("bucket", "([[:alnum:]\\-_.]+)");
-	m_Router.AddPattern("buildid", "([[:xdigit:]]{24})");
-	m_Router.AddPattern("hash", "([[:xdigit:]]{40})");
+	static constexpr AsciiSet ValidNamespaceCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
+	static constexpr AsciiSet ValidBucketCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
+	static constexpr AsciiSet ValidHexCharactersSet{"0123456789abcdefABCDEF"};
+
+	m_Router.AddMatcher("namespace",
+						[](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidNamespaceCharactersSet); });
+	m_Router.AddMatcher("bucket",
+						[](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidBucketCharactersSet); });
+	m_Router.AddMatcher("buildid", [](std::string_view Str) -> bool {
+		return Str.length() == Oid::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet);
+	});
+	m_Router.AddMatcher("hash", [](std::string_view Str) -> bool {
+		return Str.length() == IoHash::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet);
+	});
 
 	m_Router.RegisterRoute(
 		"{namespace}/{bucket}/{buildid}/blobs/{hash}",
diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp
index d8ad40621..052c3d630 100644
--- a/src/zenserver/storage/objectstore/objectstore.cpp
+++ b/src/zenserver/storage/objectstore/objectstore.cpp
@@ -271,8 +271,13 @@ HttpObjectStoreService::Inititalize()
 		CreateDirectories(BucketsPath);
 	}
 
-	m_Router.AddPattern("path", "([[:alnum:]/_.,;$~\\{\\}\\+\\-\\[\\]\\%\\(\\)]+)");
-	m_Router.AddPattern("bucket", "([[:alnum:]\\-_.]+)");
+	static constexpr AsciiSet ValidPathCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789/_.,;$~{}+-[]%()]ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
+	static constexpr AsciiSet ValidBucketCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
+
+	m_Router.AddMatcher("path",
+						[](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidPathCharactersSet); });
+	m_Router.AddMatcher("bucket",
+						[](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidBucketCharactersSet); });
 
 	m_Router.RegisterRoute(
 		"bucket",
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp
index 91c0a8af1..e3dd52ca7 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp
@@ -565,11 +565,23 @@ HttpProjectService::HttpProjectService(CidStore&					Store,
 
 	using namespace std::literals;
 
-	m_Router.AddPattern("project", "([[:alnum:]_.]+)");
-	m_Router.AddPattern("log", "([[:alnum:]_.]+)");
-	m_Router.AddPattern("op", "([[:digit:]]+?)");
-	m_Router.AddPattern("chunk", "([[:xdigit:]]{24})");
-	m_Router.AddPattern("hash", "([[:xdigit:]]{40})");
+	static constexpr AsciiSet ValidProjectCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
+	static constexpr AsciiSet ValidOplogCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
+	static constexpr AsciiSet ValidNumberCharactersSet{"0123456789"};
+	static constexpr AsciiSet ValidHexCharactersSet{"0123456789abcdefABCDEF"};
+
+	m_Router.AddMatcher("project",
+						[](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidProjectCharactersSet); });
+	m_Router.AddMatcher("log",
+						[](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidOplogCharactersSet); });
+	m_Router.AddMatcher("op",
+						[](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidNumberCharactersSet); });
+	m_Router.AddMatcher("chunk", [](std::string_view Str) -> bool {
+		return Str.length() == Oid::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet);
+	});
+	m_Router.AddMatcher("hash", [](std::string_view Str) -> bool {
+		return Str.length() == IoHash::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet);
+	});
 
 	m_Router.RegisterRoute(
 		"",
@@ -2900,6 +2912,8 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
 				};
 				tsl::robin_map<IoHash, AddedChunk, IoHash::Hasher> AddedChunks;
 
+				const std::filesystem::path CanonicalRoot = std::filesystem::canonical(Project->RootDir);
+
 				Oplog->IterateOplog(
 					[&](CbObjectView Op) {
 						bool OpRewritten = false;
@@ -2918,10 +2932,36 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
 
 								if (DataHash == IoHash::Zero)
 								{
-									std::string_view	  ServerPath = View["serverpath"sv].AsString();
-									std::filesystem::path FilePath	 = Project->RootDir / ServerPath;
-									BasicFile			  DataFile;
-									std::error_code		  Ec;
+									std::string_view ServerPath = View["serverpath"sv].AsString();
+									if (CanonicalRoot.empty())
+									{
+										ZEN_WARN("Attempting to load file '{}' from project with unset project root", ServerPath);
+										AllOk = false;
+										continue;
+									}
+
+									std::error_code				Ec;
+									const std::filesystem::path FilePath = std::filesystem::canonical(Project->RootDir / ServerPath, Ec);
+
+									if (Ec)
+									{
+										ZEN_WARN("Failed to find file '{}' in project root '{}' for 'snapshot'. Reason: '{}'",
+												 ServerPath,
+												 Project->RootDir,
+												 Ec.message());
+										AllOk = false;
+										continue;
+									}
+
+									if (std::mismatch(CanonicalRoot.begin(), CanonicalRoot.end(), FilePath.begin()).first !=
+										CanonicalRoot.end())
+									{
+										ZEN_WARN("Unable to read file '{}' outside of project root '{}'", FilePath, CanonicalRoot);
+										AllOk = false;
+										continue;
+									}
+
+									BasicFile DataFile;
 									DataFile.Open(FilePath, BasicFile::Mode::kRead, Ec);
 
 									if (Ec)
diff --git a/src/zenserver/storage/workspaces/httpworkspaces.cpp b/src/zenserver/storage/workspaces/httpworkspaces.cpp
index 3fea46b2f..dc4cc7e69 100644
--- a/src/zenserver/storage/workspaces/httpworkspaces.cpp
+++ b/src/zenserver/storage/workspaces/httpworkspaces.cpp
@@ -169,10 +169,20 @@ HttpWorkspacesService::Initialize()
 
 	ZEN_LOG_INFO(LogFs, "Initializing Workspaces Service");
 
-	m_Router.AddPattern("workspace_id", "([[:xdigit:]]{24})");
-	m_Router.AddPattern("share_id", "([[:xdigit:]]{24})");
-	m_Router.AddPattern("chunk", "([[:xdigit:]]{24})");
-	m_Router.AddPattern("share_alias", "([[:alnum:]_.\\+\\-\\[\\]]+)");
+	static constexpr AsciiSet ValidHexCharactersSet{"0123456789abcdefABCDEF"};
+
+	m_Router.AddMatcher("workspace_id", [](std::string_view Str) -> bool {
+		return Str.length() == Oid::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet);
+	});
+	m_Router.AddMatcher("share_id", [](std::string_view Str) -> bool {
+		return Str.length() == Oid::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet);
+	});
+	m_Router.AddMatcher("chunk", [](std::string_view Str) -> bool {
+		return Str.length() == Oid::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet);
+	});
+	m_Router.AddMatcher("share_alias", [](std::string_view Str) -> bool {
+		return !Str.empty() && AsciiSet::HasOnly(Str, Workspaces::ValidAliasCharactersSet);
+	});
 
 	m_Router.RegisterRoute(
 		"{workspace_id}/{share_id}/files",
diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp
index cf4936f6f..edce75d2a 100644
--- a/src/zenserver/storage/zenstorageserver.cpp
+++ b/src/zenserver/storage/zenstorageserver.cpp
@@ -17,6 +17,7 @@
 #include <zencore/sentryintegration.h>
 #include <zencore/session.h>
 #include <zencore/string.h>
+#include <zencore/system.h>
 #include <zencore/thread.h>
 #include <zencore/timer.h>
 #include <zencore/trace.h>
@@ -683,8 +684,8 @@ ZenStorageServer::Run()
 			"        \\/   \\/     \\/          \\/                         \\/ \n");
 
 		ExtendableStringBuilder<256> BuildOptions;
-		GetBuildOptions(BuildOptions);
-		ZEN_INFO("Build options: {}", BuildOptions);
+		GetBuildOptions(BuildOptions, '\n');
+		ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions);
 	}
 
 	ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId());
@@ -904,6 +905,16 @@ ZenStorageServerMain::ZenStorageServerMain(ZenStorageServerConfig& ServerOptions
 {
 }
 
+ZenStorageServerMain::~ZenStorageServerMain()
+{
+}
+
+void
+ZenStorageServerMain::InitializeLogging()
+{
+	InitializeServerLogging(m_ServerOptions, /* WithCacheService */ true);
+}
+
 void
 ZenStorageServerMain::DoRun(ZenServerState::ZenServerEntry* Entry)
 {
diff --git a/src/zenserver/storage/zenstorageserver.h b/src/zenserver/storage/zenstorageserver.h
index f79c55bc8..5ccb587d6 100644
--- a/src/zenserver/storage/zenstorageserver.h
+++ b/src/zenserver/storage/zenstorageserver.h
@@ -103,7 +103,10 @@ class ZenStorageServerMain : public ZenServerMain
 {
 public:
 	ZenStorageServerMain(ZenStorageServerConfig& ServerOptions);
+	~ZenStorageServerMain();
+
 	virtual void DoRun(ZenServerState::ZenServerEntry* Entry) override;
+	virtual void InitializeLogging() override;
 
 	ZenStorageServerMain(const ZenStorageServerMain&) = delete;
 	ZenStorageServerMain& operator=(const ZenStorageServerMain&) = delete;
diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua
index fb65fa949..6ee80dc62 100644
--- a/src/zenserver/xmake.lua
+++ b/src/zenserver/xmake.lua
@@ -22,6 +22,7 @@ target("zenserver")
     add_deps("sol2")
     add_packages("json11")
     add_packages("lua")
+    add_packages("consul")
 
     if has_config("zenmimalloc") then
         add_packages("mimalloc")
@@ -55,10 +56,7 @@ target("zenserver")
         add_ldflags("-framework Security")
         add_ldflags("-framework SystemConfiguration")
     end
-
-    add_options("compute")
-    add_options("exec")
-    
+   
     -- to work around some unfortunate Ctrl-C behaviour on Linux/Mac due to 
     -- our use of setsid() at startup we pass in `--no-detach` to zenserver
     -- ensure that it recieves signals when the user requests termination
@@ -87,17 +85,60 @@ target("zenserver")
         end
     end)
 
+
     after_build(function (target)
-        if has_config("zensentry") then
-            local crashpad_handler = "crashpad_handler"
-            if is_plat("windows") then
-                crashpad_handler = "crashpad_handler.exe"
+        local function copy_if_newer(src_file, dst_file, file_description)
+            if not os.exists(src_file) then
+                print("Source file '" .. src_file .. "' does not exist, cannot copy " .. file_description)
+                return
             end
+            local should_copy = false
+            if not os.exists(dst_file) then
+                should_copy = true
+            else
+                local src_size = os.filesize(src_file)
+                local dst_size = os.filesize(dst_file)
+                local src_mtime = os.mtime(src_file)
+                local dst_mtime = os.mtime(dst_file)
+                
+                if src_size ~= dst_size or src_mtime > dst_mtime then
+                    should_copy = true
+                end
+            end
+            
+            if should_copy then
+                os.cp(src_file, dst_file)
+                print("Copied '" .. file_description .. "' to output directory")
+            end
+        end
+
+        if has_config("zensentry") then
             local pkg = target:pkg("sentry-native")
             if pkg then
                 local installdir = pkg:installdir()
-                os.cp(path.join(installdir, "bin/" .. crashpad_handler), target:targetdir())
-                print("Copied " .. crashpad_handler .. " to output directory")
+
+                local crashpad_handler = "crashpad_handler"
+                if is_plat("windows") then
+                    crashpad_handler = "crashpad_handler.exe"
+                end
+
+                local crashpad_handler_path = path.join(installdir, "bin/" .. crashpad_handler)
+                copy_if_newer(crashpad_handler_path, path.join(target:targetdir(), crashpad_handler), crashpad_handler)
+
+                if is_plat("windows") then
+                    local crashpad_wer_path = path.join(installdir, "bin/crashpad_wer.dll")
+                    copy_if_newer(crashpad_wer_path, path.join(target:targetdir(), "crashpad_wer.dll"), "crashpad_wer.dll")
+                end
+            end
+        end
+
+        local consul_pkg = target:pkg("consul")
+        if consul_pkg then
+            local installdir = consul_pkg:installdir()
+            local consul_bin = "consul"
+            if is_plat("windows") then
+                consul_bin = "consul.exe"
             end
+            copy_if_newer(path.join(installdir, "bin", consul_bin), path.join(target:targetdir(), consul_bin), consul_bin)
         end
     end)
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 08be5475a..04f60c73e 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -123,7 +123,9 @@ ZenServerBase::Initialize(const ZenServerConfig& ServerOptions, ZenServerState::
 
 	if (m_ServerMutex.Create(MutexName) == false)
 	{
-		ThrowLastError(fmt::format("Failed to create mutex '{}'", MutexName).c_str());
+		std::error_code Ec = MakeErrorCodeFromLastError();
+		ZEN_WARN("Failed to create server mutex '{}'. Reason: '{}' ({})", MutexName, Ec.message(), Ec.value());
+		return -1;
 	}
 
 	EnqueueSigIntTimer();
@@ -150,6 +152,13 @@ ZenServerBase::Initialize(const ZenServerConfig& ServerOptions, ZenServerState::
 		EnqueueStatsReportingTimer();
 	}
 
+	m_HealthService.SetHealthInfo({.DataRoot		= ServerOptions.DataDir,
+								   .AbsLogPath		= ServerOptions.AbsLogFile,
+								   .HttpServerClass = std::string(ServerOptions.HttpConfig.ServerClass),
+								   .BuildVersion	= std::string(ZEN_CFG_VERSION_BUILD_STRING_FULL)});
+
+	LogSettingsSummary(ServerOptions);
+
 	return EffectiveBasePort;
 }
 
@@ -162,24 +171,24 @@ ZenServerBase::Finalize()
 }
 
 void
-ZenServerBase::GetBuildOptions(StringBuilderBase& OutOptions)
+ZenServerBase::GetBuildOptions(StringBuilderBase& OutOptions, char Separator) const
 {
 	ZEN_MEMSCOPE(GetZenserverTag());
 
 	OutOptions << "ZEN_ADDRESS_SANITIZER=" << (ZEN_ADDRESS_SANITIZER ? "1" : "0");
-	OutOptions << ",";
+	OutOptions << Separator;
 	OutOptions << "ZEN_USE_SENTRY=" << (ZEN_USE_SENTRY ? "1" : "0");
-	OutOptions << ",";
+	OutOptions << Separator;
 	OutOptions << "ZEN_WITH_TESTS=" << (ZEN_WITH_TESTS ? "1" : "0");
-	OutOptions << ",";
+	OutOptions << Separator;
 	OutOptions << "ZEN_USE_MIMALLOC=" << (ZEN_USE_MIMALLOC ? "1" : "0");
-	OutOptions << ",";
+	OutOptions << Separator;
 	OutOptions << "ZEN_USE_RPMALLOC=" << (ZEN_USE_RPMALLOC ? "1" : "0");
-	OutOptions << ",";
+	OutOptions << Separator;
 	OutOptions << "ZEN_WITH_HTTPSYS=" << (ZEN_WITH_HTTPSYS ? "1" : "0");
-	OutOptions << ",";
+	OutOptions << Separator;
 	OutOptions << "ZEN_WITH_MEMTRACK=" << (ZEN_WITH_MEMTRACK ? "1" : "0");
-	OutOptions << ",";
+	OutOptions << Separator;
 	OutOptions << "ZEN_WITH_TRACE=" << (ZEN_WITH_TRACE ? "1" : "0");
 }
 
@@ -373,6 +382,57 @@ ZenServerBase::HandleStatusRequest(HttpServerRequest& Request)
 	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
 }
 
+void
+ZenServerBase::LogSettingsSummary(const ZenServerConfig& ServerConfig)
+{
+	// clang-format off
+	std::list<std::pair<std::string_view, std::string>> Settings = {
+		{"DataDir"sv, 				ServerConfig.DataDir.string()},
+		{"AbsLogFile"sv,			ServerConfig.AbsLogFile.string()},
+		{"SystemRootDir"sv, 		ServerConfig.SystemRootDir.string()},
+		{"ContentDir"sv, 			ServerConfig.ContentDir.string()},
+		{"BasePort"sv, 				fmt::to_string(ServerConfig.BasePort)},
+		{"IsDebug"sv, 				fmt::to_string(ServerConfig.IsDebug)},
+		{"IsCleanStart"sv, 			fmt::to_string(ServerConfig.IsCleanStart)},
+		{"IsPowerCycle"sv, 			fmt::to_string(ServerConfig.IsPowerCycle)},
+		{"IsTest"sv, 				fmt::to_string(ServerConfig.IsTest)},
+		{"Detach"sv, 				fmt::to_string(ServerConfig.Detach)},
+		{"NoConsoleOutput"sv, 		fmt::to_string(ServerConfig.NoConsoleOutput)},
+		{"QuietConsole"sv, 			fmt::to_string(ServerConfig.QuietConsole)},
+		{"CoreLimit"sv, 			fmt::to_string(ServerConfig.CoreLimit)},
+		{"IsDedicated"sv, 			fmt::to_string(ServerConfig.IsDedicated)},
+		{"ShouldCrash"sv, 			fmt::to_string(ServerConfig.ShouldCrash)},
+		{"ChildId"sv,				ServerConfig.ChildId},
+		{"LogId"sv,					ServerConfig.LogId},
+		{"Sentry DSN"sv,			ServerConfig.SentryConfig.Dsn.empty() ? "not set" : ServerConfig.SentryConfig.Dsn},
+		{"Sentry Environment"sv,	ServerConfig.SentryConfig.Environment},
+		{"Statsd Enabled"sv,		fmt::to_string(ServerConfig.StatsConfig.Enabled)},
+	};
+	// clang-format on
+
+	if (ServerConfig.StatsConfig.Enabled)
+	{
+		Settings.emplace_back("Statsd Host", ServerConfig.StatsConfig.StatsdHost);
+		Settings.emplace_back("Statsd Port", fmt::to_string(ServerConfig.StatsConfig.StatsdPort));
+	}
+
+	size_t MaxWidth = 0;
+	for (const auto& Setting : Settings)
+	{
+		MaxWidth = std::max(MaxWidth, Setting.first.size());
+	}
+
+	ZEN_INFO("Server settings summary:");
+
+	for (const auto& Setting : Settings)
+	{
+		if (!Setting.second.empty())
+		{
+			ZEN_INFO("  {:<{}} : {}", Setting.first, MaxWidth, Setting.second);
+		}
+	}
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 ZenServerMain::ZenServerMain(ZenServerConfig& ServerOptions) : m_ServerOptions(ServerOptions)
@@ -383,6 +443,12 @@ ZenServerMain::~ZenServerMain()
 {
 }
 
+void
+ZenServerMain::InitializeLogging()
+{
+	InitializeServerLogging(m_ServerOptions, /* WithCacheService */ false);
+}
+
 int
 ZenServerMain::Run()
 {
@@ -510,7 +576,7 @@ ZenServerMain::Run()
 			}
 		}
 
-		InitializeServerLogging(m_ServerOptions, /* WithCacheService */ true);
+		InitializeLogging();
 
 		ZEN_INFO("Command line: {}", m_ServerOptions.CommandLine);
 
diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h
index 4de865a5f..ab7122fcc 100644
--- a/src/zenserver/zenserver.h
+++ b/src/zenserver/zenserver.h
@@ -10,6 +10,7 @@
 
 #include <memory>
 #include <string_view>
+#include "config/config.h"
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <asio.hpp>
@@ -45,7 +46,8 @@ public:
 protected:
 	int	 Initialize(const ZenServerConfig& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry);
 	void Finalize();
-	void GetBuildOptions(StringBuilderBase& OutOptions);
+	void GetBuildOptions(StringBuilderBase& OutOptions, char Separator = ',') const;
+	void LogSettingsSummary(const ZenServerConfig& ServerConfig);
 
 protected:
 	NamedMutex						m_ServerMutex;
@@ -122,6 +124,7 @@ protected:
 	ZenServerConfig& m_ServerOptions;
 	LockFile		 m_LockFile;
 
+	virtual void InitializeLogging();
 	virtual void DoRun(ZenServerState::ZenServerEntry* Entry) = 0;
 
 	void	 NotifyReady();