Merge branch 'main' of https://github.com/EpicGames/zen

11 files changed, 310 insertions, 45 deletions

diff --git a/zencore/include/zencore/stats.h b/zencore/include/zencore/stats.h
index dfa8dac34..2e567d614 100644
--- a/zencore/include/zencore/stats.h
+++ b/zencore/include/zencore/stats.h
@@ -209,6 +209,8 @@ public:
 		Scope(OperationTiming& Outer);
 		~Scope();
 
+		void Cancel();
+
 	private:
 		OperationTiming& m_Outer;
 		uint64_t		 m_StartTick;
diff --git a/zencore/stats.cpp b/zencore/stats.cpp
index 34dc2828f..47efba76b 100644
--- a/zencore/stats.cpp
+++ b/zencore/stats.cpp
@@ -398,7 +398,16 @@ OperationTiming::Scope::Scope(OperationTiming& Outer) : m_Outer(Outer), m_StartT
 
 OperationTiming::Scope::~Scope()
 {
-	m_Outer.Update(GetHifreqTimerValue() - m_StartTick);
+	if (m_StartTick != 0)
+	{
+		m_Outer.Update(GetHifreqTimerValue() - m_StartTick);
+	}
+}
+
+void
+OperationTiming::Scope::Cancel()
+{
+	m_StartTick = 0;
 }
 
 //////////////////////////////////////////////////////////////////////////
diff --git a/zenhttp/httpserver.cpp b/zenhttp/httpserver.cpp
index cfd1463ba..193426ed2 100644
--- a/zenhttp/httpserver.cpp
+++ b/zenhttp/httpserver.cpp
@@ -67,13 +67,16 @@ MapContentTypeToString(HttpContentType ContentType)
 //////////////////////////////////////////////////////////////////////////
 
 static constinit uint32_t HashBinary					= HashStringDjb2("application/octet-stream"sv);
-static constinit uint32_t HashJson						= HashStringDjb2("application/json"sv);
-static constinit uint32_t HashYaml						= HashStringDjb2("text/yaml"sv);
+static constinit uint32_t HashApplicationJson			= HashStringDjb2("application/json"sv);
+static constinit uint32_t HashApplicationYaml			= HashStringDjb2("text/yaml"sv);
 static constinit uint32_t HashText						= HashStringDjb2("text/plain"sv);
 static constinit uint32_t HashCompactBinary				= HashStringDjb2("application/x-ue-cb"sv);
 static constinit uint32_t HashCompactBinaryPackage		= HashStringDjb2("application/x-ue-cbpkg"sv);
 static constinit uint32_t HashCompactBinaryPackageOffer = HashStringDjb2("application/x-ue-offer"sv);
 static constinit uint32_t HashCompressedBinary			= HashStringDjb2("application/x-ue-comp"sv);
+static constinit uint32_t HashJson						= HashStringDjb2("json"sv);
+static constinit uint32_t HashYaml						= HashStringDjb2("yaml"sv);
+static constinit uint32_t HashHtml						= HashStringDjb2("text/html"sv);
 
 std::once_flag InitContentTypeLookup;
 
@@ -81,14 +84,21 @@ struct HashedTypeEntry
 {
 	uint32_t		Hash;
 	HttpContentType Type;
-} TypeHashTable[] = {{HashBinary, HttpContentType::kBinary},
-					 {HashCompactBinary, HttpContentType::kCbObject},
-					 {HashCompactBinaryPackage, HttpContentType::kCbPackage},
-					 {HashCompactBinaryPackageOffer, HttpContentType::kCbPackageOffer},
-					 {HashJson, HttpContentType::kJSON},
-					 {HashYaml, HttpContentType::kYAML},
-					 {HashText, HttpContentType::kText},
-					 {HashCompressedBinary, HttpContentType::kCompressedBinary}};
+} TypeHashTable[] = {
+	// clang-format off
+	{HashBinary,					HttpContentType::kBinary},
+	{HashCompactBinary,				HttpContentType::kCbObject},
+	{HashCompactBinaryPackage,		HttpContentType::kCbPackage},
+	{HashCompactBinaryPackageOffer, HttpContentType::kCbPackageOffer},
+	{HashJson,						HttpContentType::kJSON},
+	{HashApplicationJson,			HttpContentType::kJSON},
+	{HashYaml,						HttpContentType::kYAML},
+	{HashApplicationYaml,			HttpContentType::kYAML},
+	{HashText,						HttpContentType::kText},
+	{HashCompressedBinary,			HttpContentType::kCompressedBinary},
+	{HashHtml,						HttpContentType::kHTML},
+	// clang-format on
+};
 
 HttpContentType
 ParseContentTypeImpl(const std::string_view& ContentTypeString)
@@ -120,6 +130,16 @@ ParseContentTypeInit(const std::string_view& ContentTypeString)
 		std::sort(std::begin(TypeHashTable), std::end(TypeHashTable), [](const HashedTypeEntry& Lhs, const HashedTypeEntry& Rhs) {
 			return Lhs.Hash < Rhs.Hash;
 		});
+
+		// validate that there are no hash collisions
+
+		uint32_t LastHash = 0;
+
+		for (const auto& Item : TypeHashTable)
+		{
+			ZEN_ASSERT(LastHash != Item.Hash);
+			LastHash = Item.Hash;
+		}
 	});
 
 	ParseContentType = ParseContentTypeImpl;
diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp
index 9b2e7f832..f4d3e45fe 100644
--- a/zenhttp/httpsys.cpp
+++ b/zenhttp/httpsys.cpp
@@ -1086,6 +1086,8 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService&
 	const int PrefixLength	= Service.UriPrefixLength();
 	const int AbsPathLength = HttpRequestPtr->CookedUrl.AbsPathLength / sizeof(char16_t);
 
+	HttpContentType AcceptContentType = HttpContentType::kUnknownContentType;
+
 	if (AbsPathLength >= PrefixLength)
 	{
 		// We convert the URI immediately because most of the code involved prefers to deal
@@ -1094,15 +1096,33 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService&
 
 		WideToUtf8({(char16_t*)HttpRequestPtr->CookedUrl.pAbsPath + PrefixLength, gsl::narrow<size_t>(AbsPathLength - PrefixLength)},
 				   m_UriUtf8);
+
+		std::string_view Uri8{m_UriUtf8};
+
+		const size_t LastComponentIndex = Uri8.find_last_of('/');
+
+		if (LastComponentIndex != std::string_view::npos)
+		{
+			Uri8.remove_prefix(LastComponentIndex);
+		}
+
+		const size_t LastDotIndex = Uri8.find_last_of('.');
+
+		if (LastDotIndex != std::string_view::npos)
+		{
+			Uri8.remove_prefix(LastDotIndex + 1);
+		}
+
+		AcceptContentType = ParseContentType(Uri8);
 	}
 	else
 	{
 		m_UriUtf8.Reset();
 	}
 
-	if (auto QueryStringLength = HttpRequestPtr->CookedUrl.QueryStringLength)
+	if (uint16_t QueryStringLength = HttpRequestPtr->CookedUrl.QueryStringLength)
 	{
-		--QueryStringLength;
+		--QueryStringLength;  // We skip the leading question mark
 
 		WideToUtf8({(char16_t*)(HttpRequestPtr->CookedUrl.pQueryString) + 1, QueryStringLength / sizeof(char16_t)}, m_QueryStringUtf8);
 	}
@@ -1114,7 +1134,18 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService&
 	m_Verb			= TranslateHttpVerb(HttpRequestPtr->Verb);
 	m_ContentLength = GetContentLength(HttpRequestPtr);
 	m_ContentType	= GetContentType(HttpRequestPtr);
-	m_AcceptType	= GetAcceptType(HttpRequestPtr);
+
+	// It an explicit content type extension was specified then we'll use that over any
+	// Accept: header value that may be present
+
+	if (AcceptContentType != HttpContentType::kUnknownContentType)
+	{
+		m_AcceptType = AcceptContentType;
+	}
+	else
+	{
+		m_AcceptType = GetAcceptType(HttpRequestPtr);
+	}
 }
 
 Oid
diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp
index 0e5e73ffc..fe21aa834 100644
--- a/zenserver-test/zenserver-test.cpp
+++ b/zenserver-test/zenserver-test.cpp
@@ -1448,16 +1448,21 @@ TEST_CASE("zcache.policy")
 		return Buf;
 	};
 
-	auto GeneratePackage = [](zen::IoHash& OutAttachmentKey) -> zen::CbPackage {
+	auto GeneratePackage = [](zen::IoHash& OutRecordKey, zen::IoHash& OutAttachmentKey) -> zen::CbPackage {
 		auto Data			= zen::SharedBuffer::Clone(zen::MakeMemoryView<uint8_t>({1, 2, 3, 4, 5, 6, 7, 8, 9}));
 		auto CompressedData = zen::CompressedBuffer::Compress(Data);
 		OutAttachmentKey	= zen::IoHash::FromBLAKE3(CompressedData.GetRawHash());
-		zen::CbWriter Obj;
-		Obj.BeginObject("obj"sv);
-		Obj.AddBinaryAttachment("data", OutAttachmentKey);
-		Obj.EndObject();
+
+		zen::CbWriter Writer;
+		Writer.BeginObject("obj"sv);
+		Writer.AddBinaryAttachment("data", OutAttachmentKey);
+		Writer.EndObject();
+		CbObject CacheRecord = Writer.Save().AsObject();
+
+		OutRecordKey = IoHash::HashBuffer(CacheRecord.GetBuffer().GetView());
+
 		zen::CbPackage Package;
-		Package.SetObject(Obj.Save().AsObject());
+		Package.SetObject(CacheRecord);
 		Package.AddAttachment(zen::CbAttachment(CompressedData));
 
 		return Package;
@@ -1587,7 +1592,8 @@ TEST_CASE("zcache.policy")
 		LocalCfg.Spawn(LocalInst);
 
 		zen::IoHash	   Key;
-		zen::CbPackage Package = GeneratePackage(Key);
+		zen::IoHash	   PayloadId;
+		zen::CbPackage Package = GeneratePackage(Key, PayloadId);
 		auto		   Buf	   = ToBuffer(Package);
 
 		// Store package upstream
@@ -1623,7 +1629,8 @@ TEST_CASE("zcache.policy")
 		LocalCfg.Spawn(LocalInst);
 
 		zen::IoHash	   Key;
-		zen::CbPackage Package = GeneratePackage(Key);
+		zen::IoHash	   PayloadId;
+		zen::CbPackage Package = GeneratePackage(Key, PayloadId);
 		auto		   Buf	   = ToBuffer(Package);
 
 		// Store packge locally
@@ -1659,7 +1666,8 @@ TEST_CASE("zcache.policy")
 		LocalCfg.Spawn(LocalInst);
 
 		zen::IoHash	   Key;
-		zen::CbPackage Package = GeneratePackage(Key);
+		zen::IoHash	   PayloadId;
+		zen::CbPackage Package = GeneratePackage(Key, PayloadId);
 		auto		   Buf	   = ToBuffer(Package);
 
 		// Store package locally and upstream
@@ -1692,7 +1700,8 @@ TEST_CASE("zcache.policy")
 		LocalCfg.Spawn(LocalInst);
 
 		zen::IoHash	   Key;
-		zen::CbPackage Package = GeneratePackage(Key);
+		zen::IoHash	   PayloadId;
+		zen::CbPackage Package = GeneratePackage(Key, PayloadId);
 		auto		   Buf	   = ToBuffer(Package);
 
 		// Store package locally
@@ -1748,7 +1757,8 @@ TEST_CASE("zcache.policy")
 		LocalCfg.Spawn(LocalInst);
 
 		zen::IoHash	   Key;
-		zen::CbPackage Package = GeneratePackage(Key);
+		zen::IoHash	   PayloadId;
+		zen::CbPackage Package = GeneratePackage(Key, PayloadId);
 		auto		   Buf	   = ToBuffer(Package);
 
 		// Store package upstream
@@ -1791,6 +1801,80 @@ TEST_CASE("zcache.policy")
 			CHECK(Package.GetAttachments().size() != 0);
 		}
 	}
+
+	SUBCASE("skip - 'data' returns empty cache record/payload")
+	{
+		ZenConfig		  Cfg = ZenConfig::New();
+		ZenServerInstance Instance(TestEnv);
+		const auto		  Bucket = "test"sv;
+
+		Cfg.Spawn(Instance);
+
+		zen::IoHash	   Key;
+		zen::IoHash	   PayloadId;
+		zen::CbPackage Package = GeneratePackage(Key, PayloadId);
+		auto		   Buf	   = ToBuffer(Package);
+
+		// Store package
+		{
+			cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}"_format(Cfg.BaseUri, Bucket, Key)},
+											cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()},
+											cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}});
+			CHECK(Result.status_code == 201);
+		}
+
+		// Get package
+		{
+			cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key)},
+											cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
+			CHECK(Result.status_code == 200);
+			CHECK(Result.text.size() == 0);
+		}
+
+		// Get record
+		{
+			cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key)},
+											cpr::Header{{"Accept", "application/x-ue-cbobject"}});
+			CHECK(Result.status_code == 200);
+			CHECK(Result.text.size() == 0);
+		}
+
+		// Get payload
+		{
+			cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key, PayloadId)},
+											cpr::Header{{"Accept", "application/x-ue-comp"}});
+			CHECK(Result.status_code == 200);
+			CHECK(Result.text.size() == 0);
+		}
+	}
+
+	SUBCASE("skip - 'data' returns empty binary value")
+	{
+		ZenConfig		  Cfg = ZenConfig::New();
+		ZenServerInstance Instance(TestEnv);
+		const auto		  Bucket = "test"sv;
+
+		Cfg.Spawn(Instance);
+
+		zen::IoHash Key;
+		auto		BinaryValue = GenerateData(1024, Key);
+
+		// Store binary cache value
+		{
+			cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}"_format(Cfg.BaseUri, Bucket, Key)},
+											cpr::Body{(const char*)BinaryValue.GetData(), BinaryValue.GetSize()},
+											cpr::Header{{"Content-Type", "application/octet-stream"}});
+			CHECK(Result.status_code == 201);
+		}
+
+		// Get package
+		{
+			cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key)},
+											cpr::Header{{"Accept", "application/octet-stream"}});
+			CHECK(Result.status_code == 200);
+			CHECK(Result.text.size() == 0);
+		}
+	}
 }
 
 struct RemoteExecutionRequest
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index dc22a15d1..72fc7f3af 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -200,8 +200,10 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
 	{
 		std::string_view Key = Request.RelativeUri();
 
-		if (Key.empty())
+		if (Key.empty() || Key == "stats.json")
 		{
+			$.Cancel();
+
 			return HandleStatusRequest(Request);
 		}
 
@@ -418,10 +420,17 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 		if (ValidationResult != CbValidateError::None)
 		{
 			ZEN_WARN("GET - '{}/{}' '{}' FAILED, invalid compact binary object", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType));
+			m_CacheStats.MissCount++;
 			return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Invalid cache record"sv);
 		}
 
-		const bool SkipAttachments = zen::CachePolicy::SkipAttachments == (Policy & zen::CachePolicy::SkipAttachments);
+		if ((Policy & CachePolicy::SkipData) == CachePolicy::SkipData)
+		{
+			m_CacheStats.HitCount++;
+			return Request.WriteResponse(HttpResponseCode::OK);
+		}
+
+		const bool SkipAttachments = (Policy & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments;
 		uint32_t   AttachmentCount = 0;
 		uint32_t   ValidCount	   = 0;
 		uint64_t   AttachmentBytes = 0ull;
@@ -487,7 +496,14 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 			m_CacheStats.UpstreamHitCount++;
 		}
 
-		Request.WriteResponse(HttpResponseCode::OK, Value.Value.GetContentType(), Value.Value);
+		if ((Policy & CachePolicy::SkipData) == CachePolicy::SkipData)
+		{
+			Request.WriteResponse(HttpResponseCode::OK);
+		}
+		else
+		{
+			Request.WriteResponse(HttpResponseCode::OK, Value.Value.GetContentType(), Value.Value);
+		}
 	}
 }
 
@@ -740,7 +756,14 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques
 		m_CacheStats.UpstreamHitCount++;
 	}
 
-	Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Payload);
+	if ((Policy & CachePolicy::SkipData) == CachePolicy::SkipData)
+	{
+		Request.WriteResponse(HttpResponseCode::OK);
+	}
+	else
+	{
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Payload);
+	}
 }
 
 void
diff --git a/zenserver/config.cpp b/zenserver/config.cpp
index 759534d58..7a1efe6e8 100644
--- a/zenserver/config.cpp
+++ b/zenserver/config.cpp
@@ -213,8 +213,8 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
 	options.add_option("cache",
 					   "",
 					   "upstream-zen-url",
-					   "URL to a remote Zen server instance",
-					   cxxopts::value<std::string>(ServiceConfig.UpstreamCacheConfig.ZenConfig.Url)->default_value(""),
+					   "URL to remote Zen server. Use a comma separated list to choose the one with the best latency.",
+					   cxxopts::value<std::vector<std::string>>(ServiceConfig.UpstreamCacheConfig.ZenConfig.Urls)->default_value(""),
 					   "");
 
 	options.add_option("cache",
@@ -367,9 +367,17 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv
 
 				if (auto ZenConfig = UpstreamConfig->get<sol::optional<sol::table>>("zen"))
 				{
-					UpdateStringValueFromConfig(ZenConfig.value(),
-												std::string_view("url"),
-												ServiceConfig.UpstreamCacheConfig.ZenConfig.Url);
+					if (auto Url = ZenConfig.value().get<sol::optional<std::string>>("url"))
+					{
+						ServiceConfig.UpstreamCacheConfig.ZenConfig.Urls.push_back(Url.value());
+					}
+					else if (auto Urls = ZenConfig.value().get<sol::optional<sol::table>>("url"))
+					{
+						for (const auto& Kv : Urls.value())
+						{
+							ServiceConfig.UpstreamCacheConfig.ZenConfig.Urls.push_back(Kv.second.as<std::string>());
+						}
+					}
 				}
 			}
 		}
diff --git a/zenserver/config.h b/zenserver/config.h
index af1a24455..ec6d7340a 100644
--- a/zenserver/config.h
+++ b/zenserver/config.h
@@ -35,7 +35,7 @@ struct ZenUpstreamJupiterConfig
 
 struct ZenUpstreamZenConfig
 {
-	std::string Url;
+	std::vector<std::string> Urls;
 };
 
 enum class UpstreamCachePolicy : uint8_t
diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp
index b1966e299..a183e7ebf 100644
--- a/zenserver/upstream/upstreamcache.cpp
+++ b/zenserver/upstream/upstreamcache.cpp
@@ -111,6 +111,8 @@ namespace detail {
 
 		virtual ~JupiterUpstreamEndpoint() = default;
 
+		virtual UpstreamEndpointHealth Initialize() override { return CheckHealth(); }
+
 		virtual bool IsHealthy() const override { return m_HealthOk.load(); }
 
 		virtual UpstreamEndpointHealth CheckHealth() override
@@ -400,22 +402,70 @@ namespace detail {
 
 	class ZenUpstreamEndpoint final : public UpstreamEndpoint
 	{
+		struct ZenEndpoint
+		{
+			std::string Url;
+			std::string Reason;
+			double		Latency{};
+			bool		Ok = false;
+
+			bool operator<(const ZenEndpoint& RHS) const { return Ok && RHS.Ok ? Latency < RHS.Latency : Ok; }
+		};
+
 	public:
-		ZenUpstreamEndpoint(std::string_view ServiceUrl)
+		ZenUpstreamEndpoint(std::span<std::string const> Urls) : m_Log(zen::logging::Get("upstream")), m_DisplayName("ZEN")
 		{
-			using namespace fmt::literals;
-			m_DisplayName = "Zen - {}"_format(ServiceUrl);
-			m_Client	  = new ZenStructuredCacheClient(ServiceUrl);
+			for (const auto& Url : Urls)
+			{
+				m_Endpoints.push_back({.Url = Url});
+			}
 		}
 
 		~ZenUpstreamEndpoint() = default;
 
+		virtual UpstreamEndpointHealth Initialize() override
+		{
+			using namespace fmt::literals;
+
+			const ZenEndpoint& Ep = GetEndpoint();
+			if (Ep.Ok)
+			{
+				m_ServiceUrl  = Ep.Url;
+				m_DisplayName = "ZEN - {}"_format(m_ServiceUrl);
+				m_Client	  = new ZenStructuredCacheClient(m_ServiceUrl);
+
+				m_HealthOk = true;
+				return {.Ok = true};
+			}
+
+			m_HealthOk = false;
+			return {.Reason = Ep.Reason};
+		}
+
 		virtual bool IsHealthy() const override { return m_HealthOk; }
 
 		virtual UpstreamEndpointHealth CheckHealth() override
 		{
+			using namespace fmt::literals;
+
 			try
 			{
+				if (m_Client.IsNull())
+				{
+					const ZenEndpoint& Ep = GetEndpoint();
+					if (Ep.Ok)
+					{
+						m_ServiceUrl  = Ep.Url;
+						m_DisplayName = "ZEN - {}"_format(m_ServiceUrl);
+						m_Client	  = new ZenStructuredCacheClient(m_ServiceUrl);
+
+						m_HealthOk = true;
+						return {.Ok = true};
+					}
+
+					return {.Reason = Ep.Reason};
+				}
+
 				ZenStructuredCacheSession Session(*m_Client);
 				ZenCacheResult			  Result;
 
@@ -591,6 +641,42 @@ namespace detail {
 		virtual UpstreamEndpointStats& Stats() override { return m_Stats; }
 
 	private:
+		const ZenEndpoint& GetEndpoint()
+		{
+			for (ZenEndpoint& Ep : m_Endpoints)
+			{
+				ZenStructuredCacheClient  Client(Ep.Url);
+				ZenStructuredCacheSession Session(Client);
+				const int32_t			  SampleCount = 2;
+
+				Ep.Ok	   = false;
+				Ep.Latency = {};
+
+				for (int32_t Sample = 0; Sample < SampleCount; ++Sample)
+				{
+					ZenCacheResult Result = Session.CheckHealth();
+					Ep.Ok				  = Result.Success;
+					Ep.Reason			  = std::move(Result.Reason);
+					Ep.Latency += Result.ElapsedSeconds;
+				}
+				Ep.Latency /= double(SampleCount);
+			}
+
+			std::sort(std::begin(m_Endpoints), std::end(m_Endpoints));
+
+			for (const auto& Ep : m_Endpoints)
+			{
+				ZEN_INFO("ping ZEN endpoint '{}' latency '{:.3}s' {}", Ep.Url, Ep.Latency, Ep.Ok ? "OK" : Ep.Reason);
+			}
+
+			return m_Endpoints.front();
+		}
+
+		spdlog::logger& Log() { return m_Log; }
+
+		spdlog::logger&					 m_Log;
+		std::string						 m_ServiceUrl;
+		std::vector<ZenEndpoint>		 m_Endpoints;
 		std::string						 m_DisplayName;
 		RefPtr<ZenStructuredCacheClient> m_Client;
 		UpstreamEndpointStats			 m_Stats;
@@ -711,7 +797,7 @@ public:
 	{
 		for (auto& Endpoint : m_Endpoints)
 		{
-			const UpstreamEndpointHealth Health = Endpoint->CheckHealth();
+			const UpstreamEndpointHealth Health = Endpoint->Initialize();
 			if (Health.Ok)
 			{
 				ZEN_INFO("initialize endpoint '{}' OK", Endpoint->DisplayName());
@@ -993,9 +1079,9 @@ MakeJupiterUpstreamEndpoint(const CloudCacheClientOptions& Options)
 }
 
 std::unique_ptr<UpstreamEndpoint>
-MakeZenUpstreamEndpoint(std::string_view Url)
+MakeZenUpstreamEndpoint(std::span<std::string const> Urls)
 {
-	return std::make_unique<detail::ZenUpstreamEndpoint>(Url);
+	return std::make_unique<detail::ZenUpstreamEndpoint>(Urls);
 }
 
 }  // namespace zen
diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h
index 08f379b11..edc995da6 100644
--- a/zenserver/upstream/upstreamcache.h
+++ b/zenserver/upstream/upstreamcache.h
@@ -96,6 +96,8 @@ class UpstreamEndpoint
 public:
 	virtual ~UpstreamEndpoint() = default;
 
+	virtual UpstreamEndpointHealth Initialize() = 0;
+
 	virtual bool IsHealthy() const = 0;
 
 	virtual UpstreamEndpointHealth CheckHealth() = 0;
@@ -143,6 +145,6 @@ std::unique_ptr<UpstreamCache> MakeUpstreamCache(const UpstreamCacheOptions& Opt
 
 std::unique_ptr<UpstreamEndpoint> MakeJupiterUpstreamEndpoint(const CloudCacheClientOptions& Options);
 
-std::unique_ptr<UpstreamEndpoint> MakeZenUpstreamEndpoint(std::string_view Url);
+std::unique_ptr<UpstreamEndpoint> MakeZenUpstreamEndpoint(std::span<std::string const> Urls);
 
 }  // namespace zen
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index db1be9dea..f9b4d5677 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -193,9 +193,9 @@ public:
 
 				UpstreamCache = zen::MakeUpstreamCache(UpstreamOptions, *m_CacheStore, *m_CidStore);
 
-				if (!UpstreamConfig.ZenConfig.Url.empty())
+				if (!UpstreamConfig.ZenConfig.Urls.empty())
 				{
-					std::unique_ptr<zen::UpstreamEndpoint> ZenEndpoint = zen::MakeZenUpstreamEndpoint(UpstreamConfig.ZenConfig.Url);
+					std::unique_ptr<zen::UpstreamEndpoint> ZenEndpoint = zen::MakeZenUpstreamEndpoint(UpstreamConfig.ZenConfig.Urls);
 					UpstreamCache->RegisterEndpoint(std::move(ZenEndpoint));
 				}