builds url discovery (#334)

- Feature: Added `--host` option to use Jupiters list of cloud host and zen servers to resolve best hosts
- Feature: Use local zenserver as builds cache if it has the `builds` service enabled and `--cloud-discovery-host` is provided and no remote zenserver cache hosts can be found
- Improvement: Added `--override-host` option as a replacement for `--url` (`--url` still works, but `--override-host` is preferred)

15 files changed, 345 insertions, 77 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 689b29db5..1c24a3c5b 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -883,6 +883,7 @@ namespace {
 		std::string						   StorageName;
 		std::unique_ptr<HttpClient>		   CacheHttp;
 		std::unique_ptr<BuildStorageCache> BuildCacheStorage;
+		std::string						   CacheName;
 	};
 
 	std::vector<uint32_t> CalculateAbsoluteChunkOrders(const std::span<const IoHash>						   LocalChunkHashes,
@@ -8348,7 +8349,14 @@ BuildsCommand::BuildsCommand()
 	auto AddCloudOptions = [this, &AddAuthOptions](cxxopts::Options& Ops) {
 		AddAuthOptions(Ops);
 
-		Ops.add_option("cloud build", "", "url", "Cloud Builds URL", cxxopts::value(m_BuildsUrl), "<url>");
+		Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>");
+		Ops.add_option("cloud build",
+					   "",
+					   "url",
+					   "Cloud Builds host url (legacy - use --override-host)",
+					   cxxopts::value(m_OverrideHost),
+					   "<url>");
+		Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>");
 		Ops.add_option("cloud build",
 					   "",
 					   "assume-http2",
@@ -8657,7 +8665,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	}
 
 	auto ParseStorageOptions = [&]() {
-		if (!m_BuildsUrl.empty())
+		if (!m_OverrideHost.empty() || !m_Host.empty())
 		{
 			if (!m_StoragePath.empty())
 			{
@@ -8676,7 +8684,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	};
 
 	std::unique_ptr<AuthMgr> Auth;
-	HttpClientSettings		 ClientSettings{.AssumeHttp2 = m_AssumeHttp2, .AllowResume = true, .RetryCount = 2};
+	HttpClientSettings		 ClientSettings{.LogCategory = "httpbuildsclient",
+											.AssumeHttp2 = m_AssumeHttp2,
+											.AllowResume = true,
+											.RetryCount	 = 2};
 
 	auto CreateAuthMgr = [&]() {
 		if (!Auth)
@@ -8762,7 +8773,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			ClientSettings.AccessTokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(*Auth);
 		}
 
-		if (!m_BuildsUrl.empty() && !ClientSettings.AccessTokenProvider)
+		if (!ClientSettings.AccessTokenProvider)
 		{
 			ZEN_CONSOLE("Warning: No auth provider given, attempting operation without credentials.");
 		}
@@ -8781,21 +8792,176 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 		StorageInstance Result;
 
+		std::string BuildStorageName = ZEN_CLOUD_STORAGE;
+		std::string BuildCacheName;
+		bool		CacheAssumeHttp2 = false;
 		std::string StorageDescription;
 		std::string CacheDescription;
 
-		if (!m_BuildsUrl.empty())
+		if (!m_Host.empty() || !m_OverrideHost.empty())
 		{
 			ParseAuthOptions();
-			Result.BuildStorageHttp = std::make_unique<HttpClient>(m_BuildsUrl, ClientSettings);
-			StorageDescription		= fmt::format("cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'",
-											  m_BuildsUrl,
+		}
+
+		std::string CloudHost;
+
+		if (!m_Host.empty())
+		{
+			if (m_OverrideHost.empty() || m_ZenCacheHost.empty())
+			{
+				HttpClient			 DiscoveryHttpClient(m_Host, ClientSettings);
+				HttpClient::Response ServerInfoResponse =
+					DiscoveryHttpClient.Get("/api/v1/status/servers", HttpClient::Accept(HttpContentType::kJSON));
+				if (!ServerInfoResponse.IsSuccess())
+				{
+					throw std::runtime_error(fmt::format("Failed to get list of servers from discovery url '{}'. Reason: '{}'",
+														 m_Host,
+														 ServerInfoResponse.ErrorMessage("")));
+				}
+
+				std::string_view JsonResponse		= ServerInfoResponse.AsText();
+				CbObject		 ResponseObjectView = LoadCompactBinaryFromJson(JsonResponse).AsObject();
+
+				if (m_OverrideHost.empty())
+				{
+					CbArrayView	  ServerEndpointsArray = ResponseObjectView["serverEndpoints"sv].AsArrayView();
+					std::uint64_t ServerCount		   = ServerEndpointsArray.Num();
+					if (ServerCount == 0)
+					{
+						throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", m_Host));
+					}
+					for (CbFieldView ServerEndpointView : ServerEndpointsArray)
+					{
+						CbObjectView ServerEndpointObject = ServerEndpointView.AsObjectView();
+
+						std::string_view BaseUrl = ServerEndpointObject["baseUrl"sv].AsString();
+						if (!BaseUrl.empty())
+						{
+							const bool		 AssumeHttp2 = ServerEndpointObject["assumeHttp2"sv].AsBool(false);
+							std::string_view Name		 = ServerEndpointObject["name"sv].AsString();
+
+							HttpClientSettings TestClientSettings{.LogCategory	  = "httpbuildsclient",
+																  .ConnectTimeout = std::chrono::milliseconds{1000},
+																  .Timeout		  = std::chrono::milliseconds{2000},
+																  .AssumeHttp2	  = AssumeHttp2,
+																  .AllowResume	  = true,
+																  .RetryCount	  = 0};
+
+							HttpClient			 TestHttpClient(BaseUrl, TestClientSettings);
+							HttpClient::Response TestResponse = TestHttpClient.Get("/health/live");
+							if (TestResponse.IsSuccess())
+							{
+								CloudHost		 = BaseUrl;
+								m_AssumeHttp2	 = AssumeHttp2;
+								BuildStorageName = Name;
+								break;
+							}
+						}
+					}
+					if (CloudHost.empty())
+					{
+						throw std::runtime_error(
+							fmt::format("Failed to find any usable builds hosts out of {} using {}", ServerCount, m_Host));
+					}
+				}
+
+				auto TestCacheEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> bool {
+					HttpClientSettings	 TestClientSettings{.LogCategory	= "httpcacheclient",
+															.ConnectTimeout = std::chrono::milliseconds{1000},
+															.Timeout		= std::chrono::milliseconds{2000},
+															.AssumeHttp2	= AssumeHttp2,
+															.AllowResume	= true,
+															.RetryCount		= 0};
+					HttpClient			 TestHttpClient(BaseUrl, TestClientSettings);
+					HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds");
+					if (TestResponse.IsSuccess())
+					{
+						return true;
+					}
+					return false;
+				};
+
+				if (m_ZenCacheHost.empty())
+				{
+					CbArrayView	  CacheEndpointsArray = ResponseObjectView["cacheEndpoints"sv].AsArrayView();
+					std::uint64_t CacheCount		  = CacheEndpointsArray.Num();
+					for (CbFieldView CacheEndpointView : CacheEndpointsArray)
+					{
+						CbObjectView CacheEndpointObject = CacheEndpointView.AsObjectView();
+
+						std::string_view BaseUrl = CacheEndpointObject["baseUrl"sv].AsString();
+						if (!BaseUrl.empty())
+						{
+							const bool		 AssumeHttp2 = CacheEndpointObject["assumeHttp2"sv].AsBool(false);
+							std::string_view Name		 = CacheEndpointObject["name"sv].AsString();
+
+							if (TestCacheEndpoint(BaseUrl, AssumeHttp2))
+							{
+								m_ZenCacheHost	 = BaseUrl;
+								CacheAssumeHttp2 = AssumeHttp2;
+								BuildCacheName	 = Name;
+								break;
+							}
+						}
+					}
+					if (m_ZenCacheHost.empty())
+					{
+						ZenServerState State;
+						if (State.InitializeReadOnly())
+						{
+							State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) {
+								if (m_ZenCacheHost.empty())
+								{
+									std::string ZenServerLocalHostUrl =
+										fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load());
+									if (TestCacheEndpoint(ZenServerLocalHostUrl, false))
+									{
+										m_ZenCacheHost	 = ZenServerLocalHostUrl;
+										CacheAssumeHttp2 = false;
+										BuildCacheName	 = "localhost";
+									}
+								}
+							});
+						}
+						if (m_ZenCacheHost.empty())
+						{
+							ZEN_CONSOLE("Warning: Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host);
+						}
+					}
+				}
+				else if (TestCacheEndpoint(m_ZenCacheHost, false))
+				{
+					std::string::size_type HostnameStart  = 0;
+					std::string::size_type HostnameLength = std::string::npos;
+					if (auto StartPos = m_ZenCacheHost.find("//"); StartPos != std::string::npos)
+					{
+						HostnameStart = StartPos + 2;
+					}
+					if (auto EndPos = m_ZenCacheHost.find("/", HostnameStart); EndPos != std::string::npos)
+					{
+						HostnameLength = EndPos - HostnameStart;
+					}
+					BuildCacheName = m_ZenCacheHost.substr(HostnameStart, HostnameLength);
+				}
+			}
+		}
+		else
+		{
+			CloudHost = m_OverrideHost;
+		}
+
+		if (!CloudHost.empty())
+		{
+			Result.BuildStorageHttp = std::make_unique<HttpClient>(CloudHost, ClientSettings);
+			StorageDescription		= fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'",
+											  BuildStorageName.empty() ? "" : fmt::format("{}, ", BuildStorageName),
+											  CloudHost,
 											  Result.BuildStorageHttp->GetSessionId(),
 											  m_Namespace,
 											  m_Bucket);
 			Result.BuildStorage =
 				CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, TempPath / "storage");
-			Result.StorageName = ZEN_CLOUD_STORAGE;
+			Result.StorageName = BuildStorageName;
 		}
 		else if (!m_StoragePath.empty())
 		{
@@ -8810,39 +8976,39 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		}
 		if (!m_ZenCacheHost.empty())
 		{
-			Result.CacheHttp = std::make_unique<HttpClient>(m_ZenCacheHost,
+			Result.CacheHttp		 = std::make_unique<HttpClient>(m_ZenCacheHost,
 															HttpClientSettings{.LogCategory	   = "httpcacheclient",
-																			   .ConnectTimeout = std::chrono::milliseconds{3000},
-																			   .Timeout		   = std::chrono::milliseconds{30000},
-																			   .AssumeHttp2	   = false,
-																			   .AllowResume	   = true,
-																			   .RetryCount	   = 0});
-			if (Result.CacheHttp->Get("/health").IsSuccess())
+																					   .ConnectTimeout = std::chrono::milliseconds{3000},
+																					   .Timeout		   = std::chrono::milliseconds{30000},
+																					   .AssumeHttp2	   = CacheAssumeHttp2,
+																					   .AllowResume	   = true,
+																					   .RetryCount	   = 0});
+			Result.BuildCacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp,
+																  StorageCacheStats,
+																  m_Namespace,
+																  m_Bucket,
+																  TempPath / "zencache",
+																  m_PrimeCacheOnly);
+			CacheDescription		 = fmt::format("Zen {}{}. SessionId: '{}'",
+										   BuildCacheName.empty() ? "" : fmt::format("{}, ", BuildCacheName),
+										   m_ZenCacheHost,
+										   Result.CacheHttp->GetSessionId());
+
+			if (!m_Namespace.empty())
 			{
-				Result.BuildCacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp,
-																	  StorageCacheStats,
-																	  m_Namespace,
-																	  m_Bucket,
-																	  TempPath / "zencache",
-																	  m_PrimeCacheOnly);
-				CacheDescription		 = fmt::format("zen cache {}. SessionId: '{}'", m_ZenCacheHost, Result.CacheHttp->GetSessionId());
-				if (!m_Namespace.empty())
-				{
-					CacheDescription += fmt::format(" {}.", m_Namespace);
-				}
-				if (!m_Bucket.empty())
-				{
-					CacheDescription += fmt::format(" {}.", m_Bucket);
-				}
+				CacheDescription += fmt::format(". Namespace '{}'", m_Namespace);
 			}
-			else
+			if (!m_Bucket.empty())
 			{
-				Result.CacheHttp.reset();
+				CacheDescription += fmt::format(" Bucket '{}'", m_Bucket);
 			}
+			Result.CacheName = BuildCacheName;
+		}
+		ZEN_CONSOLE("Remote: {}", StorageDescription);
+		if (!Result.CacheName.empty())
+		{
+			ZEN_CONSOLE("Cache : {}", CacheDescription);
 		}
-		ZEN_CONSOLE("Remote: {}.{}",
-					StorageDescription,
-					CacheDescription.empty() ? std::string("") : fmt::format(" Remote Cache: {}", CacheDescription));
 		return Result;
 	};
 
@@ -9272,7 +9438,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 			std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath);
 
-			if (m_BuildsUrl.empty() && StoragePath.empty())
+			if (m_OverrideHost.empty() && StoragePath.empty())
 			{
 				StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
 				CreateDirectories(StoragePath);
@@ -9280,7 +9446,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 				m_StoragePath = StoragePath.generic_string();
 			}
 			auto _ = MakeGuard([&]() {
-				if (m_BuildsUrl.empty() && StoragePath.empty())
+				if (m_OverrideHost.empty() && StoragePath.empty())
 				{
 					DeleteDirectories(StoragePath);
 				}
@@ -9632,11 +9798,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		{
 			ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
 
-			if (m_BuildsUrl.empty() && m_StoragePath.empty())
-			{
-				throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help()));
-			}
-
 			if (m_BuildId.empty())
 			{
 				throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help()));
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index 4a77f8bd7..bc5cd235f 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -34,7 +34,8 @@ private:
 	std::string m_ZenFolderPath;
 
 	// cloud builds
-	std::string m_BuildsUrl;
+	std::string m_OverrideHost;
+	std::string m_Host;
 	bool		m_AssumeHttp2 = false;
 	std::string m_Namespace;
 	std::string m_Bucket;
diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp
index 06bfea423..c918f5683 100644
--- a/src/zenserver/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/buildstore/httpbuildstore.cpp
@@ -19,16 +19,22 @@ using namespace std::literals;
 
 ZEN_DEFINE_LOG_CATEGORY_STATIC(LogBuilds, "builds"sv);
 
-HttpBuildStoreService::HttpBuildStoreService(HttpStatsService& StatsService, BuildStore& Store)
+HttpBuildStoreService::HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store)
 : m_Log(logging::Get("builds"))
+, m_StatusService(StatusService)
 , m_StatsService(StatsService)
 , m_BuildStore(Store)
 {
 	Initialize();
+
+	m_StatusService.RegisterHandler("builds", *this);
+	m_StatsService.RegisterHandler("builds", *this);
 }
 
 HttpBuildStoreService::~HttpBuildStoreService()
 {
+	m_StatsService.UnregisterHandler("builds", *this);
+	m_StatusService.UnregisterHandler("builds", *this);
 }
 
 const char*
@@ -42,8 +48,6 @@ HttpBuildStoreService::Initialize()
 {
 	ZEN_LOG_INFO(LogBuilds, "Initializing Builds Service");
 
-	m_StatsService.RegisterHandler("builds", *this);
-
 	m_Router.AddPattern("namespace", "([[:alnum:]-_.]+)");
 	m_Router.AddPattern("bucket", "([[:alnum:]-_.]+)");
 	m_Router.AddPattern("buildid", "([[:xdigit:]]{24})");
@@ -523,4 +527,13 @@ HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request)
 	return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
 }
 
+void
+HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	ZEN_TRACE_CPU("HttpBuildStoreService::Status");
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
 }  // namespace zen
diff --git a/src/zenserver/buildstore/httpbuildstore.h b/src/zenserver/buildstore/httpbuildstore.h
index a59aa882a..50cb5db12 100644
--- a/src/zenserver/buildstore/httpbuildstore.h
+++ b/src/zenserver/buildstore/httpbuildstore.h
@@ -5,6 +5,7 @@
 #include <zencore/stats.h>
 #include <zenhttp/httpserver.h>
 #include <zenhttp/httpstats.h>
+#include <zenhttp/httpstatus.h>
 
 #include <filesystem>
 
@@ -12,16 +13,17 @@ namespace zen {
 
 class BuildStore;
 
-class HttpBuildStoreService final : public zen::HttpService, public IHttpStatsProvider
+class HttpBuildStoreService final : public zen::HttpService, public IHttpStatusProvider, public IHttpStatsProvider
 {
 public:
-	HttpBuildStoreService(HttpStatsService& StatsService, BuildStore& Store);
+	HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store);
 	virtual ~HttpBuildStoreService();
 
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
 
 	virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+	virtual void HandleStatusRequest(HttpServerRequest& Request) override;
 
 private:
 	struct BuildStoreStats
@@ -55,7 +57,8 @@ private:
 
 	HttpRequestRouter m_Router;
 
-	HttpStatsService& m_StatsService;
+	HttpStatusService& m_StatusService;
+	HttpStatsService&  m_StatsService;
 
 	BuildStore&				 m_BuildStore;
 	BuildStoreStats			 m_BuildStoreStats;
diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp
index 104b26954..dfa710ae0 100644
--- a/src/zenserver/frontend/frontend.cpp
+++ b/src/zenserver/frontend/frontend.cpp
@@ -2,6 +2,7 @@
 
 #include "frontend.h"
 
+#include <zencore/compactbinarybuilder.h>
 #include <zencore/endian.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
@@ -26,7 +27,9 @@ static unsigned char gHtmlZipData[] = {
 namespace zen {
 
 ////////////////////////////////////////////////////////////////////////////////
-HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Directory(Directory)
+HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService)
+: m_Directory(Directory)
+, m_StatusService(StatusService)
 {
 	std::filesystem::path SelfPath = GetRunningExecutablePath();
 
@@ -81,10 +84,12 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di
 	{
 		ZEN_INFO("front-end is NOT AVAILABLE");
 	}
+	m_StatusService.RegisterHandler("dashboard", *this);
 }
 
 HttpFrontendService::~HttpFrontendService()
 {
+	m_StatusService.UnregisterHandler("dashboard", *this);
 }
 
 const char*
@@ -95,6 +100,14 @@ HttpFrontendService::BaseUri() const
 
 ////////////////////////////////////////////////////////////////////////////////
 void
+HttpFrontendService::HandleStatusRequest(zen::HttpServerRequest& Request)
+{
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
 HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
 {
 	using namespace std::literals;
diff --git a/src/zenserver/frontend/frontend.h b/src/zenserver/frontend/frontend.h
index 6eac20620..84ffaac42 100644
--- a/src/zenserver/frontend/frontend.h
+++ b/src/zenserver/frontend/frontend.h
@@ -3,23 +3,26 @@
 #pragma once
 
 #include <zenhttp/httpserver.h>
+#include <zenhttp/httpstatus.h>
 #include "zipfs.h"
 
 #include <filesystem>
 
 namespace zen {
 
-class HttpFrontendService final : public zen::HttpService
+class HttpFrontendService final : public zen::HttpService, public IHttpStatusProvider
 {
 public:
-	HttpFrontendService(std::filesystem::path Directory);
+	HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService);
 	virtual ~HttpFrontendService();
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
+	virtual void		HandleStatusRequest(HttpServerRequest& Request) override;
 
 private:
 	ZipFs				  m_ZipFs;
 	std::filesystem::path m_Directory;
+	HttpStatusService&	  m_StatusService;
 };
 
 }  // namespace zen
diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp
index 5af803617..8faf12165 100644
--- a/src/zenserver/objectstore/objectstore.cpp
+++ b/src/zenserver/objectstore/objectstore.cpp
@@ -219,13 +219,17 @@ private:
 	StringBuilderBase& Builder;
 };
 
-HttpObjectStoreService::HttpObjectStoreService(ObjectStoreConfig Cfg) : m_Cfg(std::move(Cfg))
+HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg)
+: m_StatusService(StatusService)
+, m_Cfg(std::move(Cfg))
 {
 	Inititalize();
+	m_StatusService.RegisterHandler("obj", *this);
 }
 
 HttpObjectStoreService::~HttpObjectStoreService()
 {
+	m_StatusService.UnregisterHandler("obj", *this);
 }
 
 const char*
@@ -245,6 +249,14 @@ HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request)
 }
 
 void
+HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
 HttpObjectStoreService::Inititalize()
 {
 	namespace fs = std::filesystem;
diff --git a/src/zenserver/objectstore/objectstore.h b/src/zenserver/objectstore/objectstore.h
index dae979c4c..44e50e208 100644
--- a/src/zenserver/objectstore/objectstore.h
+++ b/src/zenserver/objectstore/objectstore.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include <zenhttp/httpserver.h>
+#include <zenhttp/httpstatus.h>
 #include <atomic>
 #include <filesystem>
 #include <mutex>
@@ -23,14 +24,15 @@ struct ObjectStoreConfig
 	std::vector<BucketConfig> Buckets;
 };
 
-class HttpObjectStoreService final : public zen::HttpService
+class HttpObjectStoreService final : public zen::HttpService, public IHttpStatusProvider
 {
 public:
-	HttpObjectStoreService(ObjectStoreConfig Cfg);
+	HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg);
 	virtual ~HttpObjectStoreService();
 
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
+	virtual void		HandleStatusRequest(HttpServerRequest& Request) override;
 
 private:
 	void				  Inititalize();
@@ -41,6 +43,7 @@ private:
 	void				  GetObject(zen::HttpRouterRequest& Request, const std::string_view Path);
 	void				  PutObject(zen::HttpRouterRequest& Request);
 
+	HttpStatusService&	 m_StatusService;
 	ObjectStoreConfig	 m_Cfg;
 	std::mutex			 BucketsMutex;
 	HttpRequestRouter	 m_Router;
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 6313fd69e..317a419eb 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -235,10 +235,15 @@ namespace {
 
 //////////////////////////////////////////////////////////////////////////
 
-HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr)
+HttpProjectService::HttpProjectService(CidStore&		  Store,
+									   ProjectStore*	  Projects,
+									   HttpStatusService& StatusService,
+									   HttpStatsService&  StatsService,
+									   AuthMgr&			  AuthMgr)
 : m_Log(logging::Get("project"))
 , m_CidStore(Store)
 , m_ProjectStore(Projects)
+, m_StatusService(StatusService)
 , m_StatsService(StatsService)
 , m_AuthMgr(AuthMgr)
 {
@@ -246,8 +251,6 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 
 	using namespace std::literals;
 
-	m_StatsService.RegisterHandler("prj", *this);
-
 	m_Router.AddPattern("project", "([[:alnum:]_.]+)");
 	m_Router.AddPattern("log", "([[:alnum:]_.]+)");
 	m_Router.AddPattern("op", "([[:digit:]]+?)");
@@ -366,11 +369,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 		"details\\$/{project}/{log}/{chunk}",
 		[this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); },
 		HttpVerb::kGet);
+
+	m_StatusService.RegisterHandler("prj", *this);
+	m_StatsService.RegisterHandler("prj", *this);
 }
 
 HttpProjectService::~HttpProjectService()
 {
 	m_StatsService.UnregisterHandler("prj", *this);
+	m_StatusService.UnregisterHandler("prj", *this);
 }
 
 const char*
@@ -466,6 +473,15 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
 }
 
 void
+HttpProjectService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	ZEN_TRACE_CPU("HttpProjectService::Status");
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
 HttpProjectService::HandleProjectListRequest(HttpRouterRequest& Req)
 {
 	ZEN_TRACE_CPU("ProjectService::ProjectList");
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
index 8e74c57a5..295defa5c 100644
--- a/src/zenserver/projectstore/httpprojectstore.h
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -5,6 +5,7 @@
 #include <zencore/stats.h>
 #include <zenhttp/httpserver.h>
 #include <zenhttp/httpstats.h>
+#include <zenhttp/httpstatus.h>
 #include <zenstore/cidstore.h>
 
 namespace zen {
@@ -31,16 +32,21 @@ class ProjectStore;
 // refs:
 //
 
-class HttpProjectService : public HttpService, public IHttpStatsProvider
+class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
 {
 public:
-	HttpProjectService(CidStore& Store, ProjectStore* InProjectStore, HttpStatsService& StatsService, AuthMgr& AuthMgr);
+	HttpProjectService(CidStore&		  Store,
+					   ProjectStore*	  InProjectStore,
+					   HttpStatusService& StatusService,
+					   HttpStatsService&  StatsService,
+					   AuthMgr&			  AuthMgr);
 	~HttpProjectService();
 
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(HttpServerRequest& Request) override;
 
 	virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+	virtual void HandleStatusRequest(HttpServerRequest& Request) override;
 
 private:
 	struct ProjectStats
@@ -89,6 +95,7 @@ private:
 	CidStore&				 m_CidStore;
 	HttpRequestRouter		 m_Router;
 	Ref<ProjectStore>		 m_ProjectStore;
+	HttpStatusService&		 m_StatusService;
 	HttpStatsService&		 m_StatsService;
 	AuthMgr&				 m_AuthMgr;
 	ProjectStats			 m_ProjectStats;
diff --git a/src/zenserver/vfs/vfsservice.cpp b/src/zenserver/vfs/vfsservice.cpp
index d302a10ec..bf761f8d1 100644
--- a/src/zenserver/vfs/vfsservice.cpp
+++ b/src/zenserver/vfs/vfsservice.cpp
@@ -61,7 +61,7 @@ GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb)
 // echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:8558/vfs --data-binary @-
 // echo {"method": "unmount"} | curl.exe http://localhost:8558/vfs --data-binary @-
 
-VfsService::VfsService()
+VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(StatusService)
 {
 	m_Impl = new Impl;
 
@@ -136,10 +136,12 @@ VfsService::VfsService()
 			}
 		},
 		HttpVerb::kPost);
+	m_StatusService.RegisterHandler("vfs", *this);
 }
 
 VfsService::~VfsService()
 {
+	m_StatusService.UnregisterHandler("vfs", *this);
 	delete m_Impl;
 }
 
@@ -169,8 +171,9 @@ VfsService::AddService(Ref<ZenCacheStore>&& Z$)
 
 #else
 
-VfsService::VfsService()
+VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(StatusService)
 {
+	ZEN_UNUSED(StatusService);
 }
 
 VfsService::~VfsService()
@@ -209,6 +212,14 @@ VfsService::BaseUri() const
 }
 
 void
+VfsService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
 VfsService::HandleRequest(HttpServerRequest& HttpServiceRequest)
 {
 	m_Router.HandleRequest(HttpServiceRequest);
diff --git a/src/zenserver/vfs/vfsservice.h b/src/zenserver/vfs/vfsservice.h
index dcdc71e81..0d0168e23 100644
--- a/src/zenserver/vfs/vfsservice.h
+++ b/src/zenserver/vfs/vfsservice.h
@@ -4,6 +4,7 @@
 
 #include <zenbase/refcount.h>
 #include <zenhttp/httpserver.h>
+#include <zenhttp/httpstatus.h>
 #include <zenvfs/vfs.h>
 
 #include <memory>
@@ -24,10 +25,10 @@ class ZenCacheStore;
 
  */
 
-class VfsService : public HttpService
+class VfsService : public HttpService, public IHttpStatusProvider
 {
 public:
-	VfsService();
+	explicit VfsService(HttpStatusService& StatusService);
 	~VfsService();
 
 	void Mount(std::string_view MountPoint);
@@ -39,12 +40,14 @@ public:
 protected:
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(HttpServerRequest& HttpServiceRequest) override;
+	virtual void		HandleStatusRequest(HttpServerRequest& Request) override;
 
 private:
 	struct Impl;
 	Impl* m_Impl = nullptr;
 
-	HttpRequestRouter m_Router;
+	HttpStatusService& m_StatusService;
+	HttpRequestRouter  m_Router;
 
 	friend struct VfsServiceDataSource;
 };
diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp
index ac0aaef8e..7ef84743e 100644
--- a/src/zenserver/workspaces/httpworkspaces.cpp
+++ b/src/zenserver/workspaces/httpworkspaces.cpp
@@ -73,8 +73,12 @@ namespace {
 
 }  // namespace
 
-HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, const WorkspacesServeConfig& Cfg, Workspaces& Workspaces)
+HttpWorkspacesService::HttpWorkspacesService(HttpStatusService&			  StatusService,
+											 HttpStatsService&			  StatsService,
+											 const WorkspacesServeConfig& Cfg,
+											 Workspaces&				  Workspaces)
 : m_Log(logging::Get("workspaces"))
+, m_StatusService(StatusService)
 , m_StatsService(StatsService)
 , m_Config(Cfg)
 , m_Workspaces(Workspaces)
@@ -85,6 +89,7 @@ HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, con
 HttpWorkspacesService::~HttpWorkspacesService()
 {
 	m_StatsService.UnregisterHandler("ws", *this);
+	m_StatusService.UnregisterHandler("ws", *this);
 }
 
 const char*
@@ -149,14 +154,21 @@ HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq)
 }
 
 void
+HttpWorkspacesService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	ZEN_TRACE_CPU("HttpWorkspacesService::Status");
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
 HttpWorkspacesService::Initialize()
 {
 	using namespace std::literals;
 
 	ZEN_LOG_INFO(LogFs, "Initializing Workspaces Service");
 
-	m_StatsService.RegisterHandler("ws", *this);
-
 	m_Router.AddPattern("workspace_id", "([[:xdigit:]]{24})");
 	m_Router.AddPattern("share_id", "([[:xdigit:]]{24})");
 	m_Router.AddPattern("chunk", "([[:xdigit:]]{24})");
@@ -238,6 +250,9 @@ HttpWorkspacesService::Initialize()
 		HttpVerb::kGet);
 
 	RefreshState();
+
+	m_StatsService.RegisterHandler("ws", *this);
+	m_StatusService.RegisterHandler("ws", *this);
 }
 
 std::filesystem::path
diff --git a/src/zenserver/workspaces/httpworkspaces.h b/src/zenserver/workspaces/httpworkspaces.h
index f01f58b86..89a8e8bdc 100644
--- a/src/zenserver/workspaces/httpworkspaces.h
+++ b/src/zenserver/workspaces/httpworkspaces.h
@@ -5,6 +5,7 @@
 #include <zencore/stats.h>
 #include <zenhttp/httpserver.h>
 #include <zenhttp/httpstats.h>
+#include <zenhttp/httpstatus.h>
 
 namespace zen {
 
@@ -16,16 +17,20 @@ struct WorkspacesServeConfig
 	bool				  AllowConfigurationChanges = false;
 };
 
-class HttpWorkspacesService final : public HttpService, public IHttpStatsProvider
+class HttpWorkspacesService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
 {
 public:
-	HttpWorkspacesService(HttpStatsService& StatsService, const WorkspacesServeConfig& Cfg, Workspaces& Workspaces);
+	HttpWorkspacesService(HttpStatusService&		   StatusService,
+						  HttpStatsService&			   StatsService,
+						  const WorkspacesServeConfig& Cfg,
+						  Workspaces&				   Workspaces);
 	virtual ~HttpWorkspacesService();
 
 	virtual const char* BaseUri() const override;
 	virtual void		HandleRequest(HttpServerRequest& Request) override;
 
 	virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+	virtual void HandleStatusRequest(HttpServerRequest& Request) override;
 
 private:
 	struct WorkspacesStats
@@ -80,6 +85,7 @@ private:
 	void ChunkRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId);
 	void ShareRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& InShareId);
 
+	HttpStatusService&			m_StatusService;
 	HttpStatsService&			m_StatsService;
 	const WorkspacesServeConfig m_Config;
 	HttpRequestRouter			m_Router;
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 45c91d691..c7cb2ba6e 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -251,13 +251,14 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 									  *m_JobQueue,
 									  *m_OpenProcessCache,
 									  ProjectStore::Configuration{});
-	m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatsService, *m_AuthMgr});
+	m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr});
 
 	if (ServerOptions.WorksSpacesConfig.Enabled)
 	{
 		m_Workspaces.reset(new Workspaces());
 		m_HttpWorkspacesService.reset(
-			new HttpWorkspacesService(m_StatsService,
+			new HttpWorkspacesService(m_StatusService,
+									  m_StatsService,
 									  {.SystemRootDir			  = ServerOptions.SystemRootDir,
 									   .AllowConfigurationChanges = ServerOptions.WorksSpacesConfig.AllowConfigurationChanges},
 									  *m_Workspaces));
@@ -295,7 +296,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 		m_Http->RegisterService(*m_HttpWorkspacesService);
 	}
 
-	m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot);
+	m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService);
 
 	if (m_FrontendService)
 	{
@@ -314,18 +315,18 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 			ObjCfg.Buckets.push_back(std::move(NewBucket));
 		}
 
-		m_ObjStoreService = std::make_unique<HttpObjectStoreService>(std::move(ObjCfg));
+		m_ObjStoreService = std::make_unique<HttpObjectStoreService>(m_StatusService, std::move(ObjCfg));
 		m_Http->RegisterService(*m_ObjStoreService);
 	}
 
 	if (ServerOptions.BuildStoreConfig.Enabled)
 	{
-		m_BuildStoreService = std::make_unique<HttpBuildStoreService>(m_StatsService, *m_BuildStore);
+		m_BuildStoreService = std::make_unique<HttpBuildStoreService>(m_StatusService, m_StatsService, *m_BuildStore);
 		m_Http->RegisterService(*m_BuildStoreService);
 	}
 
 #if ZEN_WITH_VFS
-	m_VfsService = std::make_unique<VfsService>();
+	m_VfsService = std::make_unique<VfsService>(m_StatusService);
 	m_VfsService->AddService(Ref<ProjectStore>(m_ProjectStore));
 	m_VfsService->AddService(Ref<ZenCacheStore>(m_CacheStore));
 	m_Http->RegisterService(*m_VfsService);