add hub instance info (#869)

- Improvement: Hub module listing now includes per-instance process metrics (memory, CPU time, working set, pagefile usage)
- Improvement: Hub now monitors provisioned instance health in the background and refreshes process metrics periodically
- Improvement: Hub no longer exposes raw `StorageServerInstance` pointers to callers; instance state is returned as value snapshots (`Hub::InstanceInfo`)
- Improvement: Hub instance access is now guarded by RAII per-instance locks (`SharedLockedPtr`/`ExclusiveLockedPtr`), preventing concurrent modifications during provisioning and deprovisioning
- Improvement: Hub instance lifecycle is now tracked as a `HubInstanceState` enum covering transitional states (Provisioning, Deprovisioning, Hibernating, Waking); exposed as a string in the HTTP API and dashboard

1 files changed, 312 insertions, 73 deletions

diff --git a/src/zenserver/hub/hub.cpp b/src/zenserver/hub/hub.cpp
index c35fa61e8..b0208db1f 100644
--- a/src/zenserver/hub/hub.cpp
+++ b/src/zenserver/hub/hub.cpp
@@ -9,6 +9,7 @@
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
+#include <zencore/timer.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <EASTL/fixed_vector.h>
@@ -150,6 +151,9 @@ Hub::Hub(const Configuration&		   Config,
 
 	ZEN_ASSERT(uint64_t(Config.BasePortNumber) + Config.InstanceLimit <= std::numeric_limits<uint16_t>::max());
 
+	m_InstanceLookup.reserve(Config.InstanceLimit);
+	m_ActiveInstances.reserve(Config.InstanceLimit);
+
 	m_FreePorts.resize(Config.InstanceLimit);
 	std::iota(m_FreePorts.begin(), m_FreePorts.end(), Config.BasePortNumber);
 
@@ -167,6 +171,7 @@ Hub::Hub(const Configuration&		   Config,
 		}
 	}
 #endif
+	m_WatchDog = std::thread([this]() { WatchDog(); });
 }
 
 Hub::~Hub()
@@ -175,26 +180,43 @@ Hub::~Hub()
 	{
 		ZEN_INFO("Hub service shutting down, deprovisioning any current instances");
 
+		m_WatchDogEvent.Set();
+		if (m_WatchDog.joinable())
+		{
+			m_WatchDog.join();
+		}
+
+		m_WatchDog = {};
+
+		// WatchDog has been joined; no concurrent access is possible
 		m_Lock.WithExclusiveLock([this] {
-			for (auto& [ModuleId, Instance] : m_Instances)
+			for (auto& [ModuleId, ActiveInstanceIndex] : m_InstanceLookup)
 			{
-				uint16_t	BasePort = Instance->GetBasePort();
-				std::string BaseUri;  // TODO?
-
-				if (m_DeprovisionedModuleCallback)
+				std::unique_ptr<StorageServerInstance>& InstanceRaw = m_ActiveInstances[ActiveInstanceIndex];
 				{
-					try
-					{
-						m_DeprovisionedModuleCallback(ModuleId, HubProvisionedInstanceInfo{.BaseUri = BaseUri, .Port = BasePort});
-					}
-					catch (const std::exception& Ex)
+					StorageServerInstance::ExclusiveLockedPtr Instance(InstanceRaw->LockExclusive(/*Wait*/ true));
+
+					uint16_t	BasePort = InstanceRaw->GetBasePort();
+					std::string BaseUri;  // TODO?
+
+					if (m_DeprovisionedModuleCallback)
 					{
-						ZEN_ERROR("Deprovision callback for module {} failed. Reason: '{}'", ModuleId, Ex.what());
+						try
+						{
+							m_DeprovisionedModuleCallback(ModuleId, HubProvisionedInstanceInfo{.BaseUri = BaseUri, .Port = BasePort});
+						}
+						catch (const std::exception& Ex)
+						{
+							ZEN_ERROR("Deprovision callback for module {} failed. Reason: '{}'", ModuleId, Ex.what());
+						}
 					}
+					Instance.Deprovision();
 				}
-				Instance->Deprovision();
+				InstanceRaw.reset();
 			}
-			m_Instances.clear();
+			m_InstanceLookup.clear();
+			m_ActiveInstances.clear();
+			m_FreeActiveInstanceIndexes.clear();
 		});
 	}
 	catch (const std::exception& e)
@@ -206,20 +228,20 @@ Hub::~Hub()
 bool
 Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, std::string& OutReason)
 {
-	StorageServerInstance* Instance		 = nullptr;
-	bool				   IsNewInstance = false;
+	StorageServerInstance::ExclusiveLockedPtr Instance;
+	bool									  IsNewInstance = false;
+	uint16_t								  AllocatedPort = 0;
 	{
 		RwLock::ExclusiveLockScope _(m_Lock);
-		uint16_t				   AllocatedPort		= 0;
-		auto					   RestoreAllocatedPort = MakeGuard([this, &AllocatedPort]() {
-			  if (AllocatedPort != 0)
+		auto					   RestoreAllocatedPort = MakeGuard([this, ModuleId, &IsNewInstance, &AllocatedPort]() {
+			  if (IsNewInstance && AllocatedPort != 0 && !m_InstanceLookup.contains(std::string(ModuleId)))
 			  {
 				  m_FreePorts.push_back(AllocatedPort);
 				  AllocatedPort = 0;
 			  }
 		  });
 
-		if (auto It = m_Instances.find(std::string(ModuleId)); It == m_Instances.end())
+		if (auto It = m_InstanceLookup.find(std::string(ModuleId)); It == m_InstanceLookup.end())
 		{
 			std::string Reason;
 			if (!CanProvisionInstance(ModuleId, /* out */ Reason))
@@ -231,11 +253,12 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s
 				return false;
 			}
 
+			IsNewInstance = true;
+
 			AllocatedPort = m_FreePorts.front();
+			ZEN_ASSERT(AllocatedPort != 0);
 			m_FreePorts.pop_front();
 
-			IsNewInstance = true;
-
 			auto NewInstance = std::make_unique<StorageServerInstance>(
 				m_RunEnvironment,
 				StorageServerInstance::Configuration{.BasePort					   = AllocatedPort,
@@ -245,63 +268,110 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s
 													 .CoreLimit					   = m_Config.InstanceCoreLimit,
 													 .ConfigPath				   = m_Config.InstanceConfigPath},
 				ModuleId);
+
 #if ZEN_PLATFORM_WINDOWS
 			if (m_JobObject.IsValid())
 			{
 				NewInstance->SetJobObject(&m_JobObject);
 			}
 #endif
-			Instance = NewInstance.get();
-			m_Instances.emplace(std::string(ModuleId), std::move(NewInstance));
-			AllocatedPort = 0;
+
+			Instance = NewInstance->LockExclusive(/*Wait*/ true);
+
+			size_t ActiveInstanceIndex = (size_t)-1;
+			if (!m_FreeActiveInstanceIndexes.empty())
+			{
+				ActiveInstanceIndex = m_FreeActiveInstanceIndexes.back();
+				m_FreeActiveInstanceIndexes.pop_back();
+				ZEN_ASSERT(m_ActiveInstances.size() > ActiveInstanceIndex);
+				m_ActiveInstances[ActiveInstanceIndex] = std::move(NewInstance);
+			}
+			else
+			{
+				ActiveInstanceIndex = m_ActiveInstances.size();
+				m_ActiveInstances.emplace_back(std::move(NewInstance));
+			}
+			ZEN_ASSERT(ActiveInstanceIndex != (size_t)-1);
+			m_InstanceLookup.insert_or_assign(std::string(ModuleId), ActiveInstanceIndex);
 
 			ZEN_INFO("Created new storage server instance for module '{}'", ModuleId);
+
+			const int CurrentInstanceCount = gsl::narrow_cast<int>(m_InstanceLookup.size());
+			int		  CurrentMaxCount	   = m_MaxInstanceCount.load();
+			const int NewMax			   = Max(CurrentMaxCount, CurrentInstanceCount);
+
+			m_MaxInstanceCount.compare_exchange_weak(CurrentMaxCount, NewMax);
 		}
 		else
 		{
-			Instance = It->second.get();
+			const size_t ActiveInstanceIndex = It->second;
+			ZEN_ASSERT(m_ActiveInstances.size() > ActiveInstanceIndex);
+
+			std::unique_ptr<StorageServerInstance>& InstanceRaw = m_ActiveInstances[ActiveInstanceIndex];
+			Instance											= InstanceRaw->LockExclusive(/*Wait*/ true);
+			AllocatedPort										= InstanceRaw->GetBasePort();
 		}
 
 		m_ProvisioningModules.emplace(std::string(ModuleId));
 	}
 
-	ZEN_ASSERT(Instance != nullptr);
+	ZEN_ASSERT(Instance);
 
 	auto RemoveProvisioningModule = MakeGuard([&] {
 		RwLock::ExclusiveLockScope _(m_Lock);
 		m_ProvisioningModules.erase(std::string(ModuleId));
+		if (IsNewInstance && AllocatedPort != 0 && !m_InstanceLookup.contains(std::string(ModuleId)))
+		{
+			m_FreePorts.push_back(AllocatedPort);
+			AllocatedPort = 0;
+		}
 	});
 
-	// NOTE: this is done while not holding the lock, as provisioning may take time
+	// NOTE: this is done while not holding the hub lock, as provisioning may take time
 	// and we don't want to block other operations. We track which modules are being
 	// provisioned using m_ProvisioningModules, and reject attempts to provision/deprovision
 	// those modules while in this state.
 
-	UpdateStats();
-
 	try
 	{
-		Instance->Provision();
+		Instance.Provision();
+		Instance = {};
 	}
 	catch (const std::exception& Ex)
 	{
 		ZEN_ERROR("Failed to provision storage server instance for module '{}': {}", ModuleId, Ex.what());
+		Instance = {};
+
 		if (IsNewInstance)
 		{
-			// Clean up
-			RwLock::ExclusiveLockScope _(m_Lock);
-			if (auto It = m_Instances.find(std::string(ModuleId)); It != m_Instances.end())
+			// Clean up failed instance provisioning
+			std::unique_ptr<StorageServerInstance> DestroyInstance;
+			{
+				RwLock::ExclusiveLockScope _(m_Lock);
+				if (auto It = m_InstanceLookup.find(std::string(ModuleId)); It != m_InstanceLookup.end())
+				{
+					const size_t ActiveInstanceIndex = It->second;
+					ZEN_ASSERT(ActiveInstanceIndex < m_ActiveInstances.size());
+					DestroyInstance = std::move(m_ActiveInstances[ActiveInstanceIndex]);
+					ZEN_ASSERT(DestroyInstance);
+					ZEN_ASSERT(!m_ActiveInstances[ActiveInstanceIndex]);
+					m_FreeActiveInstanceIndexes.push_back(ActiveInstanceIndex);
+					m_InstanceLookup.erase(It);
+				}
+			}
+			try
+			{
+				DestroyInstance.reset();
+			}
+			catch (const std::exception& Ex)
 			{
-				ZEN_ASSERT(It->second != nullptr);
-				uint16_t BasePort = It->second->GetBasePort();
-				m_FreePorts.push_back(BasePort);
-				m_Instances.erase(It);
+				ZEN_ERROR("Failed to destroy instance for failed provision module '{}': {}", ModuleId, Ex.what());
 			}
 		}
 		return false;
 	}
 
-	OutInfo.Port = Instance->GetBasePort();
+	OutInfo.Port = AllocatedPort;
 	// TODO: base URI? Would need to know what host name / IP to use
 
 	if (m_ProvisionedModuleCallback)
@@ -322,7 +392,8 @@ Hub::Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, s
 bool
 Hub::Deprovision(const std::string& ModuleId, std::string& OutReason)
 {
-	std::unique_ptr<StorageServerInstance> Instance;
+	std::unique_ptr<StorageServerInstance>	  RawInstance;
+	StorageServerInstance::ExclusiveLockedPtr Instance;
 
 	{
 		RwLock::ExclusiveLockScope _(m_Lock);
@@ -336,22 +407,31 @@ Hub::Deprovision(const std::string& ModuleId, std::string& OutReason)
 			return false;
 		}
 
-		if (auto It = m_Instances.find(ModuleId); It == m_Instances.end())
+		if (auto It = m_InstanceLookup.find(ModuleId); It == m_InstanceLookup.end())
 		{
 			ZEN_WARN("Attempted to deprovision non-existent module '{}'", ModuleId);
 
-			// Not found, OutReason should be empty
+			// Not found, OutReason left empty
 			return false;
 		}
 		else
 		{
-			Instance = std::move(It->second);
-			m_Instances.erase(It);
+			const size_t ActiveInstanceIndex = It->second;
+			ZEN_ASSERT(ActiveInstanceIndex < m_ActiveInstances.size());
+			RawInstance = std::move(m_ActiveInstances[ActiveInstanceIndex]);
+			ZEN_ASSERT(RawInstance != nullptr);
+			m_FreeActiveInstanceIndexes.push_back(ActiveInstanceIndex);
+			m_InstanceLookup.erase(It);
 			m_DeprovisioningModules.emplace(ModuleId);
+
+			Instance = RawInstance->LockExclusive(/*Wait*/ true);
 		}
 	}
 
-	uint16_t	BasePort = Instance->GetBasePort();
+	ZEN_ASSERT(RawInstance);
+	ZEN_ASSERT(Instance);
+
+	uint16_t	BasePort = RawInstance->GetBasePort();
 	std::string BaseUri;  // TODO?
 
 	if (m_DeprovisionedModuleCallback)
@@ -366,57 +446,82 @@ Hub::Deprovision(const std::string& ModuleId, std::string& OutReason)
 		}
 	}
 
-	// The module is deprovisioned outside the lock to avoid blocking other operations.
+	// The module is deprovisioned outside the hub lock to avoid blocking other operations.
 	//
 	// To ensure that no new provisioning can occur while we're deprovisioning,
 	// we add the module ID to m_DeprovisioningModules and remove it once
 	// deprovisioning is complete.
 
 	auto _ = MakeGuard([&] {
-		RwLock::ExclusiveLockScope _(m_Lock);
-		m_DeprovisioningModules.erase(ModuleId);
-		m_FreePorts.push_back(BasePort);
+		{
+			RwLock::ExclusiveLockScope _(m_Lock);
+			m_DeprovisioningModules.erase(ModuleId);
+			m_FreePorts.push_back(BasePort);
+		}
 	});
 
-	Instance->Deprovision();
+	Instance.Deprovision();
+
+	Instance = {};
 
 	return true;
 }
 
 bool
-Hub::Find(std::string_view ModuleId, StorageServerInstance** OutInstance)
+Hub::Find(std::string_view ModuleId, InstanceInfo* OutInstanceInfo)
 {
 	RwLock::SharedLockScope _(m_Lock);
-	if (auto It = m_Instances.find(std::string(ModuleId)); It != m_Instances.end())
+	if (auto It = m_InstanceLookup.find(std::string(ModuleId)); It != m_InstanceLookup.end())
 	{
-		if (OutInstance)
+		if (OutInstanceInfo)
 		{
-			*OutInstance = It->second.get();
+			const size_t ActiveInstanceIndex = It->second;
+			ZEN_ASSERT(ActiveInstanceIndex < m_ActiveInstances.size());
+			const std::unique_ptr<StorageServerInstance>& Instance = m_ActiveInstances[ActiveInstanceIndex];
+			ZEN_ASSERT(Instance);
+			InstanceInfo Info{
+				Instance->GetState(),
+				std::chrono::system_clock::now()  // TODO
+			};
+			Instance->GetProcessMetrics(Info.Metrics);
+
+			*OutInstanceInfo = Info;
 		}
 		return true;
 	}
-	else if (OutInstance)
-	{
-		*OutInstance = nullptr;
-	}
 	return false;
 }
 
 void
-Hub::EnumerateModules(std::function<void(StorageServerInstance&)> Callback)
+Hub::EnumerateModules(std::function<void(std::string_view ModuleId, const InstanceInfo&)> Callback)
 {
-	RwLock::SharedLockScope _(m_Lock);
-	for (auto& It : m_Instances)
+	std::vector<std::pair<std::string, InstanceInfo>> Infos;
+	{
+		RwLock::SharedLockScope _(m_Lock);
+		for (auto& [ModuleId, ActiveInstanceIndex] : m_InstanceLookup)
+		{
+			const std::unique_ptr<StorageServerInstance>& Instance = m_ActiveInstances[ActiveInstanceIndex];
+			ZEN_ASSERT(Instance);
+			InstanceInfo Info{
+				Instance->GetState(),
+				std::chrono::system_clock::now()  // TODO
+			};
+			Instance->GetProcessMetrics(Info.Metrics);
+
+			Infos.push_back(std::make_pair(std::string(Instance->GetModuleId()), Info));
+		}
+	}
+
+	for (const std::pair<std::string, InstanceInfo>& Info : Infos)
 	{
-		Callback(*It.second);
+		Callback(Info.first, Info.second);
 	}
 }
 
 int
 Hub::GetInstanceCount()
 {
-	RwLock::SharedLockScope _(m_Lock);
-	return gsl::narrow_cast<int>(m_Instances.size());
+	return m_Lock.WithSharedLock([this]() { return gsl::narrow_cast<int>(m_InstanceLookup.size()); });
 }
 
 void
@@ -424,13 +529,19 @@ Hub::UpdateCapacityMetrics()
 {
 	m_HostMetrics = GetSystemMetrics();
 
-	// Update per-instance metrics
+	// TODO: Should probably go into WatchDog and use atomic for update so it can be read without locks...
+	// Per-instance stats are already refreshed by WatchDog and are readable via the Find and EnumerateModules
 }
 
 void
 Hub::UpdateStats()
 {
-	m_Lock.WithSharedLock([this] { m_MaxInstanceCount = Max(m_MaxInstanceCount, gsl::narrow_cast<int>(m_Instances.size())); });
+	int CurrentInstanceCount = m_Lock.WithSharedLock([this] { return gsl::narrow_cast<int>(m_InstanceLookup.size()); });
+	int CurrentMaxCount		 = m_MaxInstanceCount.load();
+
+	int NewMax = Max(CurrentMaxCount, CurrentInstanceCount);
+
+	m_MaxInstanceCount.compare_exchange_weak(CurrentMaxCount, NewMax);
 }
 
 bool
@@ -450,19 +561,19 @@ Hub::CanProvisionInstance(std::string_view ModuleId, std::string& OutReason)
 		return false;
 	}
 
-	if (gsl::narrow_cast<int>(m_Instances.size()) >= m_Config.InstanceLimit)
+	if (gsl::narrow_cast<int>(m_InstanceLookup.size()) >= m_Config.InstanceLimit)
 	{
 		OutReason = fmt::format("instance limit ({}) exceeded", m_Config.InstanceLimit);
 
 		return false;
 	}
 
-	// Since deprovisioning happens outside the lock and we don't add the port back until the instance is full shut down we might be under
-	// the instance limit but all ports may be in use
+	// Since deprovisioning happens outside the lock and we don't return the port until the instance is fully shut down, we might be below
+	// the instance count limit but with no free ports available
 	if (m_FreePorts.empty())
 	{
 		OutReason = fmt::format("no free ports available, deprovisioning of instances might be in flight ({})",
-								m_Config.InstanceLimit - m_Instances.size());
+								m_Config.InstanceLimit - m_InstanceLookup.size());
 
 		return false;
 	}
@@ -472,6 +583,71 @@ Hub::CanProvisionInstance(std::string_view ModuleId, std::string& OutReason)
 	return true;
 }
 
+void
+Hub::WatchDog()
+{
+	constexpr uint64_t WatchDogWakeupTimeMs		= 5000;
+	constexpr uint64_t WatchDogProcessingTimeMs = 500;
+
+	size_t CheckInstanceIndex = 0;
+	while (!m_WatchDogEvent.Wait(WatchDogWakeupTimeMs))
+	{
+		try
+		{
+			size_t MaxCheckCount = m_Lock.WithSharedLock([this]() { return m_InstanceLookup.size(); });
+
+			Stopwatch Timer;
+			while (MaxCheckCount-- > 0 && Timer.GetElapsedTimeMs() < WatchDogProcessingTimeMs && !m_WatchDogEvent.Wait(5))
+			{
+				StorageServerInstance::SharedLockedPtr LockedInstance;
+				m_Lock.WithSharedLock([this, &CheckInstanceIndex, &LockedInstance]() {
+					if (m_InstanceLookup.empty())
+					{
+						return;
+					}
+
+					size_t				   MaxLoopCount = m_ActiveInstances.size();
+					StorageServerInstance* Instance		= nullptr;
+					while (MaxLoopCount-- > 0 && !Instance)
+					{
+						CheckInstanceIndex++;
+						if (CheckInstanceIndex >= m_ActiveInstances.size())
+						{
+							CheckInstanceIndex = 0;
+						}
+						Instance = (CheckInstanceIndex < m_ActiveInstances.size()) ? m_ActiveInstances[CheckInstanceIndex].get() : nullptr;
+					}
+
+					if (Instance)
+					{
+						LockedInstance = Instance->LockShared(/*Wait*/ false);
+					}
+				});
+
+				if (LockedInstance)
+				{
+					if (LockedInstance.IsRunning())
+					{
+						LockedInstance.UpdateMetrics();
+					}
+					else if (LockedInstance.GetState() == HubInstanceState::Provisioned)
+					{
+						// Process is not running but state says it should be — instance died unexpectedly.
+						// TODO: Track and attempt recovery.
+					}
+					// else: transitional state (Provisioning, Deprovisioning, Hibernating, Waking) — expected, skip.
+					LockedInstance = {};
+				}
+			}
+		}
+		catch (const std::exception& Ex)
+		{
+			// TODO: Catch specific errors such as asserts, OOM, OOD, system_error etc
+			ZEN_ERROR("Hub watchdog threw exception: {}", Ex.what());
+		}
+	}
+}
+
 #if ZEN_WITH_TESTS
 
 TEST_SUITE_BEGIN("server.hub");
@@ -507,7 +683,9 @@ TEST_CASE("hub.provision_basic")
 	REQUIRE_MESSAGE(ProvisionResult, Reason);
 	CHECK_NE(Info.Port, 0);
 	CHECK_EQ(HubInstance->GetInstanceCount(), 1);
-	CHECK(HubInstance->Find("module_a"));
+	Hub::InstanceInfo InstanceInfo;
+	REQUIRE(HubInstance->Find("module_a", &InstanceInfo));
+	CHECK_EQ(InstanceInfo.State, HubInstanceState::Provisioned);
 
 	const bool DeprovisionResult = HubInstance->Deprovision("module_a", Reason);
 	CHECK(DeprovisionResult);
@@ -541,7 +719,9 @@ TEST_CASE("hub.provision_config")
 	REQUIRE_MESSAGE(ProvisionResult, Reason);
 	CHECK_NE(Info.Port, 0);
 	CHECK_EQ(HubInstance->GetInstanceCount(), 1);
-	CHECK(HubInstance->Find("module_a"));
+	Hub::InstanceInfo InstanceInfo;
+	REQUIRE(HubInstance->Find("module_a", &InstanceInfo));
+	CHECK_EQ(InstanceInfo.State, HubInstanceState::Provisioned);
 
 	HttpClient			 Client(fmt::format("http://127.0.0.1:{}{}", Info.Port, Info.BaseUri));
 	HttpClient::Response TestResponse = Client.Get("/status/builds");
@@ -660,7 +840,10 @@ TEST_CASE("hub.enumerate_modules")
 	REQUIRE_MESSAGE(HubInstance->Provision("enum_b", Info, Reason), Reason);
 
 	std::vector<std::string> Ids;
-	HubInstance->EnumerateModules([&](StorageServerInstance& Instance) { Ids.push_back(std::string(Instance.GetModuleId())); });
+	HubInstance->EnumerateModules([&](std::string_view ModuleId, const Hub::InstanceInfo& Info) {
+		Ids.push_back(std::string(ModuleId));
+		CHECK_EQ(Info.State, HubInstanceState::Provisioned);
+	});
 	CHECK_EQ(Ids.size(), 2u);
 	const bool FoundA = std::find(Ids.begin(), Ids.end(), "enum_a") != Ids.end();
 	const bool FoundB = std::find(Ids.begin(), Ids.end(), "enum_b") != Ids.end();
@@ -669,7 +852,10 @@ TEST_CASE("hub.enumerate_modules")
 
 	HubInstance->Deprovision("enum_a", Reason);
 	Ids.clear();
-	HubInstance->EnumerateModules([&](StorageServerInstance& Instance) { Ids.push_back(std::string(Instance.GetModuleId())); });
+	HubInstance->EnumerateModules([&](std::string_view ModuleId, const Hub::InstanceInfo& Info) {
+		Ids.push_back(std::string(ModuleId));
+		CHECK_EQ(Info.State, HubInstanceState::Provisioned);
+	});
 	REQUIRE_EQ(Ids.size(), 1u);
 	CHECK_EQ(Ids[0], "enum_b");
 }
@@ -936,6 +1122,59 @@ TEST_CASE("hub.job_object")
 }
 #	endif	// ZEN_PLATFORM_WINDOWS
 
+TEST_CASE("hub.instance_state_basic")
+{
+	ScopedTemporaryDirectory TempDir;
+	Hub::Configuration		 Config;
+	Config.BasePortNumber			 = 22400;
+	std::unique_ptr<Hub> HubInstance = hub_testutils::MakeHub(TempDir.Path(), Config);
+
+	HubProvisionedInstanceInfo ProvInfo;
+	Hub::InstanceInfo		   Info;
+	std::string				   Reason;
+
+	REQUIRE_MESSAGE(HubInstance->Provision("state_a", ProvInfo, Reason), Reason);
+
+	REQUIRE(HubInstance->Find("state_a", &Info));
+	CHECK_EQ(Info.State, HubInstanceState::Provisioned);
+
+	HubInstance->Deprovision("state_a", Reason);
+	CHECK_FALSE(HubInstance->Find("state_a"));
+}
+
+TEST_CASE("hub.instance_state_enumerate")
+{
+	ScopedTemporaryDirectory TempDir;
+	Hub::Configuration		 Config;
+	Config.BasePortNumber			 = 22500;
+	std::unique_ptr<Hub> HubInstance = hub_testutils::MakeHub(TempDir.Path(), Config);
+
+	HubProvisionedInstanceInfo ProvInfo;
+	std::string				   Reason;
+	REQUIRE_MESSAGE(HubInstance->Provision("estate_a", ProvInfo, Reason), Reason);
+	REQUIRE_MESSAGE(HubInstance->Provision("estate_b", ProvInfo, Reason), Reason);
+
+	int ProvisionedCount = 0;
+	HubInstance->EnumerateModules([&](std::string_view, const Hub::InstanceInfo& InstanceInfo) {
+		if (InstanceInfo.State == HubInstanceState::Provisioned)
+		{
+			ProvisionedCount++;
+		}
+	});
+	CHECK_EQ(ProvisionedCount, 2);
+
+	HubInstance->Deprovision("estate_a", Reason);
+
+	ProvisionedCount = 0;
+	HubInstance->EnumerateModules([&](std::string_view, const Hub::InstanceInfo& InstanceInfo) {
+		if (InstanceInfo.State == HubInstanceState::Provisioned)
+		{
+			ProvisionedCount++;
+		}
+	});
+	CHECK_EQ(ProvisionedCount, 1);
+}
+
 TEST_SUITE_END();
 
 void