1 files changed, 37 insertions, 17 deletions

diff --git a/src/zenserver/hub/hub.h b/src/zenserver/hub/hub.h
index 28e77e729..f0bde10fc 100644
--- a/src/zenserver/hub/hub.h
+++ b/src/zenserver/hub/hub.h
@@ -4,6 +4,7 @@
 
 #include "hubinstancestate.h"
 #include "resourcemetrics.h"
+#include "storageserverinstance.h"
 
 #include <zencore/system.h>
 #include <zenutil/zenserverprocess.h>
@@ -18,7 +19,7 @@
 
 namespace zen {
 
-class StorageServerInstance;
+class WorkerThreadPool;
 
 /**
  * Hub
@@ -59,6 +60,7 @@ public:
 
 	Hub(const Configuration&				 Config,
 		ZenServerEnvironment&&				 RunEnvironment,
+		WorkerThreadPool*					 OptionalWorkerPool		   = nullptr,
 		AsyncModuleStateChangeCallbackFunc&& ModuleStateChangeCallback = {});
 	~Hub();
 
@@ -78,42 +80,49 @@ public:
 	 */
 	void Shutdown();
 
+	enum class EResponseCode
+	{
+		NotFound,
+		Rejected,
+		Accepted,
+		Completed
+	};
+
+	struct Response
+	{
+		EResponseCode ResponseCode = EResponseCode::Rejected;
+		std::string	  Message;
+	};
+
 	/**
 	 * Provision a storage server instance for the given module ID.
 	 *
 	 * @param ModuleId The ID of the module to provision.
-	 * @param OutInfo If successful, information about the provisioned instance will be returned here.
-	 * @param OutReason If unsuccessful, the reason will be returned here.
+	 * @param OutInfo On success, information about the provisioned instance is returned here.
 	 */
-	bool Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo, std::string& OutReason);
+	Response Provision(std::string_view ModuleId, HubProvisionedInstanceInfo& OutInfo);
 
 	/**
 	 * Deprovision a storage server instance for the given module ID.
 	 *
 	 * @param ModuleId The ID of the module to deprovision.
-	 * @param OutReason If unsuccessful, the reason will be returned here.
-	 * @return true if the instance was found and deprovisioned, false otherwise.
 	 */
-	bool Deprovision(const std::string& ModuleId, std::string& OutReason);
+	Response Deprovision(const std::string& ModuleId);
 
 	/**
 	 * Hibernate a storage server instance for the given module ID.
 	 * The instance is shut down but its data is preserved; it can be woken later.
 	 *
 	 * @param ModuleId The ID of the module to hibernate.
-	 * @param OutReason If unsuccessful, the reason will be returned here (empty = not found).
-	 * @return true if the instance was hibernated, false otherwise.
 	 */
-	bool Hibernate(const std::string& ModuleId, std::string& OutReason);
+	Response Hibernate(const std::string& ModuleId);
 
 	/**
 	 * Wake a hibernated storage server instance for the given module ID.
 	 *
 	 * @param ModuleId The ID of the module to wake.
-	 * @param OutReason If unsuccessful, the reason will be returned here (empty = not found).
-	 * @return true if the instance was woken, false otherwise.
 	 */
-	bool Wake(const std::string& ModuleId, std::string& OutReason);
+	Response Wake(const std::string& ModuleId);
 
 	/**
 	 * Find info about storage server instance for the given module ID.
@@ -144,6 +153,9 @@ public:
 private:
 	const Configuration	 m_Config;
 	ZenServerEnvironment m_RunEnvironment;
+	WorkerThreadPool*	 m_WorkerPool = nullptr;
+	Latch				 m_BackgroundWorkLatch;
+	std::atomic<bool>	 m_ShutdownFlag = false;
 
 	AsyncModuleStateChangeCallbackFunc m_ModuleStateChangeCallback;
 
@@ -155,10 +167,10 @@ private:
 #endif
 	RwLock												m_Lock;
 	std::unordered_map<std::string, size_t>				m_InstanceLookup;
-	std::unordered_set<std::string>						m_DeprovisioningModules;
-	std::unordered_set<std::string>						m_ProvisioningModules;
-	std::unordered_set<std::string>						m_HibernatingModules;
-	std::unordered_set<std::string>						m_WakingModules;
+	std::unordered_map<std::string, uint16_t>			m_DeprovisioningModules;
+	std::unordered_map<std::string, uint16_t>			m_ProvisioningModules;
+	std::unordered_map<std::string, uint16_t>			m_HibernatingModules;
+	std::unordered_map<std::string, uint16_t>			m_WakingModules;
 	std::unordered_set<std::string>						m_RecoveringModules;
 	std::vector<std::unique_ptr<StorageServerInstance>> m_ActiveInstances;
 	std::vector<size_t>									m_FreeActiveInstanceIndexes;
@@ -175,6 +187,14 @@ private:
 	void UpdateStats();
 	void UpdateCapacityMetrics();
 	bool CanProvisionInstance(std::string_view ModuleId, std::string& OutReason);
+	bool IsModuleInFlightLocked(std::string_view ModuleId) const;
+
+	Response InternalDeprovision(const std::string& ModuleId);
+	void	 CompleteProvision(StorageServerInstance::ExclusiveLockedPtr& Instance, uint16_t AllocatedPort, bool IsNewInstance);
+	void	 AbortProvision(std::string_view ModuleId);
+	void	 CompleteDeprovision(StorageServerInstance::ExclusiveLockedPtr& Instance);
+	void	 CompleteHibernate(StorageServerInstance::ExclusiveLockedPtr& Instance);
+	void	 CompleteWake(StorageServerInstance::ExclusiveLockedPtr& Instance);
 
 	class InstanceStateUpdateGuard
 	{