hub async provision/deprovision/hibernate/wake (#891)

- Improvement: Hub provision, deprovision, hibernate, and wake operations are now async. HTTP requests returns 202 Accepted while the operation completes in the background
- Improvement: Hub returns 202 Accepted (instead of 409 Conflict) when the same async operation is already in progress for a module
- Improvement: Hub returns 200 OK when a requested state transition is already satisfied

1 files changed, 172 insertions, 32 deletions

diff --git a/src/zenserver-test/hub-tests.cpp b/src/zenserver-test/hub-tests.cpp
index dbe6fa785..f86bdc5c7 100644
--- a/src/zenserver-test/hub-tests.cpp
+++ b/src/zenserver-test/hub-tests.cpp
@@ -33,6 +33,77 @@ using namespace std::literals;
 
 static const HttpClientSettings kFastTimeout{.ConnectTimeout = std::chrono::milliseconds(200)};
 
+static bool
+WaitForModuleState(HttpClient& Client, std::string_view ModuleId, std::string_view ExpectedState, int TimeoutMs = 10000)
+{
+	Stopwatch Timer;
+	while (Timer.GetElapsedTimeMs() < static_cast<uint64_t>(TimeoutMs))
+	{
+		HttpClient::Response R = Client.Get(fmt::format("modules/{}", ModuleId));
+		if (R && R.AsObject()["state"].AsString() == ExpectedState)
+		{
+			return true;
+		}
+		Sleep(100);
+	}
+	HttpClient::Response R = Client.Get(fmt::format("modules/{}", ModuleId));
+	return R && R.AsObject()["state"].AsString() == ExpectedState;
+}
+
+// Provision a module, retrying on 409 Conflict to handle the window where an async
+// deprovision has removed the module from InstanceLookup but not yet from
+// DeprovisioningModules (which CanProvisionInstance checks).
+static HttpClient::Response
+ProvisionModule(HttpClient& Client, std::string_view ModuleId, int TimeoutMs = 10000)
+{
+	Stopwatch			 Timer;
+	HttpClient::Response Result;
+	do
+	{
+		Result = Client.Post(fmt::format("modules/{}/provision", ModuleId));
+		if (Result || Result.StatusCode != HttpResponseCode::Conflict)
+		{
+			return Result;
+		}
+		Sleep(100);
+	} while (Timer.GetElapsedTimeMs() < static_cast<uint64_t>(TimeoutMs));
+	return Result;
+}
+
+// Wait for a port to stop accepting connections (i.e. the process has terminated).
+// Needed after async deprovision: WaitForModuleGone returns as soon as the module
+// leaves m_InstanceLookup (synchronous), but the background worker that kills the
+// process may not have run yet.
+static bool
+WaitForPortUnreachable(HttpClient& Client, std::string_view Path = "/health/", int TimeoutMs = 10000)
+{
+	Stopwatch Timer;
+	while (Timer.GetElapsedTimeMs() < static_cast<uint64_t>(TimeoutMs))
+	{
+		if (!Client.Get(Path))
+		{
+			return true;
+		}
+		Sleep(100);
+	}
+	return !Client.Get(Path);
+}
+
+static bool
+WaitForModuleGone(HttpClient& Client, std::string_view ModuleId, int TimeoutMs = 10000)
+{
+	Stopwatch Timer;
+	while (Timer.GetElapsedTimeMs() < static_cast<uint64_t>(TimeoutMs))
+	{
+		if (Client.Get(fmt::format("modules/{}", ModuleId)).StatusCode == HttpResponseCode::NotFound)
+		{
+			return true;
+		}
+		Sleep(100);
+	}
+	return Client.Get(fmt::format("modules/{}", ModuleId)).StatusCode == HttpResponseCode::NotFound;
+}
+
 TEST_SUITE_BEGIN("server.hub");
 
 TEST_CASE("hub.lifecycle.children")
@@ -65,9 +136,7 @@ TEST_CASE("hub.lifecycle.children")
 		AbcPort = AbcResult["port"].AsUInt16(0);
 		CHECK_NE(AbcPort, 0);
 
-		Result = Client.Get("modules/abc");
-		REQUIRE(Result);
-		CHECK_EQ(Result.AsObject()["state"].AsString(), "provisioned"sv);
+		REQUIRE(WaitForModuleState(Client, "abc", "provisioned"));
 
 		// This should be a fresh instance with no contents
 
@@ -91,6 +160,8 @@ TEST_CASE("hub.lifecycle.children")
 		DefPort = DefResult["port"].AsUInt16(0);
 		REQUIRE_NE(DefPort, 0);
 
+		REQUIRE(WaitForModuleState(Client, "def", "provisioned"));
+
 		// This should be a fresh instance with no contents
 
 		HttpClient DefClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
@@ -110,21 +181,24 @@ TEST_CASE("hub.lifecycle.children")
 
 	Result = Client.Post("modules/ghi/provision");
 	REQUIRE(Result);
+	REQUIRE(WaitForModuleState(Client, "ghi", "provisioned"));
 
 	// Tear down instances
 
 	Result = Client.Post("modules/abc/deprovision");
 	REQUIRE(Result);
+	REQUIRE(WaitForModuleGone(Client, "abc"));
 	{
 		HttpClient ModClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
-		CHECK(!ModClient.Get("/health/"));
+		CHECK(WaitForPortUnreachable(ModClient));
 	}
 
 	Result = Client.Post("modules/def/deprovision");
 	REQUIRE(Result);
+	REQUIRE(WaitForModuleGone(Client, "def"));
 	{
 		HttpClient ModClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
-		CHECK(!ModClient.Get("/health/"));
+		CHECK(WaitForPortUnreachable(ModClient));
 	}
 
 	Result = Client.Post("modules/ghi/deprovision");
@@ -132,7 +206,7 @@ TEST_CASE("hub.lifecycle.children")
 
 	// re-provision to verify that (de)hydration preserved state
 	{
-		Result = Client.Post("modules/abc/provision");
+		Result = ProvisionModule(Client, "abc");
 		REQUIRE(Result);
 
 		CbObject AbcResult = Result.AsObject();
@@ -140,6 +214,8 @@ TEST_CASE("hub.lifecycle.children")
 		AbcPort = AbcResult["port"].AsUInt16(0);
 		REQUIRE_NE(AbcPort, 0);
 
+		REQUIRE(WaitForModuleState(Client, "abc", "provisioned"));
+
 		// This should contain the content from the previous run
 
 		HttpClient AbcClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
@@ -156,7 +232,7 @@ TEST_CASE("hub.lifecycle.children")
 	}
 
 	{
-		Result = Client.Post("modules/def/provision");
+		Result = ProvisionModule(Client, "def");
 		REQUIRE(Result);
 
 		CbObject DefResult = Result.AsObject();
@@ -164,6 +240,8 @@ TEST_CASE("hub.lifecycle.children")
 		DefPort = DefResult["port"].AsUInt16(0);
 		REQUIRE_NE(DefPort, 0);
 
+		REQUIRE(WaitForModuleState(Client, "def", "provisioned"));
+
 		// This should contain the content from the previous run
 
 		HttpClient DefClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
@@ -181,22 +259,24 @@ TEST_CASE("hub.lifecycle.children")
 
 	Result = Client.Post("modules/abc/deprovision");
 	REQUIRE(Result);
+	REQUIRE(WaitForModuleGone(Client, "abc"));
 	{
 		HttpClient ModClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
-		CHECK(!ModClient.Get("/health/"));
+		CHECK(WaitForPortUnreachable(ModClient));
 	}
 
 	Result = Client.Post("modules/def/deprovision");
 	REQUIRE(Result);
+	REQUIRE(WaitForModuleGone(Client, "def"));
 	{
 		HttpClient ModClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
-		CHECK(!ModClient.Get("/health/"));
+		CHECK(WaitForPortUnreachable(ModClient));
 	}
 
 	// re-provision to verify that (de)hydration preserved state, including
 	// state which was generated after the very first dehydration
 	{
-		Result = Client.Post("modules/abc/provision");
+		Result = ProvisionModule(Client, "abc");
 		REQUIRE(Result);
 
 		CbObject AbcResult = Result.AsObject();
@@ -204,6 +284,8 @@ TEST_CASE("hub.lifecycle.children")
 		AbcPort = AbcResult["port"].AsUInt16(0);
 		REQUIRE_NE(AbcPort, 0);
 
+		REQUIRE(WaitForModuleState(Client, "abc", "provisioned"));
+
 		// This should contain the content from the previous two runs
 
 		HttpClient AbcClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
@@ -221,7 +303,7 @@ TEST_CASE("hub.lifecycle.children")
 	}
 
 	{
-		Result = Client.Post("modules/def/provision");
+		Result = ProvisionModule(Client, "def");
 		REQUIRE(Result);
 
 		CbObject DefResult = Result.AsObject();
@@ -229,6 +311,8 @@ TEST_CASE("hub.lifecycle.children")
 		DefPort = DefResult["port"].AsUInt16(0);
 		REQUIRE_NE(DefPort, 0);
 
+		REQUIRE(WaitForModuleState(Client, "def", "provisioned"));
+
 		// This should contain the content from the previous two runs
 
 		HttpClient DefClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
@@ -247,16 +331,18 @@ TEST_CASE("hub.lifecycle.children")
 
 	Result = Client.Post("modules/abc/deprovision");
 	REQUIRE(Result);
+	REQUIRE(WaitForModuleGone(Client, "abc"));
 	{
 		HttpClient ModClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
-		CHECK(!ModClient.Get("/health/"));
+		CHECK(WaitForPortUnreachable(ModClient));
 	}
 
 	Result = Client.Post("modules/def/deprovision");
 	REQUIRE(Result);
+	REQUIRE(WaitForModuleGone(Client, "def"));
 	{
 		HttpClient ModClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
-		CHECK(!ModClient.Get("/health/"));
+		CHECK(WaitForPortUnreachable(ModClient));
 	}
 
 	// final sanity check that the hub is still responsive and all modules are gone
@@ -393,7 +479,7 @@ TEST_CASE("hub.consul.provision.registration")
 	HttpClient::Response Result = HubClient.Post("modules/testmod/provision");
 	REQUIRE(Result);
 
-	CHECK(Client.HasService("testmod"));
+	REQUIRE(WaitForConsulService(Client, "testmod", true, 10000));
 	{
 		const uint16_t ModulePort = Result.AsObject()["port"].AsUInt16(0);
 		REQUIRE(ModulePort != 0);
@@ -457,6 +543,7 @@ TEST_CASE("hub.consul.provision.registration")
 
 		Result = HubClient.Post("modules/testmod/deprovision");
 		REQUIRE(Result);
+		REQUIRE(WaitForConsulService(Client, "testmod", false, 10000));
 
 		{
 			HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
@@ -482,13 +569,12 @@ TEST_CASE("hub.hibernate.lifecycle")
 	// Provision
 	HttpClient::Response Result = Client.Post("modules/testmod/provision");
 	REQUIRE(Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
 	CHECK_EQ(Result.AsObject()["moduleId"].AsString(), "testmod"sv);
 	const uint16_t ModulePort = Result.AsObject()["port"].AsUInt16(0);
 	REQUIRE_NE(ModulePort, 0);
 
-	Result = Client.Get("modules/testmod");
-	REQUIRE(Result);
-	CHECK_EQ(Result.AsObject()["state"].AsString(), "provisioned"sv);
+	REQUIRE(WaitForModuleState(Client, "testmod", "provisioned"));
 	{
 		HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
 		CHECK(ModClient.Get("/health/"));
@@ -502,11 +588,10 @@ TEST_CASE("hub.hibernate.lifecycle")
 	// Hibernate - state should become "hibernated", server should be unreachable
 	Result = Client.Post("modules/testmod/hibernate");
 	REQUIRE(Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
 	CHECK_EQ(Result.AsObject()["moduleId"].AsString(), "testmod"sv);
 
-	Result = Client.Get("modules/testmod");
-	REQUIRE(Result);
-	CHECK_EQ(Result.AsObject()["state"].AsString(), "hibernated"sv);
+	REQUIRE(WaitForModuleState(Client, "testmod", "hibernated"));
 	{
 		HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
 		CHECK(!ModClient.Get("/health/"));
@@ -515,11 +600,10 @@ TEST_CASE("hub.hibernate.lifecycle")
 	// Wake - state should return to "provisioned", server should be reachable, data should be intact
 	Result = Client.Post("modules/testmod/wake");
 	REQUIRE(Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
 	CHECK_EQ(Result.AsObject()["moduleId"].AsString(), "testmod"sv);
 
-	Result = Client.Get("modules/testmod");
-	REQUIRE(Result);
-	CHECK_EQ(Result.AsObject()["state"].AsString(), "provisioned"sv);
+	REQUIRE(WaitForModuleState(Client, "testmod", "provisioned"));
 	{
 		HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
 		CHECK(ModClient.Get("/health/"));
@@ -532,17 +616,20 @@ TEST_CASE("hub.hibernate.lifecycle")
 	// Deprovision - server should become unreachable
 	Result = Client.Post("modules/testmod/deprovision");
 	REQUIRE(Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
+	REQUIRE(WaitForModuleGone(Client, "testmod"));
 	{
 		HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
-		CHECK(!ModClient.Get("/health/"));
+		CHECK(WaitForPortUnreachable(ModClient));
 	}
 
 	// Re-provision - server should be reachable on its (potentially new) port
-	Result = Client.Post("modules/testmod/provision");
+	Result = ProvisionModule(Client, "testmod");
 	REQUIRE(Result);
 	CHECK_EQ(Result.AsObject()["moduleId"].AsString(), "testmod"sv);
 	const uint16_t ModulePort2 = Result.AsObject()["port"].AsUInt16(0);
 	REQUIRE_NE(ModulePort2, 0);
+	REQUIRE(WaitForModuleState(Client, "testmod", "provisioned"));
 	{
 		HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort2), kFastTimeout);
 		CHECK(ModClient.Get("/health/"));
@@ -551,9 +638,10 @@ TEST_CASE("hub.hibernate.lifecycle")
 	// Final deprovision - server should become unreachable
 	Result = Client.Post("modules/testmod/deprovision");
 	REQUIRE(Result);
+	REQUIRE(WaitForModuleGone(Client, "testmod"));
 	{
 		HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort2), kFastTimeout);
-		CHECK(!ModClient.Get("/health/"));
+		CHECK(WaitForPortUnreachable(ModClient));
 	}
 }
 
@@ -574,24 +662,76 @@ TEST_CASE("hub.hibernate.errors")
 	CHECK(!Result);
 	CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound);
 
-	// Double-hibernate: first call succeeds, second returns 400 (wrong state)
+	Result = Client.Post("modules/unknown/deprovision");
+	CHECK(!Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound);
+
+	Result = Client.Delete("modules/unknown");
+	CHECK(!Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound);
+
+	// Double-provision: second call while first is in-flight returns 202 Accepted with the same port.
 	Result = Client.Post("modules/errmod/provision");
 	REQUIRE(Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
+	const uint16_t ErrmodPort = Result.AsObject()["port"].AsUInt16(0);
+	REQUIRE_NE(ErrmodPort, 0);
+
+	// Provisioning the same module while in-flight returns 202 Accepted with the allocated port.
+	// Evaluated synchronously before WorkerPool dispatch, so safe regardless of timing.
+	Result = Client.Post("modules/errmod/provision");
+	CHECK(Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
+	CHECK_EQ(Result.AsObject()["port"].AsUInt16(0), ErrmodPort);
+
+	REQUIRE(WaitForModuleState(Client, "errmod", "provisioned"));
+
+	// Already provisioned: provision and wake both return 200 Completed.
+	Result = Client.Post("modules/errmod/provision");
+	CHECK(Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
+
+	Result = Client.Post("modules/errmod/wake");
+	CHECK(Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
 
+	// Double-hibernate: second call while first is in-flight returns 202 Accepted.
 	Result = Client.Post("modules/errmod/hibernate");
 	REQUIRE(Result);
 
 	Result = Client.Post("modules/errmod/hibernate");
-	CHECK(!Result);
-	CHECK_EQ(Result.StatusCode, HttpResponseCode::BadRequest);
+	CHECK(Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
+
+	REQUIRE(WaitForModuleState(Client, "errmod", "hibernated"));
 
-	// Wake on provisioned: succeeds (state restored), then waking again returns 400
+	// Already hibernated: hibernate returns 200 Completed.
+	Result = Client.Post("modules/errmod/hibernate");
+	CHECK(Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
+
+	// Double-wake: second call while first is in-flight returns 202 Accepted.
 	Result = Client.Post("modules/errmod/wake");
 	REQUIRE(Result);
 
 	Result = Client.Post("modules/errmod/wake");
-	CHECK(!Result);
-	CHECK_EQ(Result.StatusCode, HttpResponseCode::BadRequest);
+	CHECK(Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
+
+	// Double-deprovision: second call while first is in-flight returns 202 Accepted.
+	// errmod2 is a fresh module to avoid waiting on the still-waking errmod.
+	Result = Client.Post("modules/errmod2/provision");
+	REQUIRE(Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
+	REQUIRE(WaitForModuleState(Client, "errmod2", "provisioned"));
+
+	Result = Client.Post("modules/errmod2/deprovision");
+	REQUIRE(Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
+
+	Result = Client.Post("modules/errmod2/deprovision");
+	CHECK(Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
 }
 
 TEST_SUITE_END();