zen hub command (#877)

- Feature: Added `zen hub` command for managing a hub server and its provisioned module instances:
  - `zen hub up` - Start a hub server (equivalent to `zen up` in hub mode)
  - `zen hub down` - Shut down a hub server
  - `zen hub provision <moduleid>` - Provision a storage server instance for a module
  - `zen hub deprovision <moduleid>` - Deprovision a storage server instance
  - `zen hub hibernate <moduleid>` - Hibernate a provisioned instance (shut down, data preserved)
  - `zen hub wake <moduleid>` - Wake a hibernated instance
  - `zen hub status [moduleid]` - Show state of all instances or a specific module
- Feature: Added new hub HTTP endpoints for instance lifecycle management:
  - `POST /hub/modules/{moduleid}/hibernate` - Hibernate the instance for the given module
  - `POST /hub/modules/{moduleid}/wake` - Wake a hibernated instance for the given module
- Improvement: `zen up` refactored to use shared `StartupZenServer`/`ShutdownZenServer` helpers (also used by `zen hub up`/`zen hub down`)
- Bugfix: Fixed shutdown event not being cleared after the server process exits in `ZenServerInstance::Shutdown()`, which could cause stale state on reuse

1 files changed, 300 insertions, 145 deletions

diff --git a/src/zenserver-test/hub-tests.cpp b/src/zenserver-test/hub-tests.cpp
index a372b11e5..dbe6fa785 100644
--- a/src/zenserver-test/hub-tests.cpp
+++ b/src/zenserver-test/hub-tests.cpp
@@ -31,216 +31,238 @@ namespace zen::tests::hub {
 
 using namespace std::literals;
 
+static const HttpClientSettings kFastTimeout{.ConnectTimeout = std::chrono::milliseconds(200)};
+
 TEST_SUITE_BEGIN("server.hub");
 
-TEST_CASE("hub.lifecycle.basic")
+TEST_CASE("hub.lifecycle.children")
 {
-	{
-		ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kHubServer);
+	ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kHubServer);
 
-		const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady();
-		CHECK(PortNumber != 0);
+	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady("--hub-instance-corelimit=2 --hub-instance-http-threads=6");
+	REQUIRE(PortNumber != 0);
 
-		HttpClient Client(Instance.GetBaseUri() + "/hub/");
+	HttpClient Client(Instance.GetBaseUri() + "/hub/", kFastTimeout);
 
+	// Verify the hub starts with no modules
+	{
 		HttpClient::Response Result = Client.Get("status");
 		REQUIRE(Result);
 		CHECK_EQ(Result.AsObject()["modules"].AsArrayView().Num(), 0u);
 	}
-}
 
-TEST_CASE("hub.lifecycle.children")
-{
-	ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kHubServer);
+	HttpClient::Response Result;
 
-	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady("--hub-instance-corelimit=2 --hub-instance-http-threads=6");
-	REQUIRE(PortNumber != 0);
+	uint16_t AbcPort = 0;
+	uint16_t DefPort = 0;
 
-	SUBCASE("spawn")
 	{
-		HttpClient Client(Instance.GetBaseUri() + "/hub/");
-
-		HttpClient::Response Result = Client.Get("status");
+		Result = Client.Post("modules/abc/provision");
 		REQUIRE(Result);
 
-		{
-			Result = Client.Post("modules/abc/provision");
-			REQUIRE(Result);
+		CbObject AbcResult = Result.AsObject();
+		CHECK(AbcResult["moduleId"].AsString() == "abc"sv);
+		AbcPort = AbcResult["port"].AsUInt16(0);
+		CHECK_NE(AbcPort, 0);
 
-			CbObject AbcResult = Result.AsObject();
-			CHECK(AbcResult["moduleId"].AsString() == "abc"sv);
-			const uint16_t AbcPort = AbcResult["port"].AsUInt16(0);
-			CHECK_NE(AbcPort, 0);
+		Result = Client.Get("modules/abc");
+		REQUIRE(Result);
+		CHECK_EQ(Result.AsObject()["state"].AsString(), "provisioned"sv);
 
-			Result = Client.Get("modules/abc");
-			REQUIRE(Result);
-			CHECK_EQ(Result.AsObject()["state"].AsString(), "provisioned"sv);
+		// This should be a fresh instance with no contents
 
-			// This should be a fresh instance with no contents
+		HttpClient AbcClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
+		CHECK(AbcClient.Get("/health/"));
 
-			HttpClient AbcClient(fmt::format("http://localhost:{}", AbcPort));
+		Result = AbcClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567");
+		CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound);
 
-			Result = AbcClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567");
-			CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound);
+		Result = AbcClient.Put("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567",
+							   IoBufferBuilder::MakeFromMemory(MakeMemoryView("abcdef"sv)));
+		CHECK_EQ(Result.StatusCode, HttpResponseCode::Created);
+	}
 
-			Result = AbcClient.Put("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567",
-								   IoBufferBuilder::MakeFromMemory(MakeMemoryView("abcdef"sv)));
-			CHECK_EQ(Result.StatusCode, HttpResponseCode::Created);
-		}
+	{
+		Result = Client.Post("modules/def/provision");
+		REQUIRE(Result);
 
-		{
-			Result = Client.Post("modules/def/provision");
-			REQUIRE(Result);
+		CbObject DefResult = Result.AsObject();
+		CHECK(DefResult["moduleId"].AsString() == "def"sv);
+		DefPort = DefResult["port"].AsUInt16(0);
+		REQUIRE_NE(DefPort, 0);
 
-			CbObject DefResult = Result.AsObject();
-			CHECK(DefResult["moduleId"].AsString() == "def"sv);
-			const uint16_t DefPort = DefResult["port"].AsUInt16(0);
-			REQUIRE_NE(DefPort, 0);
+		// This should be a fresh instance with no contents
 
-			// This should be a fresh instance with no contents
+		HttpClient DefClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
+		CHECK(DefClient.Get("/health/"));
 
-			HttpClient DefClient(fmt::format("http://localhost:{}", DefPort));
+		Result = DefClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567");
+		CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound);
 
-			Result = DefClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567");
-			CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound);
+		Result = DefClient.Put("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567",
+							   IoBufferBuilder::MakeFromMemory(MakeMemoryView("AbcDef"sv)));
+		CHECK_EQ(Result.StatusCode, HttpResponseCode::Created);
+	}
 
-			Result = DefClient.Put("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567",
-								   IoBufferBuilder::MakeFromMemory(MakeMemoryView("AbcDef"sv)));
-			CHECK_EQ(Result.StatusCode, HttpResponseCode::Created);
-		}
+	// this should be rejected because of the invalid module id
+	Result = Client.Post("modules/!!!!!/provision");
+	CHECK(!Result);
 
-		// this should be rejected because of the invalid module id
-		Result = Client.Post("modules/!!!!!/provision");
-		CHECK(!Result);
+	Result = Client.Post("modules/ghi/provision");
+	REQUIRE(Result);
 
-		Result = Client.Post("modules/ghi/provision");
-		REQUIRE(Result);
+	// Tear down instances
 
-		// Tear down instances
+	Result = Client.Post("modules/abc/deprovision");
+	REQUIRE(Result);
+	{
+		HttpClient ModClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
+		CHECK(!ModClient.Get("/health/"));
+	}
 
-		Result = Client.Post("modules/abc/deprovision");
-		REQUIRE(Result);
+	Result = Client.Post("modules/def/deprovision");
+	REQUIRE(Result);
+	{
+		HttpClient ModClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
+		CHECK(!ModClient.Get("/health/"));
+	}
 
-		Result = Client.Post("modules/def/deprovision");
-		REQUIRE(Result);
+	Result = Client.Post("modules/ghi/deprovision");
+	REQUIRE(Result);
 
-		Result = Client.Post("modules/ghi/deprovision");
+	// re-provision to verify that (de)hydration preserved state
+	{
+		Result = Client.Post("modules/abc/provision");
 		REQUIRE(Result);
 
-		// re-provision to verify that (de)hydration preserved state
-		{
-			Result = Client.Post("modules/abc/provision");
-			REQUIRE(Result);
+		CbObject AbcResult = Result.AsObject();
+		CHECK(AbcResult["moduleId"].AsString() == "abc"sv);
+		AbcPort = AbcResult["port"].AsUInt16(0);
+		REQUIRE_NE(AbcPort, 0);
 
-			CbObject AbcResult = Result.AsObject();
-			CHECK(AbcResult["moduleId"].AsString() == "abc"sv);
-			const uint16_t AbcPort = AbcResult["port"].AsUInt16(0);
-			REQUIRE_NE(AbcPort, 0);
+		// This should contain the content from the previous run
 
-			// This should contain the content from the previous run
+		HttpClient AbcClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
+		CHECK(AbcClient.Get("/health/"));
 
-			HttpClient AbcClient(fmt::format("http://localhost:{}", AbcPort));
+		Result = AbcClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567");
+		CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
 
-			Result = AbcClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567");
-			CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
+		CHECK_EQ(Result.AsText(), "abcdef"sv);
 
-			CHECK_EQ(Result.AsText(), "abcdef"sv);
+		Result = AbcClient.Put("/z$/ns1/b/1123456789abcdef0123456789abcdef01234567",
+							   IoBufferBuilder::MakeFromMemory(MakeMemoryView("ghijklmnop"sv)));
+		CHECK_EQ(Result.StatusCode, HttpResponseCode::Created);
+	}
 
-			Result = AbcClient.Put("/z$/ns1/b/1123456789abcdef0123456789abcdef01234567",
-								   IoBufferBuilder::MakeFromMemory(MakeMemoryView("ghijklmnop"sv)));
-			CHECK_EQ(Result.StatusCode, HttpResponseCode::Created);
-		}
+	{
+		Result = Client.Post("modules/def/provision");
+		REQUIRE(Result);
 
-		{
-			Result = Client.Post("modules/def/provision");
-			REQUIRE(Result);
+		CbObject DefResult = Result.AsObject();
+		CHECK(DefResult["moduleId"].AsString() == "def"sv);
+		DefPort = DefResult["port"].AsUInt16(0);
+		REQUIRE_NE(DefPort, 0);
 
-			CbObject DefResult = Result.AsObject();
-			CHECK(DefResult["moduleId"].AsString() == "def"sv);
-			const uint16_t DefPort = DefResult["port"].AsUInt16(0);
-			REQUIRE_NE(DefPort, 0);
+		// This should contain the content from the previous run
 
-			// This should contain the content from the previous run
+		HttpClient DefClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
+		CHECK(DefClient.Get("/health/"));
 
-			HttpClient DefClient(fmt::format("http://localhost:{}", DefPort));
+		Result = DefClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567");
+		CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
 
-			Result = DefClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567");
-			CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
+		CHECK_EQ(Result.AsText(), "AbcDef"sv);
 
-			CHECK_EQ(Result.AsText(), "AbcDef"sv);
+		Result = DefClient.Put("/z$/ns1/b/1123456789abcdef0123456789abcdef01234567",
+							   IoBufferBuilder::MakeFromMemory(MakeMemoryView("GhijklmNop"sv)));
+		CHECK_EQ(Result.StatusCode, HttpResponseCode::Created);
+	}
 
-			Result = DefClient.Put("/z$/ns1/b/1123456789abcdef0123456789abcdef01234567",
-								   IoBufferBuilder::MakeFromMemory(MakeMemoryView("GhijklmNop"sv)));
-			CHECK_EQ(Result.StatusCode, HttpResponseCode::Created);
-		}
+	Result = Client.Post("modules/abc/deprovision");
+	REQUIRE(Result);
+	{
+		HttpClient ModClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
+		CHECK(!ModClient.Get("/health/"));
+	}
 
-		Result = Client.Post("modules/abc/deprovision");
-		REQUIRE(Result);
+	Result = Client.Post("modules/def/deprovision");
+	REQUIRE(Result);
+	{
+		HttpClient ModClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
+		CHECK(!ModClient.Get("/health/"));
+	}
 
-		Result = Client.Post("modules/def/deprovision");
+	// re-provision to verify that (de)hydration preserved state, including
+	// state which was generated after the very first dehydration
+	{
+		Result = Client.Post("modules/abc/provision");
 		REQUIRE(Result);
 
-		// re-provision to verify that (de)hydration preserved state, including
-		// state which was generated after the very first dehydration
-		{
-			Result = Client.Post("modules/abc/provision");
-			REQUIRE(Result);
-
-			CbObject AbcResult = Result.AsObject();
-			CHECK(AbcResult["moduleId"].AsString() == "abc"sv);
-			const uint16_t AbcPort = AbcResult["port"].AsUInt16(0);
-			REQUIRE_NE(AbcPort, 0);
+		CbObject AbcResult = Result.AsObject();
+		CHECK(AbcResult["moduleId"].AsString() == "abc"sv);
+		AbcPort = AbcResult["port"].AsUInt16(0);
+		REQUIRE_NE(AbcPort, 0);
 
-			// This should contain the content from the previous two runs
+		// This should contain the content from the previous two runs
 
-			HttpClient AbcClient(fmt::format("http://localhost:{}", AbcPort));
+		HttpClient AbcClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
+		CHECK(AbcClient.Get("/health/"));
 
-			Result = AbcClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567");
-			CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
+		Result = AbcClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567");
+		CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
 
-			CHECK_EQ(Result.AsText(), "abcdef"sv);
+		CHECK_EQ(Result.AsText(), "abcdef"sv);
 
-			Result = AbcClient.Get("/z$/ns1/b/1123456789abcdef0123456789abcdef01234567");
-			CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
+		Result = AbcClient.Get("/z$/ns1/b/1123456789abcdef0123456789abcdef01234567");
+		CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
 
-			CHECK_EQ(Result.AsText(), "ghijklmnop"sv);
-		}
-
-		{
-			Result = Client.Post("modules/def/provision");
-			REQUIRE(Result);
+		CHECK_EQ(Result.AsText(), "ghijklmnop"sv);
+	}
 
-			CbObject DefResult = Result.AsObject();
-			REQUIRE(DefResult["moduleId"].AsString() == "def"sv);
-			const uint16_t DefPort = DefResult["port"].AsUInt16(0);
-			REQUIRE_NE(DefPort, 0);
+	{
+		Result = Client.Post("modules/def/provision");
+		REQUIRE(Result);
 
-			// This should contain the content from the previous two runs
+		CbObject DefResult = Result.AsObject();
+		REQUIRE(DefResult["moduleId"].AsString() == "def"sv);
+		DefPort = DefResult["port"].AsUInt16(0);
+		REQUIRE_NE(DefPort, 0);
 
-			HttpClient DefClient(fmt::format("http://localhost:{}", DefPort));
+		// This should contain the content from the previous two runs
 
-			Result = DefClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567");
-			CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
+		HttpClient DefClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
+		CHECK(DefClient.Get("/health/"));
 
-			CHECK_EQ(Result.AsText(), "AbcDef"sv);
+		Result = DefClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567");
+		CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
 
-			Result = DefClient.Get("/z$/ns1/b/1123456789abcdef0123456789abcdef01234567");
-			CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
+		CHECK_EQ(Result.AsText(), "AbcDef"sv);
 
-			CHECK_EQ(Result.AsText(), "GhijklmNop"sv);
-		}
+		Result = DefClient.Get("/z$/ns1/b/1123456789abcdef0123456789abcdef01234567");
+		CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
 
-		Result = Client.Post("modules/abc/deprovision");
-		REQUIRE(Result);
+		CHECK_EQ(Result.AsText(), "GhijklmNop"sv);
+	}
 
-		Result = Client.Post("modules/def/deprovision");
-		REQUIRE(Result);
+	Result = Client.Post("modules/abc/deprovision");
+	REQUIRE(Result);
+	{
+		HttpClient ModClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
+		CHECK(!ModClient.Get("/health/"));
+	}
 
-		// final sanity check that the hub is still responsive and all modules are gone
-		Result = Client.Get("status");
-		REQUIRE(Result);
-		CHECK_EQ(Result.AsObject()["modules"].AsArrayView().Num(), 0u);
+	Result = Client.Post("modules/def/deprovision");
+	REQUIRE(Result);
+	{
+		HttpClient ModClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
+		CHECK(!ModClient.Get("/health/"));
 	}
+
+	// final sanity check that the hub is still responsive and all modules are gone
+	Result = Client.Get("status");
+	REQUIRE(Result);
+	CHECK_EQ(Result.AsObject()["modules"].AsArrayView().Num(), 0u);
 }
 
 static bool
@@ -339,7 +361,7 @@ TEST_CASE("hub.consul.hub.registration.token")
 		   "--consul-token-env=ZEN_TEST_CONSUL_TOKEN");
 	REQUIRE(PortNumber != 0);
 
-	// Use a plain client — dev-mode Consul doesn't enforce ACLs, but the
+	// Use a plain client -- dev-mode Consul doesn't enforce ACLs, but the
 	// server has exercised the ConsulTokenEnv -> GetEnvVariable -> ConsulClient path.
 	consul::ConsulClient Client("http://localhost:8500/");
 
@@ -366,7 +388,7 @@ TEST_CASE("hub.consul.provision.registration")
 
 	REQUIRE(WaitForConsulService(Client, "zen-hub-test-instance", true, 5000));
 
-	HttpClient HubClient(Instance.GetBaseUri() + "/hub/");
+	HttpClient HubClient(Instance.GetBaseUri() + "/hub/", kFastTimeout);
 
 	HttpClient::Response Result = HubClient.Post("modules/testmod/provision");
 	REQUIRE(Result);
@@ -376,6 +398,11 @@ TEST_CASE("hub.consul.provision.registration")
 		const uint16_t ModulePort = Result.AsObject()["port"].AsUInt16(0);
 		REQUIRE(ModulePort != 0);
 
+		{
+			HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
+			CHECK(ModClient.Get("/health/"));
+		}
+
 		std::string		JsonError;
 		CbFieldIterator ServicesRoot = LoadCompactBinaryFromJson(Client.GetAgentServicesJson(), JsonError);
 		REQUIRE(JsonError.empty());
@@ -427,10 +454,15 @@ TEST_CASE("hub.consul.provision.registration")
 			CHECK_EQ(HubService["Service"sv].AsString(), "zen-hub"sv);
 			CHECK_EQ(HubService["Port"sv].AsDouble(0), double(PortNumber));
 		}
-	}
 
-	Result = HubClient.Post("modules/testmod/deprovision");
-	REQUIRE(Result);
+		Result = HubClient.Post("modules/testmod/deprovision");
+		REQUIRE(Result);
+
+		{
+			HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
+			CHECK(!ModClient.Get("/health/"));
+		}
+	}
 
 	CHECK(!Client.HasService("testmod"));
 
@@ -439,6 +471,129 @@ TEST_CASE("hub.consul.provision.registration")
 	ConsulProc.StopConsulAgent();
 }
 
+TEST_CASE("hub.hibernate.lifecycle")
+{
+	ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kHubServer);
+	const uint16_t	  PortNumber = Instance.SpawnServerAndWaitUntilReady("--hub-instance-corelimit=2 --hub-instance-http-threads=6");
+	REQUIRE(PortNumber != 0);
+
+	HttpClient Client(Instance.GetBaseUri() + "/hub/", kFastTimeout);
+
+	// Provision
+	HttpClient::Response Result = Client.Post("modules/testmod/provision");
+	REQUIRE(Result);
+	CHECK_EQ(Result.AsObject()["moduleId"].AsString(), "testmod"sv);
+	const uint16_t ModulePort = Result.AsObject()["port"].AsUInt16(0);
+	REQUIRE_NE(ModulePort, 0);
+
+	Result = Client.Get("modules/testmod");
+	REQUIRE(Result);
+	CHECK_EQ(Result.AsObject()["state"].AsString(), "provisioned"sv);
+	{
+		HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
+		CHECK(ModClient.Get("/health/"));
+
+		// Write data to verify it survives the hibernate/wake cycle
+		Result = ModClient.Put("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567",
+							   IoBufferBuilder::MakeFromMemory(MakeMemoryView("hibernatetest"sv)));
+		CHECK_EQ(Result.StatusCode, HttpResponseCode::Created);
+	}
+
+	// Hibernate - state should become "hibernated", server should be unreachable
+	Result = Client.Post("modules/testmod/hibernate");
+	REQUIRE(Result);
+	CHECK_EQ(Result.AsObject()["moduleId"].AsString(), "testmod"sv);
+
+	Result = Client.Get("modules/testmod");
+	REQUIRE(Result);
+	CHECK_EQ(Result.AsObject()["state"].AsString(), "hibernated"sv);
+	{
+		HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
+		CHECK(!ModClient.Get("/health/"));
+	}
+
+	// Wake - state should return to "provisioned", server should be reachable, data should be intact
+	Result = Client.Post("modules/testmod/wake");
+	REQUIRE(Result);
+	CHECK_EQ(Result.AsObject()["moduleId"].AsString(), "testmod"sv);
+
+	Result = Client.Get("modules/testmod");
+	REQUIRE(Result);
+	CHECK_EQ(Result.AsObject()["state"].AsString(), "provisioned"sv);
+	{
+		HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
+		CHECK(ModClient.Get("/health/"));
+
+		Result = ModClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567");
+		CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
+		CHECK_EQ(Result.AsText(), "hibernatetest"sv);
+	}
+
+	// Deprovision - server should become unreachable
+	Result = Client.Post("modules/testmod/deprovision");
+	REQUIRE(Result);
+	{
+		HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
+		CHECK(!ModClient.Get("/health/"));
+	}
+
+	// Re-provision - server should be reachable on its (potentially new) port
+	Result = Client.Post("modules/testmod/provision");
+	REQUIRE(Result);
+	CHECK_EQ(Result.AsObject()["moduleId"].AsString(), "testmod"sv);
+	const uint16_t ModulePort2 = Result.AsObject()["port"].AsUInt16(0);
+	REQUIRE_NE(ModulePort2, 0);
+	{
+		HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort2), kFastTimeout);
+		CHECK(ModClient.Get("/health/"));
+	}
+
+	// Final deprovision - server should become unreachable
+	Result = Client.Post("modules/testmod/deprovision");
+	REQUIRE(Result);
+	{
+		HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort2), kFastTimeout);
+		CHECK(!ModClient.Get("/health/"));
+	}
+}
+
+TEST_CASE("hub.hibernate.errors")
+{
+	ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kHubServer);
+	const uint16_t	  PortNumber = Instance.SpawnServerAndWaitUntilReady("--hub-instance-corelimit=2 --hub-instance-http-threads=6");
+	REQUIRE(PortNumber != 0);
+
+	HttpClient Client(Instance.GetBaseUri() + "/hub/", kFastTimeout);
+
+	// Hibernate/wake on an unknown module id should return 404
+	HttpClient::Response Result = Client.Post("modules/unknown/hibernate");
+	CHECK(!Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound);
+
+	Result = Client.Post("modules/unknown/wake");
+	CHECK(!Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound);
+
+	// Double-hibernate: first call succeeds, second returns 400 (wrong state)
+	Result = Client.Post("modules/errmod/provision");
+	REQUIRE(Result);
+
+	Result = Client.Post("modules/errmod/hibernate");
+	REQUIRE(Result);
+
+	Result = Client.Post("modules/errmod/hibernate");
+	CHECK(!Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::BadRequest);
+
+	// Wake on provisioned: succeeds (state restored), then waking again returns 400
+	Result = Client.Post("modules/errmod/wake");
+	REQUIRE(Result);
+
+	Result = Client.Post("modules/errmod/wake");
+	CHECK(!Result);
+	CHECK_EQ(Result.StatusCode, HttpResponseCode::BadRequest);
+}
+
 TEST_SUITE_END();
 
 }  // namespace zen::tests::hub