hub instance state refactor (#892)

- Improvement: Provisioning a hibernated instance now automatically wakes it instead of requiring an explicit wake call first
- Improvement: Deprovisioning now accepts instances in Crashed or Hibernated states, not just Provisioned
- Improvement: Added `--consul-health-interval-seconds` and `--consul-deregister-after-seconds` options to control Consul health check behavior (defaults: 10s and 30s)
- Improvement: Consul registration now occurs when provisioning starts; health check intervals are applied once provisioning completes

1 files changed, 107 insertions, 11 deletions

diff --git a/src/zenserver-test/hub-tests.cpp b/src/zenserver-test/hub-tests.cpp
index f86bdc5c7..b2da552fc 100644
--- a/src/zenserver-test/hub-tests.cpp
+++ b/src/zenserver-test/hub-tests.cpp
@@ -394,14 +394,45 @@ TEST_CASE("hub.consul.hub.registration")
 	ConsulProc.SpawnConsulAgent();
 
 	ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kHubServer);
-	const uint16_t	  PortNumber =
-		Instance.SpawnServerAndWaitUntilReady("--consul-endpoint=http://localhost:8500/ --instance-id=test-instance");
+	const uint16_t	  PortNumber = Instance.SpawnServerAndWaitUntilReady(
+		   "--consul-endpoint=http://localhost:8500/ --instance-id=test-instance "
+		   "--consul-health-interval-seconds=5 --consul-deregister-after-seconds=60");
 	REQUIRE(PortNumber != 0);
 
 	consul::ConsulClient Client("http://localhost:8500/");
-
 	REQUIRE(WaitForConsulService(Client, "zen-hub-test-instance", true, 5000));
 
+	// Verify custom intervals flowed through to the registered check
+	{
+		std::string		JsonError;
+		CbFieldIterator ChecksRoot = LoadCompactBinaryFromJson(Client.GetAgentChecksJson(), JsonError);
+		REQUIRE(JsonError.empty());
+
+		CbObjectView HubCheck;
+		for (CbFieldView F : ChecksRoot)
+		{
+			if (!F.IsObject())
+			{
+				continue;
+			}
+			for (CbFieldView C : F.AsObjectView())
+			{
+				CbObjectView Check = C.AsObjectView();
+				if (Check["ServiceID"sv].AsString() == "zen-hub-test-instance"sv)
+				{
+					HubCheck = Check;
+					break;
+				}
+			}
+		}
+		REQUIRE(HubCheck);
+		// Consul does not reflect DeregisterCriticalServiceAfter back in /v1/agent/checks for
+		// service-embedded checks; Definition is always an empty object. Only Type and Interval
+		// are accessible at the top level.
+		CHECK_EQ(HubCheck["Type"sv].AsString(), "http"sv);
+		CHECK_EQ(HubCheck["Interval"sv].AsString(), "5s"sv);
+	}
+
 	Instance.Shutdown();
 
 	CHECK(!Client.HasService("zen-hub-test-instance"));
@@ -479,16 +510,15 @@ TEST_CASE("hub.consul.provision.registration")
 	HttpClient::Response Result = HubClient.Post("modules/testmod/provision");
 	REQUIRE(Result);
 
+	// Service is registered in Consul during Provisioning (before the child process starts),
+	// so this returns as soon as the state transition fires, not when the server is ready.
 	REQUIRE(WaitForConsulService(Client, "testmod", true, 10000));
-	{
-		const uint16_t ModulePort = Result.AsObject()["port"].AsUInt16(0);
-		REQUIRE(ModulePort != 0);
 
-		{
-			HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
-			CHECK(ModClient.Get("/health/"));
-		}
+	const uint16_t ModulePort = Result.AsObject()["port"].AsUInt16(0);
+	REQUIRE(ModulePort != 0);
 
+	// Consul fields are set during Provisioning and can be verified before the server is ready.
+	{
 		std::string		JsonError;
 		CbFieldIterator ServicesRoot = LoadCompactBinaryFromJson(Client.GetAgentServicesJson(), JsonError);
 		REQUIRE(JsonError.empty());
@@ -503,7 +533,7 @@ TEST_CASE("hub.consul.provision.registration")
 		}
 		REQUIRE(ServicesMap);
 
-		// Verify fields registered by OnProvisioned
+		// Verify fields registered by OnModuleStateChanged
 		{
 			CbObjectView ModService = ServicesMap["testmod"].AsObjectView();
 			CHECK_EQ(ModService["ID"sv].AsString(), "testmod"sv);
@@ -541,6 +571,72 @@ TEST_CASE("hub.consul.provision.registration")
 			CHECK_EQ(HubService["Port"sv].AsDouble(0), double(PortNumber));
 		}
 
+		// Verify hub health check endpoint URL (registered from startup with an active interval)
+		{
+			std::string		ChecksJsonError;
+			CbFieldIterator ChecksRoot = LoadCompactBinaryFromJson(Client.GetAgentChecksJson(), ChecksJsonError);
+			REQUIRE(ChecksJsonError.empty());
+
+			CbObjectView HubCheck;
+			for (CbFieldView F : ChecksRoot)
+			{
+				if (!F.IsObject())
+				{
+					continue;
+				}
+				for (CbFieldView C : F.AsObjectView())
+				{
+					CbObjectView Check = C.AsObjectView();
+					if (Check["ServiceID"sv].AsString() == "zen-hub-test-instance"sv)
+					{
+						HubCheck = Check;
+					}
+				}
+			}
+			REQUIRE(HubCheck);
+			// Consul does not reflect HTTP URL back in /v1/agent/checks for service-embedded checks.
+			CHECK_EQ(HubCheck["Type"sv].AsString(), "http"sv);
+		}
+	}
+
+	// Wait for Provisioned before touching the module's HTTP endpoint.
+	REQUIRE(WaitForModuleState(HubClient, "testmod", "provisioned"));
+
+	// Verify module health check endpoint URL. No health check is registered during Provisioning
+	// (to avoid Consul marking the service critical before the child process is ready); it is added
+	// on transition to Provisioned.
+	{
+		std::string		ChecksJsonError;
+		CbFieldIterator ChecksRoot = LoadCompactBinaryFromJson(Client.GetAgentChecksJson(), ChecksJsonError);
+		REQUIRE(ChecksJsonError.empty());
+
+		CbObjectView ModCheck;
+		for (CbFieldView F : ChecksRoot)
+		{
+			if (!F.IsObject())
+			{
+				continue;
+			}
+			for (CbFieldView C : F.AsObjectView())
+			{
+				CbObjectView Check = C.AsObjectView();
+				if (Check["ServiceID"sv].AsString() == "testmod"sv)
+				{
+					ModCheck = Check;
+				}
+			}
+		}
+		REQUIRE(ModCheck);
+		// Consul does not reflect HTTP URL back in /v1/agent/checks for service-embedded checks.
+		CHECK_EQ(ModCheck["Type"sv].AsString(), "http"sv);
+	}
+
+	{
+		HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
+		CHECK(ModClient.Get("/health/"));
+	}
+
+	{
 		Result = HubClient.Post("modules/testmod/deprovision");
 		REQUIRE(Result);
 		REQUIRE(WaitForConsulService(Client, "testmod", false, 10000));