aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver-test/hub-tests.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenserver-test/hub-tests.cpp')
-rw-r--r--src/zenserver-test/hub-tests.cpp322
1 files changed, 279 insertions, 43 deletions
diff --git a/src/zenserver-test/hub-tests.cpp b/src/zenserver-test/hub-tests.cpp
index dbe6fa785..b2da552fc 100644
--- a/src/zenserver-test/hub-tests.cpp
+++ b/src/zenserver-test/hub-tests.cpp
@@ -33,6 +33,77 @@ using namespace std::literals;
static const HttpClientSettings kFastTimeout{.ConnectTimeout = std::chrono::milliseconds(200)};
+static bool
+WaitForModuleState(HttpClient& Client, std::string_view ModuleId, std::string_view ExpectedState, int TimeoutMs = 10000)
+{
+ Stopwatch Timer;
+ while (Timer.GetElapsedTimeMs() < static_cast<uint64_t>(TimeoutMs))
+ {
+ HttpClient::Response R = Client.Get(fmt::format("modules/{}", ModuleId));
+ if (R && R.AsObject()["state"].AsString() == ExpectedState)
+ {
+ return true;
+ }
+ Sleep(100);
+ }
+ HttpClient::Response R = Client.Get(fmt::format("modules/{}", ModuleId));
+ return R && R.AsObject()["state"].AsString() == ExpectedState;
+}
+
+// Provision a module, retrying on 409 Conflict to handle the window where an async
+// deprovision has removed the module from InstanceLookup but not yet from
+// DeprovisioningModules (which CanProvisionInstance checks).
+static HttpClient::Response
+ProvisionModule(HttpClient& Client, std::string_view ModuleId, int TimeoutMs = 10000)
+{
+ Stopwatch Timer;
+ HttpClient::Response Result;
+ do
+ {
+ Result = Client.Post(fmt::format("modules/{}/provision", ModuleId));
+ if (Result || Result.StatusCode != HttpResponseCode::Conflict)
+ {
+ return Result;
+ }
+ Sleep(100);
+ } while (Timer.GetElapsedTimeMs() < static_cast<uint64_t>(TimeoutMs));
+ return Result;
+}
+
+// Wait for a port to stop accepting connections (i.e. the process has terminated).
+// Needed after async deprovision: WaitForModuleGone returns as soon as the module
+// leaves m_InstanceLookup (synchronous), but the background worker that kills the
+// process may not have run yet.
+static bool
+WaitForPortUnreachable(HttpClient& Client, std::string_view Path = "/health/", int TimeoutMs = 10000)
+{
+ Stopwatch Timer;
+ while (Timer.GetElapsedTimeMs() < static_cast<uint64_t>(TimeoutMs))
+ {
+ if (!Client.Get(Path))
+ {
+ return true;
+ }
+ Sleep(100);
+ }
+ return !Client.Get(Path);
+}
+
+static bool
+WaitForModuleGone(HttpClient& Client, std::string_view ModuleId, int TimeoutMs = 10000)
+{
+ Stopwatch Timer;
+ while (Timer.GetElapsedTimeMs() < static_cast<uint64_t>(TimeoutMs))
+ {
+ if (Client.Get(fmt::format("modules/{}", ModuleId)).StatusCode == HttpResponseCode::NotFound)
+ {
+ return true;
+ }
+ Sleep(100);
+ }
+ return Client.Get(fmt::format("modules/{}", ModuleId)).StatusCode == HttpResponseCode::NotFound;
+}
+
TEST_SUITE_BEGIN("server.hub");
TEST_CASE("hub.lifecycle.children")
@@ -65,9 +136,7 @@ TEST_CASE("hub.lifecycle.children")
AbcPort = AbcResult["port"].AsUInt16(0);
CHECK_NE(AbcPort, 0);
- Result = Client.Get("modules/abc");
- REQUIRE(Result);
- CHECK_EQ(Result.AsObject()["state"].AsString(), "provisioned"sv);
+ REQUIRE(WaitForModuleState(Client, "abc", "provisioned"));
// This should be a fresh instance with no contents
@@ -91,6 +160,8 @@ TEST_CASE("hub.lifecycle.children")
DefPort = DefResult["port"].AsUInt16(0);
REQUIRE_NE(DefPort, 0);
+ REQUIRE(WaitForModuleState(Client, "def", "provisioned"));
+
// This should be a fresh instance with no contents
HttpClient DefClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
@@ -110,21 +181,24 @@ TEST_CASE("hub.lifecycle.children")
Result = Client.Post("modules/ghi/provision");
REQUIRE(Result);
+ REQUIRE(WaitForModuleState(Client, "ghi", "provisioned"));
// Tear down instances
Result = Client.Post("modules/abc/deprovision");
REQUIRE(Result);
+ REQUIRE(WaitForModuleGone(Client, "abc"));
{
HttpClient ModClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
- CHECK(!ModClient.Get("/health/"));
+ CHECK(WaitForPortUnreachable(ModClient));
}
Result = Client.Post("modules/def/deprovision");
REQUIRE(Result);
+ REQUIRE(WaitForModuleGone(Client, "def"));
{
HttpClient ModClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
- CHECK(!ModClient.Get("/health/"));
+ CHECK(WaitForPortUnreachable(ModClient));
}
Result = Client.Post("modules/ghi/deprovision");
@@ -132,7 +206,7 @@ TEST_CASE("hub.lifecycle.children")
// re-provision to verify that (de)hydration preserved state
{
- Result = Client.Post("modules/abc/provision");
+ Result = ProvisionModule(Client, "abc");
REQUIRE(Result);
CbObject AbcResult = Result.AsObject();
@@ -140,6 +214,8 @@ TEST_CASE("hub.lifecycle.children")
AbcPort = AbcResult["port"].AsUInt16(0);
REQUIRE_NE(AbcPort, 0);
+ REQUIRE(WaitForModuleState(Client, "abc", "provisioned"));
+
// This should contain the content from the previous run
HttpClient AbcClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
@@ -156,7 +232,7 @@ TEST_CASE("hub.lifecycle.children")
}
{
- Result = Client.Post("modules/def/provision");
+ Result = ProvisionModule(Client, "def");
REQUIRE(Result);
CbObject DefResult = Result.AsObject();
@@ -164,6 +240,8 @@ TEST_CASE("hub.lifecycle.children")
DefPort = DefResult["port"].AsUInt16(0);
REQUIRE_NE(DefPort, 0);
+ REQUIRE(WaitForModuleState(Client, "def", "provisioned"));
+
// This should contain the content from the previous run
HttpClient DefClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
@@ -181,22 +259,24 @@ TEST_CASE("hub.lifecycle.children")
Result = Client.Post("modules/abc/deprovision");
REQUIRE(Result);
+ REQUIRE(WaitForModuleGone(Client, "abc"));
{
HttpClient ModClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
- CHECK(!ModClient.Get("/health/"));
+ CHECK(WaitForPortUnreachable(ModClient));
}
Result = Client.Post("modules/def/deprovision");
REQUIRE(Result);
+ REQUIRE(WaitForModuleGone(Client, "def"));
{
HttpClient ModClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
- CHECK(!ModClient.Get("/health/"));
+ CHECK(WaitForPortUnreachable(ModClient));
}
// re-provision to verify that (de)hydration preserved state, including
// state which was generated after the very first dehydration
{
- Result = Client.Post("modules/abc/provision");
+ Result = ProvisionModule(Client, "abc");
REQUIRE(Result);
CbObject AbcResult = Result.AsObject();
@@ -204,6 +284,8 @@ TEST_CASE("hub.lifecycle.children")
AbcPort = AbcResult["port"].AsUInt16(0);
REQUIRE_NE(AbcPort, 0);
+ REQUIRE(WaitForModuleState(Client, "abc", "provisioned"));
+
// This should contain the content from the previous two runs
HttpClient AbcClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
@@ -221,7 +303,7 @@ TEST_CASE("hub.lifecycle.children")
}
{
- Result = Client.Post("modules/def/provision");
+ Result = ProvisionModule(Client, "def");
REQUIRE(Result);
CbObject DefResult = Result.AsObject();
@@ -229,6 +311,8 @@ TEST_CASE("hub.lifecycle.children")
DefPort = DefResult["port"].AsUInt16(0);
REQUIRE_NE(DefPort, 0);
+ REQUIRE(WaitForModuleState(Client, "def", "provisioned"));
+
// This should contain the content from the previous two runs
HttpClient DefClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
@@ -247,16 +331,18 @@ TEST_CASE("hub.lifecycle.children")
Result = Client.Post("modules/abc/deprovision");
REQUIRE(Result);
+ REQUIRE(WaitForModuleGone(Client, "abc"));
{
HttpClient ModClient(fmt::format("http://localhost:{}", AbcPort), kFastTimeout);
- CHECK(!ModClient.Get("/health/"));
+ CHECK(WaitForPortUnreachable(ModClient));
}
Result = Client.Post("modules/def/deprovision");
REQUIRE(Result);
+ REQUIRE(WaitForModuleGone(Client, "def"));
{
HttpClient ModClient(fmt::format("http://localhost:{}", DefPort), kFastTimeout);
- CHECK(!ModClient.Get("/health/"));
+ CHECK(WaitForPortUnreachable(ModClient));
}
// final sanity check that the hub is still responsive and all modules are gone
@@ -308,14 +394,45 @@ TEST_CASE("hub.consul.hub.registration")
ConsulProc.SpawnConsulAgent();
ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kHubServer);
- const uint16_t PortNumber =
- Instance.SpawnServerAndWaitUntilReady("--consul-endpoint=http://localhost:8500/ --instance-id=test-instance");
+ const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(
+ "--consul-endpoint=http://localhost:8500/ --instance-id=test-instance "
+ "--consul-health-interval-seconds=5 --consul-deregister-after-seconds=60");
REQUIRE(PortNumber != 0);
consul::ConsulClient Client("http://localhost:8500/");
-
REQUIRE(WaitForConsulService(Client, "zen-hub-test-instance", true, 5000));
+ // Verify custom intervals flowed through to the registered check
+ {
+ std::string JsonError;
+ CbFieldIterator ChecksRoot = LoadCompactBinaryFromJson(Client.GetAgentChecksJson(), JsonError);
+ REQUIRE(JsonError.empty());
+
+ CbObjectView HubCheck;
+ for (CbFieldView F : ChecksRoot)
+ {
+ if (!F.IsObject())
+ {
+ continue;
+ }
+ for (CbFieldView C : F.AsObjectView())
+ {
+ CbObjectView Check = C.AsObjectView();
+ if (Check["ServiceID"sv].AsString() == "zen-hub-test-instance"sv)
+ {
+ HubCheck = Check;
+ break;
+ }
+ }
+ }
+ REQUIRE(HubCheck);
+ // Consul does not reflect DeregisterCriticalServiceAfter back in /v1/agent/checks for
+ // service-embedded checks; Definition is always an empty object. Only Type and Interval
+ // are accessible at the top level.
+ CHECK_EQ(HubCheck["Type"sv].AsString(), "http"sv);
+ CHECK_EQ(HubCheck["Interval"sv].AsString(), "5s"sv);
+ }
+
Instance.Shutdown();
CHECK(!Client.HasService("zen-hub-test-instance"));
@@ -393,16 +510,15 @@ TEST_CASE("hub.consul.provision.registration")
HttpClient::Response Result = HubClient.Post("modules/testmod/provision");
REQUIRE(Result);
- CHECK(Client.HasService("testmod"));
- {
- const uint16_t ModulePort = Result.AsObject()["port"].AsUInt16(0);
- REQUIRE(ModulePort != 0);
+ // Service is registered in Consul during Provisioning (before the child process starts),
+ // so this returns as soon as the state transition fires, not when the server is ready.
+ REQUIRE(WaitForConsulService(Client, "testmod", true, 10000));
- {
- HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
- CHECK(ModClient.Get("/health/"));
- }
+ const uint16_t ModulePort = Result.AsObject()["port"].AsUInt16(0);
+ REQUIRE(ModulePort != 0);
+ // Consul fields are set during Provisioning and can be verified before the server is ready.
+ {
std::string JsonError;
CbFieldIterator ServicesRoot = LoadCompactBinaryFromJson(Client.GetAgentServicesJson(), JsonError);
REQUIRE(JsonError.empty());
@@ -417,7 +533,7 @@ TEST_CASE("hub.consul.provision.registration")
}
REQUIRE(ServicesMap);
- // Verify fields registered by OnProvisioned
+ // Verify fields registered by OnModuleStateChanged
{
CbObjectView ModService = ServicesMap["testmod"].AsObjectView();
CHECK_EQ(ModService["ID"sv].AsString(), "testmod"sv);
@@ -455,8 +571,75 @@ TEST_CASE("hub.consul.provision.registration")
CHECK_EQ(HubService["Port"sv].AsDouble(0), double(PortNumber));
}
+ // Verify hub health check endpoint URL (registered from startup with an active interval)
+ {
+ std::string ChecksJsonError;
+ CbFieldIterator ChecksRoot = LoadCompactBinaryFromJson(Client.GetAgentChecksJson(), ChecksJsonError);
+ REQUIRE(ChecksJsonError.empty());
+
+ CbObjectView HubCheck;
+ for (CbFieldView F : ChecksRoot)
+ {
+ if (!F.IsObject())
+ {
+ continue;
+ }
+ for (CbFieldView C : F.AsObjectView())
+ {
+ CbObjectView Check = C.AsObjectView();
+ if (Check["ServiceID"sv].AsString() == "zen-hub-test-instance"sv)
+ {
+ HubCheck = Check;
+ }
+ }
+ }
+ REQUIRE(HubCheck);
+ // Consul does not reflect HTTP URL back in /v1/agent/checks for service-embedded checks.
+ CHECK_EQ(HubCheck["Type"sv].AsString(), "http"sv);
+ }
+ }
+
+ // Wait for Provisioned before touching the module's HTTP endpoint.
+ REQUIRE(WaitForModuleState(HubClient, "testmod", "provisioned"));
+
+ // Verify module health check endpoint URL. No health check is registered during Provisioning
+ // (to avoid Consul marking the service critical before the child process is ready); it is added
+ // on transition to Provisioned.
+ {
+ std::string ChecksJsonError;
+ CbFieldIterator ChecksRoot = LoadCompactBinaryFromJson(Client.GetAgentChecksJson(), ChecksJsonError);
+ REQUIRE(ChecksJsonError.empty());
+
+ CbObjectView ModCheck;
+ for (CbFieldView F : ChecksRoot)
+ {
+ if (!F.IsObject())
+ {
+ continue;
+ }
+ for (CbFieldView C : F.AsObjectView())
+ {
+ CbObjectView Check = C.AsObjectView();
+ if (Check["ServiceID"sv].AsString() == "testmod"sv)
+ {
+ ModCheck = Check;
+ }
+ }
+ }
+ REQUIRE(ModCheck);
+ // Consul does not reflect HTTP URL back in /v1/agent/checks for service-embedded checks.
+ CHECK_EQ(ModCheck["Type"sv].AsString(), "http"sv);
+ }
+
+ {
+ HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
+ CHECK(ModClient.Get("/health/"));
+ }
+
+ {
Result = HubClient.Post("modules/testmod/deprovision");
REQUIRE(Result);
+ REQUIRE(WaitForConsulService(Client, "testmod", false, 10000));
{
HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
@@ -482,13 +665,12 @@ TEST_CASE("hub.hibernate.lifecycle")
// Provision
HttpClient::Response Result = Client.Post("modules/testmod/provision");
REQUIRE(Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
CHECK_EQ(Result.AsObject()["moduleId"].AsString(), "testmod"sv);
const uint16_t ModulePort = Result.AsObject()["port"].AsUInt16(0);
REQUIRE_NE(ModulePort, 0);
- Result = Client.Get("modules/testmod");
- REQUIRE(Result);
- CHECK_EQ(Result.AsObject()["state"].AsString(), "provisioned"sv);
+ REQUIRE(WaitForModuleState(Client, "testmod", "provisioned"));
{
HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
CHECK(ModClient.Get("/health/"));
@@ -502,11 +684,10 @@ TEST_CASE("hub.hibernate.lifecycle")
// Hibernate - state should become "hibernated", server should be unreachable
Result = Client.Post("modules/testmod/hibernate");
REQUIRE(Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
CHECK_EQ(Result.AsObject()["moduleId"].AsString(), "testmod"sv);
- Result = Client.Get("modules/testmod");
- REQUIRE(Result);
- CHECK_EQ(Result.AsObject()["state"].AsString(), "hibernated"sv);
+ REQUIRE(WaitForModuleState(Client, "testmod", "hibernated"));
{
HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
CHECK(!ModClient.Get("/health/"));
@@ -515,11 +696,10 @@ TEST_CASE("hub.hibernate.lifecycle")
// Wake - state should return to "provisioned", server should be reachable, data should be intact
Result = Client.Post("modules/testmod/wake");
REQUIRE(Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
CHECK_EQ(Result.AsObject()["moduleId"].AsString(), "testmod"sv);
- Result = Client.Get("modules/testmod");
- REQUIRE(Result);
- CHECK_EQ(Result.AsObject()["state"].AsString(), "provisioned"sv);
+ REQUIRE(WaitForModuleState(Client, "testmod", "provisioned"));
{
HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
CHECK(ModClient.Get("/health/"));
@@ -532,17 +712,20 @@ TEST_CASE("hub.hibernate.lifecycle")
// Deprovision - server should become unreachable
Result = Client.Post("modules/testmod/deprovision");
REQUIRE(Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
+ REQUIRE(WaitForModuleGone(Client, "testmod"));
{
HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort), kFastTimeout);
- CHECK(!ModClient.Get("/health/"));
+ CHECK(WaitForPortUnreachable(ModClient));
}
// Re-provision - server should be reachable on its (potentially new) port
- Result = Client.Post("modules/testmod/provision");
+ Result = ProvisionModule(Client, "testmod");
REQUIRE(Result);
CHECK_EQ(Result.AsObject()["moduleId"].AsString(), "testmod"sv);
const uint16_t ModulePort2 = Result.AsObject()["port"].AsUInt16(0);
REQUIRE_NE(ModulePort2, 0);
+ REQUIRE(WaitForModuleState(Client, "testmod", "provisioned"));
{
HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort2), kFastTimeout);
CHECK(ModClient.Get("/health/"));
@@ -551,9 +734,10 @@ TEST_CASE("hub.hibernate.lifecycle")
// Final deprovision - server should become unreachable
Result = Client.Post("modules/testmod/deprovision");
REQUIRE(Result);
+ REQUIRE(WaitForModuleGone(Client, "testmod"));
{
HttpClient ModClient(fmt::format("http://localhost:{}", ModulePort2), kFastTimeout);
- CHECK(!ModClient.Get("/health/"));
+ CHECK(WaitForPortUnreachable(ModClient));
}
}
@@ -574,24 +758,76 @@ TEST_CASE("hub.hibernate.errors")
CHECK(!Result);
CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound);
- // Double-hibernate: first call succeeds, second returns 400 (wrong state)
+ Result = Client.Post("modules/unknown/deprovision");
+ CHECK(!Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound);
+
+ Result = Client.Delete("modules/unknown");
+ CHECK(!Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound);
+
+ // Double-provision: second call while first is in-flight returns 202 Accepted with the same port.
Result = Client.Post("modules/errmod/provision");
REQUIRE(Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
+ const uint16_t ErrmodPort = Result.AsObject()["port"].AsUInt16(0);
+ REQUIRE_NE(ErrmodPort, 0);
+ // Provisioning the same module while in-flight returns 202 Accepted with the allocated port.
+ // Evaluated synchronously before WorkerPool dispatch, so safe regardless of timing.
+ Result = Client.Post("modules/errmod/provision");
+ CHECK(Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
+ CHECK_EQ(Result.AsObject()["port"].AsUInt16(0), ErrmodPort);
+
+ REQUIRE(WaitForModuleState(Client, "errmod", "provisioned"));
+
+ // Already provisioned: provision and wake both return 200 Completed.
+ Result = Client.Post("modules/errmod/provision");
+ CHECK(Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
+
+ Result = Client.Post("modules/errmod/wake");
+ CHECK(Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
+
+ // Double-hibernate: second call while first is in-flight returns 202 Accepted.
Result = Client.Post("modules/errmod/hibernate");
REQUIRE(Result);
Result = Client.Post("modules/errmod/hibernate");
- CHECK(!Result);
- CHECK_EQ(Result.StatusCode, HttpResponseCode::BadRequest);
+ CHECK(Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
+
+ REQUIRE(WaitForModuleState(Client, "errmod", "hibernated"));
- // Wake on provisioned: succeeds (state restored), then waking again returns 400
+ // Already hibernated: hibernate returns 200 Completed.
+ Result = Client.Post("modules/errmod/hibernate");
+ CHECK(Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::OK);
+
+ // Double-wake: second call while first is in-flight returns 202 Accepted.
Result = Client.Post("modules/errmod/wake");
REQUIRE(Result);
Result = Client.Post("modules/errmod/wake");
- CHECK(!Result);
- CHECK_EQ(Result.StatusCode, HttpResponseCode::BadRequest);
+ CHECK(Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
+
+ // Double-deprovision: second call while first is in-flight returns 202 Accepted.
+ // errmod2 is a fresh module to avoid waiting on the still-waking errmod.
+ Result = Client.Post("modules/errmod2/provision");
+ REQUIRE(Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
+ REQUIRE(WaitForModuleState(Client, "errmod2", "provisioned"));
+
+ Result = Client.Post("modules/errmod2/deprovision");
+ REQUIRE(Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
+
+ Result = Client.Post("modules/errmod2/deprovision");
+ CHECK(Result);
+ CHECK_EQ(Result.StatusCode, HttpResponseCode::Accepted);
}
TEST_SUITE_END();
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-28 22:03:32 +0000