diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 168817f9d..1707b245a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,10 @@ jobs: os_image: ubuntu:resolute container: image: ${{ matrix.os_image }} - timeout-minutes: 60 + # Must stay above the build plus the test step's own budget below, or the job cap kills the + # run before that step's cap can - which loses the "which step ran long" answer. The build + # side of this job is around 18 minutes. + timeout-minutes: 90 defaults: run: shell: bash @@ -86,10 +89,13 @@ jobs: ./scripts/ccache_report.sh "${{ matrix.ros_distro }}" - name: Run unit and integration tests - # The suite grew past the old 15-minute budget: on lyrical it was already taking - # 14m13s on main, so any new package tipped it over. This is real work finishing, not - # a hang. - timeout-minutes: 25 + # The suite keeps outgrowing this budget as packages land, and every overrun so far has + # been real work finishing rather than a hang: 15 minutes was raised to 25 when lyrical + # reached 14m13s, and 25 was reached again once the graph watchdog's end-to-end scenarios + # arrived (lyrical 22m50s before them, jazzy 28m15s after). The jazzy job below caps the + # whole job instead of this step, so only these two distros can be killed mid-package - + # which reads as a test failure and hides every test the kill cut off. + timeout-minutes: 45 env: # FastRTPS 2.6 on Humble has a known use-after-free in the # discovery-teardown path (EDP::unpairWriterProxy) that segfaults diff --git a/docs/design/index.rst b/docs/design/index.rst index 29ab9016f..da8c5d3a0 100644 --- a/docs/design/index.rst +++ b/docs/design/index.rst @@ -14,6 +14,7 @@ This section contains design documentation for the ros2_medkit project packages. ros2_medkit_fault_reporter/index ros2_medkit_gateway/index ros2_medkit_graph_provider/index + ros2_medkit_graph_watchdog/graph_watchdog ros2_medkit_integration_tests/index ros2_medkit_linux_introspection/index ros2_medkit_msgs/index diff --git a/docs/design/ros2_medkit_graph_watchdog b/docs/design/ros2_medkit_graph_watchdog new file mode 120000 index 000000000..09b35af13 --- /dev/null +++ b/docs/design/ros2_medkit_graph_watchdog @@ -0,0 +1 @@ +../../src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design \ No newline at end of file diff --git a/src/ros2_medkit_integration_tests/CMakeLists.txt b/src/ros2_medkit_integration_tests/CMakeLists.txt index 669b2c199..ce57c6baf 100644 --- a/src/ros2_medkit_integration_tests/CMakeLists.txt +++ b/src/ros2_medkit_integration_tests/CMakeLists.txt @@ -34,6 +34,7 @@ find_package(example_interfaces REQUIRED) find_package(ros2_medkit_msgs REQUIRED) find_package(diagnostic_msgs REQUIRED) find_package(rclcpp_lifecycle REQUIRED) +find_package(lifecycle_msgs REQUIRED) find_package(ros2_medkit_gateway REQUIRED) # === Demo node executables === @@ -93,6 +94,13 @@ medkit_target_dependencies(demo_param_beacon_node rclcpp) add_executable(managed_lifecycle demo_nodes/managed_lifecycle_node.cpp) medkit_target_dependencies(managed_lifecycle rclcpp rclcpp_lifecycle) +# Proof fixture for GRAPH_NODE_UNREADABLE (see the file doc): a plain rclcpp::Node that +# advertises get_state/change_state like managed_lifecycle above, but whose get_state +# never answers until its start_answering parameter is flipped at runtime. +add_executable(unreadable_lifecycle demo_nodes/unreadable_lifecycle_node.cpp) +target_include_directories(unreadable_lifecycle PRIVATE ${_demo_include_dir}) +medkit_target_dependencies(unreadable_lifecycle rclcpp lifecycle_msgs) + add_executable(demo_unresponsive_param_node demo_nodes/unresponsive_param_node.cpp) target_include_directories(demo_unresponsive_param_node PRIVATE ${_demo_include_dir}) medkit_target_dependencies(demo_unresponsive_param_node rclcpp rcl_interfaces) @@ -111,6 +119,7 @@ install(TARGETS demo_beacon_publisher demo_param_beacon_node managed_lifecycle + unreadable_lifecycle demo_unresponsive_param_node DESTINATION lib/${PROJECT_NAME} ) diff --git a/src/ros2_medkit_integration_tests/demo_nodes/unreadable_lifecycle_node.cpp b/src/ros2_medkit_integration_tests/demo_nodes/unreadable_lifecycle_node.cpp new file mode 100644 index 000000000..c3d067b50 --- /dev/null +++ b/src/ros2_medkit_integration_tests/demo_nodes/unreadable_lifecycle_node.cpp @@ -0,0 +1,194 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file unreadable_lifecycle_node.cpp + * @brief Demo node that LOOKS like a managed lifecycle node but never answers GetState + * + * Proof fixture for GRAPH_NODE_UNREADABLE (ros2_medkit_graph_watchdog's + * lifecycle_expectation detector): a required node whose lifecycle state can never be + * read must be reported UNREADABLE, not silently folded into GRAPH_NODE_INACTIVE. No + * existing demo node can produce that state - managed_lifecycle_node.cpp is a REAL + * rclcpp_lifecycle::LifecycleNode, and a real one always answers GetState immediately. + * + * `find_lifecycle_get_state_path()` + * (ros2_medkit_gateway/core/status/lifecycle_state_reader.hpp) does not check that a + * node IS an rclcpp_lifecycle::LifecycleNode; it matches purely on discovered SERVICE + * TYPE: an App counts as "managed" once its services include one of type + * `lifecycle_msgs/srv/GetState` and one of type `lifecycle_msgs/srv/ChangeState` + * (see test_lifecycle_status_helpers.cpp's FindPathRequiresBothServices). So a plain + * rclcpp::Node that advertises both service types under the conventional `~/get_state` + * / `~/change_state` names is indistinguishable from a real LifecycleNode to the + * gateway's discovery layer, while its GetState handler simply never responds. + * + * GetState is registered with rclcpp's DEFERRED-RESPONSE service callback signature - + * `void(std::shared_ptr, std::shared_ptr)`, no response + * parameter - rather than the ordinary `(request, response)` or + * `(header, request, response)` forms. Both of those always allocate a response and + * `Service::handle_request()` sends it unconditionally once the callback returns + * (see rclcpp/any_service_callback.hpp's `dispatch()` and rclcpp/service.hpp's + * `handle_request()`), so neither form could ever leave a request unanswered. The + * deferred form's `dispatch()` returns nullptr instead, so `handle_request()` sends + * nothing - the request stays open until something calls `Service::send_response()` + * on it directly, which this fixture does only from `answer_all_pending()` below. + * Every request received while `answering_` is false is stored (never answered) in + * `pending_`, deliberately LEAKED for the lifetime of this process: it is a demo-only + * fixture that lives for one CTest run, so an unbounded queue of never-answered + * `rmw_request_id_t`/Request pairs is memory this process's exit reclaims wholesale, + * not a leak in a long-running system. In practice the queue never grows past a + * handful of entries anyway - LifecycleWatcher (lifecycle_watcher.cpp) seeds a newly + * discovered node once and re-seeds a non-active one at most `kReseedAttempts` (2) + * further times, then stops calling GetState on it at all. + * + * ChangeState answers normally (SUCCESS always): only GetState needs to hang, and this + * fixture is never driven through a lifecycle transition by anything that reads its + * result. + * + * `start_answering` (bool parameter, default false) is how the e2e proves the CLEAR + * half of the story: setting it true - a real `ros2 param set` / + * `rcl_interfaces/srv/SetParameters` call against this node's own auto-started + * parameter service - answers every currently-held GetState request with "active", and + * every future one immediately. A parameter rather than a wall timer, so the e2e + * controls exactly WHEN the answer arrives (after it has independently confirmed + * GRAPH_NODE_UNREADABLE actually raised) instead of racing a fixed delay against the + * 60-tick hold's own timing. + * + * Flipping `start_answering` ALSO publishes one `~/transition_event` - not optional + * decoration, load-bearing. LifecycleWatcher (lifecycle_watcher.cpp) spends its GetState + * re-seed budget (the initial seed plus `kReseedAttempts`, 2 more) within the first few + * ticks after discovery, all while this fixture is still deliberately silent, and never + * calls GetState on this node again afterwards - a sustained "" is treated as the + * entry's terminal state. Past that point the ONLY channel that can still move the + * cached label is `~/transition_event`, exactly as it is for a real rclcpp_lifecycle + * node's ACTIVATE. Answering later GetState calls immediately (see `send_active` below) + * is therefore necessary but not sufficient for the watcher to ever notice: nothing + * calls GetState again to collect that answer unless this event arrives first. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "ros2_medkit_integration_tests/demo_node_main.hpp" + +class UnreadableLifecycleNode : public rclcpp::Node { + public: + UnreadableLifecycleNode() : Node("unreadable_lifecycle") { + answering_ = this->declare_parameter("start_answering", false); + + // Reliable + volatile (the QoS default LifecycleWatcher's own subscription + // requires - see its "stay volatile" comment), matching how rclcpp_lifecycle + // itself publishes ~/transition_event. + transition_event_pub_ = this->create_publisher( + "~/transition_event", rclcpp::QoS(rclcpp::KeepLast(10)).reliable()); + + get_state_service_ = this->create_service( + "~/get_state", [this](const std::shared_ptr & header, + const std::shared_ptr & /*request*/) { + // No lock needed: this node spins on a single-threaded executor (see main() + // below), so this callback and the parameter callback that flips + // `answering_` below can never run concurrently with each other. + if (answering_) { + send_active(header); + return; + } + pending_.push_back(header); // deliberately leaked for this process's lifetime - see the file doc + }); + + change_state_service_ = this->create_service( + "~/change_state", [](const std::shared_ptr & /*request*/, + const std::shared_ptr & response) { + // Not exercised by anything: only find_lifecycle_get_state_path()'s type + // check needs this service to exist at all. + response->success = true; + }); + + param_callback_handle_ = + this->add_on_set_parameters_callback([this](const std::vector & params) { + rcl_interfaces::msg::SetParametersResult result; + result.successful = true; + for (const auto & param : params) { + if (param.get_name() == "start_answering" && param.as_bool()) { + answer_all_pending(); + } + } + return result; + }); + + RCLCPP_INFO(get_logger(), + "unreadable_lifecycle started: advertises get_state/change_state like a managed " + "lifecycle node, but get_state never answers until start_answering:=true"); + } + + ~UnreadableLifecycleNode() override { + get_state_service_.reset(); + change_state_service_.reset(); + param_callback_handle_.reset(); + transition_event_pub_.reset(); + } + UnreadableLifecycleNode(const UnreadableLifecycleNode &) = delete; + UnreadableLifecycleNode & operator=(const UnreadableLifecycleNode &) = delete; + UnreadableLifecycleNode(UnreadableLifecycleNode &&) = delete; + UnreadableLifecycleNode & operator=(UnreadableLifecycleNode &&) = delete; + + private: + void send_active(const std::shared_ptr & header) { + lifecycle_msgs::srv::GetState::Response response; + response.current_state.id = lifecycle_msgs::msg::State::PRIMARY_STATE_ACTIVE; + response.current_state.label = "active"; + get_state_service_->send_response(*header, response); + } + + void answer_all_pending() { + if (answering_) { + return; // already flipped - a second start_answering:=true is a no-op, not a re-send + } + answering_ = true; + for (const auto & header : pending_) { + send_active(header); + } + pending_.clear(); + + // The event LifecycleWatcher actually needs - see the file doc's "load-bearing" + // note. start_state is filled in for a realistic message shape; only goal_state + // (read as the new cached label) and start_state (checked only for the + // "errorprocessing" edge, which this is not) are ever consulted by the watcher. + lifecycle_msgs::msg::TransitionEvent event; + event.start_state.id = lifecycle_msgs::msg::State::PRIMARY_STATE_UNCONFIGURED; + event.start_state.label = "unconfigured"; + event.goal_state.id = lifecycle_msgs::msg::State::PRIMARY_STATE_ACTIVE; + event.goal_state.label = "active"; + transition_event_pub_->publish(event); + } + + bool answering_ = false; + std::vector> pending_; + rclcpp::Service::SharedPtr get_state_service_; + rclcpp::Service::SharedPtr change_state_service_; + rclcpp::Publisher::SharedPtr transition_event_pub_; + rclcpp::node_interfaces::OnSetParametersCallbackHandle::SharedPtr param_callback_handle_; +}; + +int main(int argc, char ** argv) { + return ros2_medkit_integration_tests::run_demo_node(argc, argv, [] { + return std::make_shared(); + }); +} diff --git a/src/ros2_medkit_integration_tests/package.xml b/src/ros2_medkit_integration_tests/package.xml index fd5e26937..4509690f8 100644 --- a/src/ros2_medkit_integration_tests/package.xml +++ b/src/ros2_medkit_integration_tests/package.xml @@ -16,6 +16,7 @@ rclcpp rclcpp_action rclcpp_lifecycle + lifecycle_msgs std_msgs std_srvs sensor_msgs diff --git a/src/ros2_medkit_integration_tests/ros2_medkit_test_utils/launch_helpers.py b/src/ros2_medkit_integration_tests/ros2_medkit_test_utils/launch_helpers.py index d1d25f7d8..944ab5ab8 100644 --- a/src/ros2_medkit_integration_tests/ros2_medkit_test_utils/launch_helpers.py +++ b/src/ros2_medkit_integration_tests/ros2_medkit_test_utils/launch_helpers.py @@ -62,6 +62,10 @@ # the "active" lifecycle state (-> status "ready"). Distinct node name so it # can run alongside the unconfigured 'managed_lifecycle' in the same test. 'managed_lifecycle_active': ('managed_lifecycle', 'managed_lifecycle_active', ''), + # Proof fixture for GRAPH_NODE_UNREADABLE: advertises get_state/change_state like a + # managed lifecycle node, but get_state never answers until its start_answering + # parameter is set true (see unreadable_lifecycle_node.cpp's file doc). + 'unreadable_lifecycle': ('unreadable_lifecycle', 'unreadable_lifecycle', ''), # Regression fixture (#531): parameter services are discoverable # (wait_for_service succeeds) but list_parameters never replies. 'unresponsive_param': ('demo_unresponsive_param_node', 'unresponsive_param', ''), @@ -90,7 +94,8 @@ # --------------------------------------------------------------------------- def create_gateway_node(*, port=DEFAULT_PORT, name='ros2_medkit_gateway', - extra_params=None, coverage=True, extra_env=None): + extra_params=None, coverage=True, extra_env=None, + respawn=False, respawn_delay=1.0): """Create a ``gateway_node`` launch action with standard config. Parameters @@ -108,6 +113,18 @@ def create_gateway_node(*, port=DEFAULT_PORT, name='ros2_medkit_gateway', Additional environment variables merged into ``additional_env`` on top of the coverage env. Useful for setting ``ROS_DOMAIN_ID`` to isolate a multi-gateway test's peers into distinct DDS domains. + respawn : bool + If True, ``launch`` restarts the gateway whenever it exits before the + launch itself is shutting down. Only for tests whose subject IS a + gateway restart (state that must survive one, or must not be + resurrected by one): a test kills the process by PID and lets launch + bring the same configuration back. Off by default, so an unexpected + gateway death stays a visible failure everywhere else. + respawn_delay : float + Seconds ``launch`` waits before restarting. Non-zero so the HTTP port + and the DDS participant are released before the replacement binds + them, which also gives a test a window in which the port is provably + down - the only way to tell "restarted" from "never died". Returns ------- @@ -130,6 +147,8 @@ def create_gateway_node(*, port=DEFAULT_PORT, name='ros2_medkit_gateway', output='screen', parameters=[params], additional_env=env, + respawn=respawn, + respawn_delay=respawn_delay, # Default SIGINT->SIGTERM escalation is 5s and SIGTERM->SIGKILL is 5s. # Under TSan/ASan/coverage the gateway shutdown sequence (mdns stop, # REST server stop, transport teardown, plugin shutdown, plus flushing diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt index 73f97ddb8..9ccd6c794 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt @@ -185,6 +185,15 @@ if(BUILD_TESTING) medkit_target_dependencies(test_reliability_gate ros2_medkit_gateway rclcpp ros2_medkit_msgs lifecycle_msgs) target_link_libraries(test_reliability_gate nlohmann_json::nlohmann_json) + # Pure matching logic over hand-built inputs: no rclcpp::Node anywhere in it. Depends on + # ros2_medkit_gateway/nlohmann_json only because it assembles a description through the + # real AggregatedFault::describe (R13/C1/C3's cap-filling test), the same reason + # test_aggregated_fault below needs it - not because anything here builds a live graph. + medkit_add_gtest(test_lifecycle_expectation_tracker test/test_lifecycle_expectation_tracker.cpp) + target_include_directories(test_lifecycle_expectation_tracker PRIVATE include) + medkit_target_dependencies(test_lifecycle_expectation_tracker ros2_medkit_gateway rclcpp ros2_medkit_msgs) + target_link_libraries(test_lifecycle_expectation_tracker nlohmann_json::nlohmann_json) + # Pure matching logic over hand-built inputs: no rclcpp::Node anywhere in it. medkit_add_gtest(test_orphan_policy test/test_orphan_policy.cpp) target_include_directories(test_orphan_policy PRIVATE include) @@ -202,7 +211,7 @@ if(BUILD_TESTING) medkit_target_dependencies(test_aggregated_fault ros2_medkit_gateway rclcpp ros2_medkit_msgs) target_link_libraries(test_aggregated_fault nlohmann_json::nlohmann_json) - # Real DDS endpoints read through the live graph API. The two that ALSO register a fake + # Real DDS endpoints read through the live graph API. The three that ALSO register a fake # /fault_manager/report_fault service share a RESOURCE_LOCK, because they spin real nodes for # many seconds and running them concurrently on this box starves the discovery each one waits # on - a domain of their own does not help with that. @@ -254,6 +263,33 @@ if(BUILD_TESTING) set_property(TEST test_param_drift_integration APPEND PROPERTY RESOURCE_LOCK graph_watchdog_integration_domain) + # Registers a fake /fault_manager/report_fault service like orphan/param_drift above, so it + # shares the same RESOURCE_LOCK (see the comment on the integration block). + medkit_add_gtest(test_lifecycle_expectation_integration + test/test_lifecycle_expectation_integration.cpp + src/detectors/lifecycle_expectation_detector.cpp + src/detector_registry.cpp + src/reliability_gate.cpp + src/lifecycle_watcher.cpp + ${LIFECYCLE_STATE_READER_SOURCES}) + target_include_directories(test_lifecycle_expectation_integration PRIVATE include ${GATEWAY_SRC_INCLUDE_DIR}) + medkit_target_dependencies(test_lifecycle_expectation_integration + ros2_medkit_gateway rclcpp rcutils ros2_medkit_msgs lifecycle_msgs) + target_link_libraries(test_lifecycle_expectation_integration nlohmann_json::nlohmann_json) + set_tests_properties(test_lifecycle_expectation_integration PROPERTIES TIMEOUT 120) + set_property(TEST test_lifecycle_expectation_integration APPEND PROPERTY + RESOURCE_LOCK graph_watchdog_integration_domain) + + # Fixture node for the lifecycle_expectation e2e scenarios that need a node which looks + # managed, answers a chosen lifecycle label, and can stop advertising its lifecycle + # services on command - see test/e2e/droppable_lifecycle_node.cpp's file doc. It lives + # here rather than with the shared demo nodes because it exists purely for this + # package's own detector claims. Installed into lib/${PROJECT_NAME} so + # launch_ros.actions.Node(package=..., executable=...) can resolve it. + add_executable(droppable_lifecycle_node test/e2e/droppable_lifecycle_node.cpp) + medkit_target_dependencies(droppable_lifecycle_node rclcpp lifecycle_msgs) + install(TARGETS droppable_lifecycle_node RUNTIME DESTINATION lib/${PROJECT_NAME}) + # === QoS-mismatch e2e (launch_testing) === # # Acceptance gate: proves GRAPH_QOS_MISMATCH raises and clears through the @@ -351,6 +387,205 @@ if(BUILD_TESTING) set_tests_properties(test_launch_test_guard_system_build PROPERTIES LABELS "integration") medkit_test_needs_no_domain(test_launch_test_guard_system_build) + # === Lifecycle-expectation e2e (launch_testing) === + # + # Acceptance gate for the lifecycle_expectation detector: proves GRAPH_NODE_INACTIVE + # raises and heals through the REAL gateway + plugin .so + fault_manager stack against + # a REAL managed rclcpp_lifecycle demo node driven through real ChangeState + # transitions - the C++ integration test can only puppet the lifecycle label. + # + # One source file, eight CTest targets (the config_plumbing pattern): the plugin reads + # its config once at set_context() time, so each distinct config/fixture combination + # (main, unconfigured-by-default, negative-control, unreadable, healing_threshold, + # departure_keeps, not_managed, restart_loop) needs its own gateway process. + # WATCHDOG_E2E_SCENARIO selects which launch and which assertions run in each. + # + # 780 s. The budget is never shorter than the sum of the file's own internal deadlines + # plus the teardown the launch itself configures, because a budget under that sum is + # killed by ctest before the message a failing deadline exists to print reaches the log - + # turning every one of those diagnostics into a bare "timed out with no output". The sum, + # per test case: + # test_01: arming gate (60) + raise poll (60) + entity-scoped poll (30) = 150 + # test_02: CONFIGURE (30 service wait + 30 across the call attempts + 10 for the + # already-applied check a retry can need) + still-raised poll (30) = 100 + # test_02b: pre-restart record (30) + port-down wait (60) + post-restart arming + # gate (90) + heal-sequence settle (4) + post record (30) + poll (30) = 244 + # test_03: heal precondition (30) + ACTIVATE (30 + 30 + 10) + per-entity gate (60) + # + heal poll (60) = 220 + # 714 in total, and the run that exhausts it is the slow-but-PASSING run - every poll + # succeeding near its deadline. On top of that, teardown runs inside the same CTest + # budget: every launched process carries sigterm_timeout 30 + sigkill_timeout 15 + # (launch_helpers.py), so a process that sits out SIGTERM costs up to ~45 s AFTER the last + # assertion. 714 + 45 = 759, and a passing run that reached the teardown would otherwise + # be flipped into a timeout with no output. + medkit_add_launch_test(test_lifecycle_expectation_e2e_main + test/e2e/test_lifecycle_expectation_e2e.test.py TIMEOUT 780 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19190" "WATCHDOG_E2E_SCENARIO=main" "${_WATCHDOG_E2E_ENV}") + + # Shipped-default silence: no detectors.lifecycle_expectation config at all, with the + # unconfigured lifecycle node present - the README's zero-false-positive default claim, + # falsifiable nowhere else. 240 s: arming gate (60) + fault-surface gate (30) + + # label-read poll (30) + sustained silence window (20) + the post-window re-pin of the + # trigger (15) = 155 internal, plus the launch's own teardown (sigterm 30 + sigkill 15). + medkit_add_launch_test(test_lifecycle_expectation_e2e_default_config + test/e2e/test_lifecycle_expectation_e2e.test.py TIMEOUT 240 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19200" "WATCHDOG_E2E_SCENARIO=default_config" "${_WATCHDOG_E2E_ENV}") + + # Negative control: the same grace and cadence as the main scenario, with require_active + # naming the self-activating variant of the same executable instead - an active required + # node must never raise, so the discriminating variable is the node's real lifecycle + # state. 240 s: per-entity arming gate (60) + fault-surface gate (30) + label-read poll + # (30) + sustained silence window (20) + the post-window re-pin (15) = 155 internal, plus + # the launch's own teardown (sigterm 30 + sigkill 15). + medkit_add_launch_test(test_lifecycle_expectation_e2e_negative_control + test/e2e/test_lifecycle_expectation_e2e.test.py TIMEOUT 240 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19210" "WATCHDOG_E2E_SCENARIO=negative_control" "${_WATCHDOG_E2E_ENV}") + + # Proof that GRAPH_NODE_UNREADABLE itself raises and clears against a REAL, sustained + # GetState failure - launches unreadable_lifecycle_node.cpp (advertises get_state / + # change_state like a managed lifecycle node, but get_state never answers until told + # to) instead of the real rclcpp_lifecycle node every other scenario here uses. Needs + # its own, much faster tick cadence: kUnmeasuredHoldTicks (60, fixed - not + # configurable) is far more ticks than any `grace` window above ever waits out. + # 450 s: arming gate (60) + faults-live gate (30) + unread-label poll (30) = 120 + # (test_01); silence poll (10) + end-of-window label poll (15) = 25 (test_02); raise + # poll (60, see UNREADABLE_RAISE_TIMEOUT_SEC's own arithmetic comment in the .test.py + # for why 60 s covers an expected ~8 s of tick-driven work) + INACTIVE re-check poll + # (3) + entity-scoped poll (30) = 93 (test_03); heal-precondition poll (30) + + # SetParameters call (30) + active-label poll (30) + heal poll (30) = 120 (test_04) - + # 358 internal, plus the launch's own teardown (sigterm 30 + sigkill 15). + medkit_add_launch_test(test_lifecycle_expectation_e2e_unreadable + test/e2e/test_lifecycle_expectation_e2e.test.py TIMEOUT 450 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19230" "WATCHDOG_E2E_SCENARIO=unreadable" "${_WATCHDOG_E2E_ENV}") + + # The withheld-clear guard's pending leg, at the tier that runs the REAL fault_manager + # debounce state machine: a required node already reported stuck is SIGTERM'd and + # respawned twice (a real snapshot blink, not a lifecycle transition), each blink well + # inside the tracker's absence grace, against a sensitive healing_threshold of 1 - the + # value this package's own README recommends. 360 s: arming gate (60) + label poll (30) + # + raise poll (60) + two blinks (SIGTERM + up to 15 s rediscovery poll each, ~35 total) + # + settle (3) + post-blink record poll (30) + final label poll (15) = 233 internal, + # plus the launch's own teardown for TWO processes (sigterm 30 + sigkill 15 each, not + # fully serial) - the same generous margin the other scenarios budget. + medkit_add_launch_test(test_lifecycle_expectation_e2e_healing_threshold + test/e2e/test_lifecycle_expectation_e2e.test.py TIMEOUT 360 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19220" "WATCHDOG_E2E_SCENARIO=healing_threshold" "${_WATCHDOG_E2E_ENV}") + + # A sibling to the "unreadable" scenario above, not an extension of it: it needs a + # PERMANENT kill (no respawn) of the same fixture process, which cannot share a + # gateway process with test_04's start_answering-driven clear without racing it - + # see TestLifecycleExpectationDepartureKeeps's own docstring. Proves that an + # already-reported node LEAVING the graph does not heal its fault: a lifecycle + # promise that was never verified is no more verified once the node is gone. + # Previously proven only by the integration tier's + # UnreadableNodeAlreadyReportedThatVanishesKeepsItsOwnRecord. + # 420 s: arming gate (60) + faults-live gate (30) + unread-label poll (30) = 120 + # (test_01); raise poll (60, same UNREADABLE_RAISE_TIMEOUT_SEC arithmetic as the + # "unreadable" scenario above) + mutual-exclusion window (3) = 63 (test_02); departure + # poll (30) + settle (10) + record poll (30) + still-active poll (30) + description + # poll (10) + mutual-exclusion window (3) = 113 (test_03) - 296 internal, plus the + # launch's own teardown for TWO processes (sigterm 30 + sigkill 15 each, not fully + # serial) - the same generous margin healing_threshold budgets for the same launch shape. + medkit_add_launch_test(test_lifecycle_expectation_e2e_departure_keeps + test/e2e/test_lifecycle_expectation_e2e.test.py TIMEOUT 420 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19240" "WATCHDOG_E2E_SCENARIO=departure_keeps" "${_WATCHDOG_E2E_ENV}") + + # The not-managed sibling of "unreadable"/"departure_keeps": proves GRAPH_NODE_NOT_MANAGED's + # own raise (naming the node, WARN severity, mutually exclusive with the other two codes) + # and the same evidence retention across a departure, against a PLAIN demo node with no + # lifecycle interface at all (`calibration`) rather than a purpose-built fixture - none + # was needed for this cause. 300 s: arming gate (60) + faults-live gate (30) + presence + # poll (5) = 95 (test_01); raise poll (60, same NOT_MANAGED_RAISE_TIMEOUT_SEC arithmetic + # as the "unreadable" scenario's UNREADABLE_RAISE_TIMEOUT_SEC - if anything cheaper, since + # there is no blocking GetState round trip to pay for) + two mutual-exclusion windows (6) + + # entity-scoped poll (30) = 96 (test_02); raise precondition (60) + departure poll (30) + + # settle (10) + record poll (30) + still-active poll (30) + description poll (10) + + # mutual-exclusion window (3) = 173 (test_03) - 364 internal is already over 300, so this + # budgets from the same internal-deadline-sum discipline as the sibling scenarios but + # rounds the target up to 480 s to also cover the launch's own teardown for TWO processes + # (sigterm 30 + sigkill 15 each, not fully serial). + medkit_add_launch_test(test_lifecycle_expectation_e2e_not_managed + test/e2e/test_lifecycle_expectation_e2e.test.py TIMEOUT 480 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19250" "WATCHDOG_E2E_SCENARIO=not_managed" "${_WATCHDOG_E2E_ENV}") + + # The scenario the evidence-retention model exists for: a required node in a CRASH LOOP, + # killed and respawned on a cadence that never lets it accumulate the 60 consecutive + # PRESENT ticks kUnmeasuredHoldTicks needs. A node that keeps dying touches absence by + # construction, so a detector that discarded its clock there would stay silent forever - + # which is exactly the node this detector most exists to catch. Uses the same plain + # `calibration` fixture as "not_managed", with launch's own respawn. + # 300 s: arming gate (60) + faults-live gate (30) + presence poll (30) = 120; the kill + # loop's own wall-clock budget (45, RESTART_LOOP_WINDOW_SEC) plus the per-cycle + # departure/return polls it contains (30 + 30 on the worst single cycle) = 105 - 225 + # internal, plus the launch's own teardown for TWO processes (sigterm 30 + sigkill 15 + # each, not fully serial). + medkit_add_launch_test(test_lifecycle_expectation_e2e_restart_loop + test/e2e/test_lifecycle_expectation_e2e.test.py TIMEOUT 300 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19260" "WATCHDOG_E2E_SCENARIO=restart_loop" "${_WATCHDOG_E2E_ENV}") + + # A FULL tracked-node cap: two required nodes and `tracked_node_cap: 1`, so one of them is + # refused on every tick. Reachable only because the cap is a config key. Proves the refusal + # is visible on the status route, that it WITHHOLDS GRAPH_NODE_INACTIVE's clear, and that a + # departed entry is collapsed so the present broken node is checked instead. + # 420 s: arming gate (60) + fault-surface gate (30) + two presence polls (60) + label poll + # (30) + raise poll (60) + saturation poll (30) = 270 (test_01); record poll (30) + + # SetParameters (30) + not-managed raise poll (60) + saturation re-check (15) + withhold + # window (10) + record poll (30) = 175 (test_02); departure poll (30) + presence poll (30) + # + admission poll (60) + saturation poll (30) + two record polls (60) = 210 (test_03). + # The sum is the SLOW-but-passing run, in which every poll succeeds near its deadline; + # the expected run is a small fraction of it, and CI slowness is what the margin is for. + # Rounded to 900 to also cover the launch's own teardown for THREE processes (sigterm 30 + + # sigkill 15 each, not fully serial). + medkit_add_launch_test(test_lifecycle_expectation_e2e_cap_pressure + test/e2e/test_lifecycle_expectation_e2e.test.py TIMEOUT 900 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19270" "WATCHDOG_E2E_SCENARIO=cap_pressure" "${_WATCHDOG_E2E_ENV}") + + # One missed sweep before a clean shutdown must raise nothing, while the same drop held + # long enough to be corroborated must still be reported - two legs of one fixture whose + # only difference is the hold, so the pair discriminates instead of merely proving silence. + # Runs at a deliberately slow 500 ms cadence so the test can place its actions between + # ticks rather than race them. 480 s: arming gate (60) + fault-surface gate (30) + two + # label polls (60) = 150 (test_01); null-label poll (15) + departure poll (30) = 45 + # (test_02); null-label poll (15) + the deliberate settled hold (5) + departure poll (30) + + # raise poll (90) = 140 (test_03); three record polls (15) = 15 (test_04) - 350 internal, + # plus the launch's own teardown for THREE processes. + medkit_add_launch_test(test_lifecycle_expectation_e2e_unsettled_departure + test/e2e/test_lifecycle_expectation_e2e.test.py TIMEOUT 480 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19280" "WATCHDOG_E2E_SCENARIO=unsettled_departure" "${_WATCHDOG_E2E_ENV}") + + # `grace` at the value that used to be the accepted maximum. Under it the detector can + # neither raise nor heal GRAPH_NODE_INACTIVE for days, so the value must be refused and the + # documented default applied - measured by the fault actually appearing. + # 300 s: arming gate (60) + fault-surface gate (30) + label poll (30) + raise poll (60) = + # 180 (test_01); departure poll (30) + description poll (30) + record poll (10) = 70 + # (test_02) - 250 internal, plus the launch's own teardown for TWO processes. + medkit_add_launch_test(test_lifecycle_expectation_e2e_wide_grace + test/e2e/test_lifecycle_expectation_e2e.test.py TIMEOUT 300 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19290" "WATCHDOG_E2E_SCENARIO=wide_grace" "${_WATCHDOG_E2E_ENV}") + + # Records where "a departure never heals a fault" ends: it holds within one gateway + # lifetime, and a restart re-baselines because the restarted detector cannot tell an entry + # for a departed node from a misspelt one. Restarts the gateway by PID, like the "main" + # scenario. 420 s: arming gate (60) + label poll (30) + raise poll (60) + departure poll + # (30) + settle (4) + record poll (30) = 214 (test_01); port-down wait (60) + arming gate + # (90) + heal poll (90) + record poll (30) = 270 (test_02) - 484 internal is already over + # 420, so this budgets 600 and also covers the launch's own teardown for TWO processes. + medkit_add_launch_test(test_lifecycle_expectation_e2e_restart_departed + test/e2e/test_lifecycle_expectation_e2e.test.py TIMEOUT 600 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19300" "WATCHDOG_E2E_SCENARIO=restart_departed" "${_WATCHDOG_E2E_ENV}") + ros2_medkit_relax_vendor_warnings() endif() diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md index 2359957b9..5d878b839 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md @@ -10,8 +10,8 @@ and is driven by the plugin's own executor from the tick thread, never by the ga ROS executor. This package carries the plugin skeleton, the central reliability gate that holds raises -until the graph has quiesced, and three detectors, `qos_mismatch`, `orphan` and -`param_drift`. The remaining +until the graph has quiesced, and four detectors, `qos_mismatch`, `orphan`, +`param_drift` and `lifecycle_expectation`. The remaining silent-fault classes land in follow-up changes, each against its own issue; their fault codes are already reserved in the frozen `GRAPH_*` namespace (see "Fault codes"). @@ -47,6 +47,16 @@ read produces one startup warning naming the key and listing the ones that exist typo like `allow_list:` for `allowlist:` cannot silently do nothing. A typo in the detector id itself is warned about the same way, with the registered ids listed. +### `lifecycle_expectation` keys + +| Key | Type | Default | Meaning | +|-----|------|---------|---------| +| `mode` | string \| bool | `raise` | As above. | +| `require_active` | string[] | `[]` | Node names that must be in the `active` lifecycle state, each matched against a live node by its `App::id`, its full FQN (`/ns/name`), or the bare leaf of that FQN. A bare name matches that node in EVERY namespace (a fleet-wide "all `controller_server`s must be active"); use a full FQN to pin one robot's. Both the bare and the FQN form match against the node's stable FQN, so they survive the `App::id` renaming that a same-bare-name collision triggers; an entry written as an `App::id` also works, but can stop matching on a multi-robot graph for exactly that reason. Empty = the detector checks nothing and emits nothing. A `require_active` that is not a string array warns and is ignored; an empty-string entry warns and is skipped - never dropped silently, since the operator would go on believing the node is covered. | +| `grace` | int | `5` | Consecutive not-active ticks a required node tolerates before being reported inactive - bringup time for a managed node to reach `active` (configure + activate) before the expectation is enforced. Counted per NODE, not per matching entry: a node named by both a bare-name and a full-FQN entry still advances its streak once per tick, so mixing the two documented forms cannot halve the grace that was configured. Past the absence grace the streak keeps advancing while the node is ABSENT, so a node that leaves the graph while violating is still confirmed. Accepted range 0..300 - five minutes at the shipped 1 s cadence, and already an extravagant allowance for a managed node to reach `active`. The upper end is a real bound rather than a formality: `grace` also decides how long a node that LEFT the graph while not-active sits unsettled in the tracker, and `GRAPH_NODE_INACTIVE`'s clear is withheld for EVERY node while it does, so at the old maximum of `INT_MAX - 1` the fault could neither raise nor heal for about 24 days. The check runs on the wide integer, so a value that only fits after truncation is rejected rather than silently turned into a hair-trigger. Anything outside the range warns and keeps the default. | +| `prune_grace` | int | `60` | Consecutive ticks an IDLE tracked node - both clocks at zero and no matured ownership, so nothing to lose - may stay ABSENT from the graph before its bookkeeping is reclaimed. A node carrying evidence is never reclaimed by age at all, however long it stays gone, so this key cannot erase a fault's own state; the map is bounded instead by `tracked_node_cap` (see "Bounded by evidence, not by age"). Injected for every detector at plugin scope and overridable per detector; a value outside 0..3600 warns and this detector keeps its own default of 60. The range check runs on the wide integer, so an out-of-int-range value is rejected rather than truncated. The value is used as written - there is no `grace + 1` clamp, because there is no longer anything for one to protect. | +| `tracked_node_cap` | int | `512` | The most nodes this detector keeps bookkeeping for at once. It is what bounds the map, since evidence is never reclaimed by age (see "Bounded by evidence, not by age"): at the cap, idle entries are reclaimed first, then entries for DEPARTED nodes are collapsed into a count, and only if every tracked node is PRESENT and carrying evidence is a newly matched node refused - which withholds `GRAPH_NODE_INACTIVE`'s clear and is reported both in the log and on `GET /x-medkit-watchdog`. 512 is comfortably above every node in a realistic graph (a full Nav2 stack plus perception is roughly a hundred nodes; a ten-robot fleet sharing one domain a few hundred), so raising it is only needed where `require_active` legitimately matches more PRESENT nodes than that. Accepted range 1..16384 - 16384 is about 8 MB of bookkeeping at roughly 500 bytes per tracked node, and a deployment needing more distinct required-node identities alive at once has identity churn rather than a large fleet. Anything outside the range, including 0 (a cap of nothing would mean checking nothing), warns and keeps the default; the check runs on the wide integer, so an out-of-int-range value is rejected rather than truncated. | + ### `orphan` keys | Key | Type | Default | Meaning | @@ -585,6 +595,610 @@ then a start. Accepting a new value as the baseline without a restart does not e ROS parser types as a YAML 1.1 boolean rather than a string, which is the form operators actually write. +#### `lifecycle_expectation` (GRAPH_NODE_INACTIVE, GRAPH_NODE_UNREADABLE, GRAPH_NODE_NOT_MANAGED) + +A configurable "this node must be active" check. The process is ALIVE and in the graph, +but a node the operator declared critical-active is either sitting in a non-active +lifecycle state (`inactive`/`unconfigured`/`finalized`), or its lifecycle promise could +not be MEASURED at all - a managed node whose label has never been read, or a node with +no tracked lifecycle whatsoever. On a Nav2 stack a `controller_server` stuck inactive +means the robot silently will not act - no crash, no log, nothing on `/diagnostics` or +`/rosout`. Presence itself is a different fault class (`GRAPH_NODE_DISAPPEARED`, reserved +in the frozen namespace for a follow-up change): this detector only ever STARTS reporting a +node it has measured while PRESENT. A node that leaves the graph having only ever been +measured healthy is left entirely to the presence class - but one that leaves while already +under one of these three faults keeps it, because a node the operator declared must-be- +active going away is not an answer. See "Absence continues, it never erases" below. + +**The model: one observed state per node per tick, two clocks.** All three fault codes +are outputs of a single per-node state machine +(`LifecycleExpectationTracker`, `lifecycle_expectation_tracker.hpp`) - not three +independent trackers that happen to share a detector. Every tick, a matched node is +classified into exactly one of `ACTIVE` (label `"active"`), `INACTIVE` (label non-empty +and not `"active"`), `UNREADABLE` (a managed node whose label has never been read - +`lifecycle_state_of()` returned `optional("")`), or `NOT_MANAGED` (`lifecycle_state_of()` +returned `nullopt` - no tracked lifecycle at all); a node not matched at all this tick is +a fifth, separate case, `ABSENT`. Two clocks track that history, and only two: + +- The **violation streak** advances on `INACTIVE` and resets ONLY on `ACTIVE`. Past + `grace`, the node is `GRAPH_NODE_INACTIVE`'s content. +- The **unmeasured clock** advances on `UNREADABLE` OR `NOT_MANAGED` and resets ONLY on a + real measurement (`ACTIVE` or `INACTIVE`). It is deliberately BLIND to which of the two + causes it is seeing on any given tick - a node alternating between "matched but never + read" and "not a tracked lifecycle node at all" keeps this one clock climbing instead of + each cause resetting a separate counter of its own. That blindness is what closes an + entire class of alternation: three review rounds each found one pair of "I cannot + measure this" causes that could erase each other's progress and leave a node invisible + to every code that exists, and the fix is not a fourth special case, it is refusing to + keep separate counters for causes the operator's actual question ("can I trust this + node's state?") does not distinguish. Past `unmeasured_hold_ticks` (a fixed 60 ticks, + not configurable), the clock MATURES and takes ownership of the node. + +**Ownership is exclusive.** A node is content of at most one of the three fault codes at +a time. The moment the unmeasured clock matures, the violation streak is RELEASED - +actually reset to 0, not merely excluded from `GRAPH_NODE_INACTIVE`'s content this tick - +so that fault's clear is free to flow immediately. A node returning to `INACTIVE` +afterwards therefore re-earns `grace` from zero, exactly like a node that had never gone +unmeasured. Which of the two unmeasured codes a matured node reports under is STICKY: +`GRAPH_NODE_UNREADABLE` or `GRAPH_NODE_NOT_MANAGED`, whichever the clock matured under, +keeps reporting under that code even if the LIVE cause later flips - a node's own maturity +never re-derives its fault code from the current tick's read, only from a real +measurement resetting the clock entirely. The rejected alternative, current-cause-wins, +is more literal ("report exactly what is true right now") but makes a node that flaps +between the two causes flap the fault surface too - a raise/clear/raise churn across two +different fault codes for a node whose actual situation (still cannot be measured, either +way) never changed. + +**Absence continues the last CORROBORATED observation, it never erases.** A node not +matched at all this tick is ABSENT, a separate fact from any observed state (there is no +label to classify). For up to `absence_grace` (a fixed 3 ticks) consecutive absent ticks a +node's whole state - both clocks, whichever fault (if any) currently owns it - is held +unchanged, so a node blinking out of one snapshot in every few is not indistinguishable +from one that was never measured. PAST `absence_grace` absence advances whichever clock the +node's SETTLED observation had started, and resets nothing: + +| Settled observation | What sustained absence does | +|---|---| +| `inactive` (or any non-active label) | the violation streak keeps climbing, so the node is eventually reported under `GRAPH_NODE_INACTIVE` even though it is gone | +| unread label, or no tracked lifecycle | the unmeasured clock keeps climbing under that same cause, so the node is eventually reported under `GRAPH_NODE_UNREADABLE` / `GRAPH_NODE_NOT_MANAGED` | +| `active` | nothing. Anything the node had started but not corroborated is released, the entry becomes idle and is reclaimed silently | + +**What "settled" means.** A real measurement - `active` or a non-active label - settles +immediately: a lifecycle label is a fact about the node, and no discovery artifact invents +one. An UNMEASURED observation settles immediately too on a node nothing has ever measured, +because it is then the only thing known about it (this is what keeps a crash-looping node +that is unreadable whenever it is up accumulating toward its report). It is only when an +unmeasured reading would OVERRIDE a real measurement that it must first hold for 6 +consecutive matched ticks. The reason is that "no tracked lifecycle" is producible by the +discovery layer missing a node's service path for a single sweep, on a node that is present +and perfectly healthy - `LifecycleWatcher::update()` drops a tracked id whose path is absent +from the current sweep, and `discover_apps()` can yield an app with no services when a sweep +races service enumeration. Without corroboration, one such sweep immediately before a clean +shutdown would mature a healthy departure into a permanent `GRAPH_NODE_NOT_MANAGED`. Six is +the same bar the "probably a typo" warning below already sets against the same transient, +and a tenth of the 60-tick unmeasured hold, so a node that is genuinely unmeasurable when it +leaves has corroborated that long before the hold that reports it. + +Three consequences worth stating plainly: + +- **A departure never heals a fault, within one gateway lifetime.** A node measured + not-active, or corroborated as unmeasurable, that then leaves the graph keeps its fault - + the operator declared it must be ACTIVE, it was not, and being gone is not an answer. The + fault's description switches to saying the node has since left the graph, so nobody is + sent looking for it. **Across a gateway restart it does not hold**, and that is a real + boundary rather than an oversight: the restarted detector has no measurements at all, the + departed node is not in the graph, and its `require_active` entry therefore matches + nothing - which is indistinguishable from a misspelt entry, since the only component that + knows the difference is the fault store and this detector does not read it at startup. The + hold for a never-matched entry is deliberately bounded (a typo must not block healing + forever), so once it expires the level-triggered clear flows and the record heals without + anything having been measured. Re-seeding the tracker from the fault store at startup + would change this; it is not implemented. The e2e scenario `restart_departed` pins the + behaviour as it actually is. +- **A departure never STARTS one.** A node measured `active` that shuts down raises + nothing, and neither does one whose lifecycle services went missing from a sweep or two + immediately before it did. Reporting a healthy node that left is the presence class's job + (`GRAPH_NODE_DISAPPEARED`), which still has no detector in this package - the one gap + left here, and a much narrower one than "a node absent past the grace is invisible + whichever clock it was on". +- **A departed node never crowds out a present one.** Entries for departed nodes are + collapsed into a count when the tracked-node cap needs the slot - see "Bounded by + evidence, not by age" below. + +Why: every erasure horizon is an evasion for a node that touches it periodically. A node +in a restart loop - start, crash, respawn delay, start - touches absence by construction, +and that is the node this detector most exists to catch. Discarding its evidence on +absence let it alternate `(unreadable, absent x N)`, `(not-managed, absent x N)` or +`(inactive, absent x N)` forever without ever accumulating enough of anything to be +reported. + +**Content follows the clocks, not the snapshot.** Whether a node happened to be in this +tick's matches decides nothing about what it reports: a node whose streak is past `grace` +stays in `GRAPH_NODE_INACTIVE`'s content while it blinks, and a matured node stays in its +own code's content. A fault's content is what the detector has MEASURED, and a missing +snapshot entry measures nothing either way. + +**Safe default: off.** `require_active` is empty by default, so nothing is checked and +nothing is emitted - no raises AND no clears, not a single fault_manager request - until +an operator opts specific nodes in. Zero false positives out of the box, the same +config-scoped posture as `param_drift`'s `expect`. + +```yaml +plugins: + graph_watchdog: + detectors: + lifecycle_expectation: + require_active: ["controller_server", "planner_server"] + grace: 5 +``` + +**The fault names the NODE, not the config entry.** The bare-name form is deliberately +fleet-wide, so one entry legitimately covers N nodes - and keying the violation on the +entry would leave the fault unable to say which of them broke. On a dual-arm robot +`require_active: ["controller_server"]` would produce `node controller_server expected +active but is inactive`, with nothing pointing at `/left` or `/right`, and with two +offenders only one would survive into the report at all. `source_id` cannot +disambiguate either (every `GRAPH_*` fault shares one), so the description is the only +carrier. Each offending node is its own entry, named by its stable FQN, with the entry +that demanded it as context. Two entries naming the SAME node (the documented bare-name +plus pinned-FQN mix) advance its clocks once per tick, not once per matching entry - +counting per match would silently halve whatever `grace` or the unmeasured hold the +operator configured. + +**A single-snapshot blink does not restart anything.** A matched node that drops out of +the snapshot has its WHOLE state (both clocks, whichever fault currently owns it) held +unchanged for a small absence grace (3 ticks, fixed); discarding it on the first blink +would mean a genuinely stuck node that drops out of one snapshot in every few never +accumulates enough consecutive ticks on either clock to be reported at all. That is +reachable at the shipped cadence: the tick is 1s and the entity cache is graph-event +driven with a 1s debounce, so on a churning graph the snapshot turns over roughly once +per tick. Past the absence grace the clocks resume advancing rather than restarting - see +"Absence continues, it never erases" above. + +**An entry that has never matched anything is reported.** It would otherwise be silent +in every path: it never reaches any of the three fault codes, and a presence detector +cannot backfill it either, because it only ever tracks nodes that were present at least +once. So a misspelt entry - or a required node that crashed during launch, or was down +when the gateway started - would produce no fault and no log line anywhere. After 10 +consecutive ticks matching nothing, the detector says so in the log, once per entry per +configuration. An entry whose node WAS matched and later left the graph is deliberately +not accused: for it both halves of that sentence are false, and a departed node is +precisely what the presence class reports. + +**The typo warning does not latch.** An entry that matches present nodes but none with +a tracked lifecycle state is probably a typo, or names a plain (non-lifecycle) node - +so the detector warns about it, but only after more than 5 CONSECUTIVE such ticks, and +the warning clears if the entry resolves. One transient tick is not evidence: the gateway's +`discover_apps()` wraps per-node service enumeration in a try/catch and pushes the app +regardless, so a sweep that races service discovery yields an app with no services and +no tracked state. The once-per-entry bookkeeping is reset on every `configure()`, so an +entry removed and re-added by a reconfigure warns again - which is exactly when the +operator wants to hear that it still names nothing managed. The warning is suppressed +when no reliability gate is wired, since then no entry has a lifecycle state to begin +with. This is a SEPARATE, log-only mechanism from `GRAPH_NODE_NOT_MANAGED` below: the +warning fires after 5 ticks and never raises a fault; the fault code fires only after +the same 60-tick unmeasured hold as its `GRAPH_NODE_UNREADABLE` sibling. + +**Bounded by evidence, not by age.** One bare-name entry can match a node of that name in +every namespace, so under identity churn (nodes reappearing under ever-new FQNs) the +tracker's map would grow without a bound. Since absence no longer erases anything, an +entry carrying a live clock is not reclaimed by age either - the two are the same defect +seen twice, and moving a horizon rather than removing it would leave the evasion in place +at a different N. So: + +- **`prune_grace` reclaims IDLE bookkeeping only** - both clocks at zero and no matured + ownership, i.e. an entry with nothing to lose. An idle node absent for more than + `prune_grace` consecutive ticks is reclaimed atomically: one node, one map entry, gone + in the same tick, never partially. There is no `grace + 1` clamp any more, and none is + needed: a node carrying evidence is exempt by construction rather than by arithmetic, so + `prune_grace: 0` means exactly 0. +- **A non-idle entry is never pruned by age at all.** It cannot grow without bound in time + either: past the absence grace its clock advances every tick, so it matures within at most + `grace + absence_grace + 1` ticks (a violation streak) or `60 + absence_grace + 1` (an + unmeasured clock) and gets reported. Those two numbers are also the longest + `GRAPH_NODE_INACTIVE`'s clear can be withheld by one departed node, which is why `grace` is + capped at 300 rather than accepted up to the int maximum: at the old maximum the bound was + about 24 days at the shipped cadence, i.e. the fault could neither raise nor heal for + anybody, with no warning and no way to tell it from a working detector finding nothing. +- **The map is bounded by `tracked_node_cap`** (default 512, accepted range 1..16384). At + the cap, idle entries are reclaimed first (free - they carry nothing), then entries for + DEPARTED nodes are collapsed into a count so a PRESENT node always wins a slot; only if + every tracked node is PRESENT and carrying evidence is the NEWCOMER refused - never a live + violation evicted. 512 is comfortably above every node in a realistic graph (a full Nav2 + stack plus perception is roughly a hundred nodes; a ten-robot fleet sharing one domain a + few hundred), so only identity churn or a genuinely larger required set reaches it. + + Collapsing rather than holding follows from the fault being keyed by CODE, not by node: + five hundred entries for dead identities keep the same one fault raised that a single entry + would, and the description can only ever name a handful of them - so holding them buys + nothing, while the slots they occupy can cost total blindness. Under identity churn a cap + full of the dead would refuse a genuinely broken PRESENT node, which then never enters the + detector's content at all, and `GRAPH_NODE_INACTIVE` would emit a level-triggered CLEAR + every tick while that node sat there not-active. At most three departed entries stay named + (three maximally-long details are all one 480-character description holds, so a fourth name + could never be shown anyway); the rest become one line reading "and N more required node(s) + left the graph ...". That line IS content, so collapsing an entry never heals its fault - + and it is ordered ahead of the individually named entries but behind anything crossing on + this tick, so a node that just broke is never displaced by it. The count only grows within + one gateway lifetime: once collapsed, an entry's fqn is no longer known, so a node + returning under it is tracked and measured afresh and cannot decrement the count. A + non-zero count therefore says two things at once - those nodes left carrying evidence, and + `require_active` is matching more identities than `tracked_node_cap` can hold. + + Refusing a newcomer means a required node goes unchecked, which is why it is never silent: + it withholds `GRAPH_NODE_INACTIVE`'s clear for as long as it lasts (a detector that + declined to check a required node cannot assert that every required node is healthy), it is + logged once per saturation EPISODE - the latch re-arms when saturation ends, so a later, + real one is not silenced by an earlier one - and it is reported on `GET /x-medkit-watchdog` + under `detectors.lifecycle_expectation.tracking_saturated`, beside the current + `tracked_nodes` count and the `tracked_node_cap` in force. Unlike the never-matched hold, + this withhold is NOT bounded: that hold is bounded because a typo must not block healing + forever, while saturation - once departed entries can no longer crowd out present ones - + means genuinely more required PRESENT nodes than the cap allows, which is a capacity + condition the operator resolves by pinning fewer identities or raising the key. + +**Why grace, not the reliability gate.** Every detector's raise passes through +`ctx.raise_fault()`'s central `reliability_allows(gate, source_id)` gate, which ANDs +the entity's warmup state with `LifecycleWatcher::node_ok()` - true only when a managed +node's lifecycle is `active` (or it has no tracked lifecycle at all). Gating THIS +detector's raise the same way on the required node's own lifecycle would be +self-defeating: `node_ok()` is exactly FALSE for the inactive node this detector exists +to catch, so the central gate would suppress the signal forever. Instead the detector's +own pure, ROS-free core (`LifecycleExpectationTracker`, independently unit-tested in +`test_lifecycle_expectation_tracker.cpp`) counts consecutive not-active ticks itself +and raises only past its own `grace` - completely independent of `reliability_allows`. +The aggregated fault's outer `ctx.raise_fault` call is still subject to the central +gate for its own `source_id` (`graph_watchdog`), same as every other detector - see +"Reliability (bringup-quiesce)" below - just not on the required node's lifecycle +state, which is this detector's own job. + +**A clear is withheld until the required set has actually been measured - this section +is entirely about `GRAPH_NODE_INACTIVE`.** `GRAPH_NODE_UNREADABLE` and +`GRAPH_NODE_NOT_MANAGED` have no withheld-clear guard of their own - see "Three +independent faults, not one shared record" below for why they don't need one. A clear +asserts that every required node is free of a CONFIRMED violation, and the +`GRAPH_NODE_INACTIVE` raised before a gateway restart is still in the store while the +restarted detector counts from zero - so a clear emitted on the strength of an empty +affected map would spuriously HEAL a still-real fault. Two things produce that empty map +without the assertion being true, both the ordinary state of bookkeeping that just +started over (a restarted gateway, a reconfigure, a node that respawned stuck): + +1. **An entry has not matched anything yet.** Before the entity snapshot catches up with + the graph, a `require_active` entry matches no node at all - so a clear would be about + a node the detector has never once looked at. Every restart passes through this + window: the plugin starts ticking as soon as it is loaded. Bounded the same way the + node-keyed state below is (60 ticks), so a misspelt entry cannot block healing for the + process lifetime. +2. **A node's status is UNSETTLED.** The tracker's own `pending` set - a violation streak + that has not yet passed `grace`, an unmeasured clock still climbing (whichever of the + two causes), or a streak HELD while the node is inside an unmeasured spell. A node + whose unmeasured clock has MATURED is deliberately NOT in this set: ownership passed to + its own fault code and the violation streak was released (see "Ownership is exclusive" + above), so a node stops counting toward this withhold the exact tick it stops being + uncertain - it does not linger as a reason to keep `GRAPH_NODE_INACTIVE` waiting once + the question about it has a real answer somewhere else. Absence never puts a node here + on its own: content follows the clocks, so a node already past `grace` stays in the + fault's content through a blink rather than dropping into a withheld limbo. + +Either reason withholds the emission entirely, neither raise nor clear, so a fault +already in the store keeps its state instead of being healed by a detector that has not +re-established the node is fine. + +What never blocks `GRAPH_NODE_INACTIVE`'s raise: a RAISE is never withheld (a violation +read from the nodes that did answer is real regardless of the unmeasured ones). An entry +that HAS matched and later stops matching does not re-enter reason 1 either. Every hold is +bounded, and every one of them releases by SETTLING the node's status rather than by +giving up on it: the never-matched hold and both unmeasured-clock causes release after 60 +consecutive ticks (a minute at the shipped 1s cadence), whether the node is present or +gone; a below-`grace` violation streak releases as soon as the node reads `active` or its +streak passes `grace` and the fault is raised - which, past the absence grace, happens +while the node is absent too. +Because a withheld clear is indistinguishable from a detector that is working and +finding nothing, a hold that lives past 10 consecutive ticks is explained in the log once +per episode, naming every reason in force and, for the node-keyed ones, how many nodes +are behind each and one of them by name - the not-managed and unreadable reasons are +named separately even though both now release the same way (into their own fault code), +since an operator reading the log wants to know WHICH of the two is happening, not just +that one of them is. + +**Three independent faults, not one shared record.** `GRAPH_NODE_INACTIVE`, +`GRAPH_NODE_UNREADABLE` and `GRAPH_NODE_NOT_MANAGED` are each raised through the shared +`AggregatedFault` helper (see "Aggregated fault, not per-node" below), but as three +SEPARATE, fixed-severity members - `GRAPH_NODE_INACTIVE` always `SEVERITY_ERROR`, the +other two always `SEVERITY_WARN` - the same shape `orphan` and `param_drift` use, not a +single record whose severity has to be picked from that tick's mixed content. Raises are +fully independent: each fault's content comes from its own measurement, and one raising, +healing, or changing severity never forces, blocks, or reflects onto the others. +`GRAPH_NODE_INACTIVE`'s own clear is not simply "nothing CONFIRMED non-active this +tick", though - it is withheld exactly as described under "A clear is withheld..." +above; that is the ORIGINAL withheld-clear guarantee this detector always gave, scoped +to `GRAPH_NODE_INACTIVE` alone. `GRAPH_NODE_UNREADABLE` and `GRAPH_NODE_NOT_MANAGED` +have no such guard: each one's own clear needs nothing beyond its own content going +empty, because once the unmeasured clock has matured "still cannot be measured" is a +settled fact, not a pending one - there is nothing left to wait on. A node is content of +at most one of the three at a time, and healing one never forces, blocks, or changes the +severity of another. Content under either unmeasured code survives for as long as a node +stays that way, with no further bound past the initial hold. + +**A re-bind is a fresh binding.** The lifecycle label cache is keyed by `App::id`, and +an id can survive a graph sweep while pointing at a DIFFERENT node (id assignment +shifts under bare-name collisions). The shared `LifecycleWatcher` therefore re-checks +every tracked id's binding identity - its FQN and its `GetState` service path, both +captured at first sighting - on every tick. A moved binding is two events at once: the +OLD binding is recorded as departed under ITS OWN FQN (same record and retention as a +vanish), and the id is re-seeded through the ordinary new-node path (fresh `GetState`, +fresh `~/transition_event` subscription). For this detector that means the departed +node's label is never enforced against the node the id now binds: the new binding +starts unknown (benign) until its own label is read. No straggler from the old +subscription has to be filtered out - dropping the entry destroys that subscription on +the tick thread, the only thread that ever pumps these callbacks, so nothing it had +queued is delivered afterwards. + +**Aggregated fault, not per-node**, via the shared `AggregatedFault` helper - the same +rationale as every other detector here: the fault_manager identifies a fault by +`fault_code` alone, so one `GRAPH_NODE_INACTIVE` per stuck node would collide into a +single record under the shared code. Three graph-level faults, each enumerating every +currently affected node for ITS OWN code, each description capped independently at 480 +characters with a truncation marker. Like `orphan` and `param_drift`, all three are +fixed-severity `AggregatedFault` members at class scope (`aggregated_inactive_`, +`aggregated_unreadable_`, `aggregated_not_managed_`) rather than being rebuilt per tick: +unlike `qos_mismatch_detector`'s `any_starved ? kStarvedSeverity : kPartialSeverity`, +there is no record anywhere in this detector whose severity has to be picked from mixed +content. `GRAPH_NODE_INACTIVE` clears (`EVENT_PASSED`) on a tick where nothing is +CONFIRMED non-active AND the withheld-clear guard above is satisfied - not on every tick +where the tracker's affected map happens to be empty. The other two clear on any tick +where no matched node is owned by them, with no guard of their own to satisfy. All three +reaching HEALED needs the same `healing_enabled` requirement described in "Closing the +loop" above. + +**New violations are named first, not alphabetically - in EACH fault's own description +independently.** The description used to list affected nodes in fqn order +(`AggregatedFault::emit`'s default), which is fine when every entry is equally +interesting but not once the cap is full: a fleet sharing +`require_active: ["controller_server"]` across a dozen robots fills the 480-char cap from +the alphabetically-earliest ones, and a THIRTEENTH robot going inactive afterward would +be silently invisible forever - one shared `fault_code`, one record, no way to tell the +operator which of thirteen actually broke. The tracker reports which fqns entered EACH +fault's content on THIS tick - `newly_affected`, `newly_unreadable`, `newly_not_managed` - +and each list orders only its OWN fault's `AggregatedFault::emit_ordered` call, since the +three faults never share a description: a fresh entry is named FIRST in whichever fault +it belongs to, and every other affected node in that same fault still appears, in the +same fqn order as before, once the fresh ones are placed. When every node crosses on the +same tick (`grace: 0` during a bringup burst, for instance), there is nothing to +distinguish them by and the order degrades to fqn order. Two budgets protect each +description, applied before it ever reaches the 480-char cap: the lifecycle label - +which arrives verbatim off a remote `~/transition_event` and is therefore untrusted and +unbounded, and only ever appears in `GRAPH_NODE_INACTIVE`'s own detail, never the two +unmeasured faults' (there is no live label to show for either of them) - is trimmed to 32 +characters before it is interpolated (more than double the longest label a conforming +implementation produces, `errorprocessing` at 15 characters), and the whole per-node +detail is then capped at 150 characters as a backstop against a pathological fqn or a +long `require_active` "required by" list, sized so at least three worst-case details +still fit inside the 480-char cap (`3 * 150 + 2 * 2 = 454 <= 480`). + +**Test tiers:** + +1. **Unit** (`test/test_lifecycle_expectation_tracker.cpp`): pure + `LifecycleExpectationTracker` logic over hand-built matches. Covers the violation + streak (stuck-inactive past grace raising keyed by the node with the entry as context, + active never raising, reaching active within grace resetting the streak, both + namesakes reported separately and a healthy namesake not masking a broken one, two + entries naming one node not halving its grace, a violating read winning a duplicate + match's tie-break); the unmeasured clock shared by UNREADABLE and NOT_MANAGED (neither + ever confirms a violation while climbing; each matures into its OWN fault at the exact + hold boundary, tick-exact; the clock resets ONLY on a real measurement, in both + directions; maturity RELEASES the violation streak entirely, so a node returning to + inactive re-earns grace from zero rather than re-crossing immediately; the sticky-cause + design decision - a node stays reported under whichever code it matured under even + after the LIVE cause flips, pinned directly against the rejected current-cause-wins + alternative); the alternation this slice's redesign exists to close - a node + alternating between unreadable and not-managed, indefinitely and on every single tick, + still matures the shared clock, closing the exact hole that let a node evade both + codes when they were counted separately; the alternation between a MEASURED not-active + read and a not-managed one across absence gaps longer than the absence grace; that the + violation streak survives non-maturing unmeasured ticks and RESUMES rather than + restarting (counted exactly, and read through `pending_violation` - the only field that + differs during the climbing window); absence CONTINUING whichever clock the node's last + real observation started (a blink holds it unchanged; past the blink tolerance it + advances, so a node absent long enough matures or crosses grace on absence alone and its + detail says the node has left the graph), a matured fault surviving a departure, a node + measured ACTIVE that vanishes raising nothing and being reclaimed, and the three + `(X, absent x N)` restart-loop shapes for N at the absence grace and past it; the + `pending` set and its per-reason breakdown; new-first ordering for all three + fault-shaped maps, including the lexicographic tie-break when several cross together; + the remote-supplied label's own trim budget ahead of the whole-detail backstop, and the + same backstop reapplied to a matured unreadable node's detail (which carries no label, + only a fqn and a "required by" list); the age horizon now reclaiming IDLE entries only, + atomically, and never touching one that carries evidence even when it undercuts the + absence grace; the SETTLING rule that decides what absence may continue - an + uncorroborated unmeasured run before a healthy departure raising nothing, swept across + every run length below the bound, with the entry released rather than left holding the + clear hostage; the same run one tick longer still being reported; and a single MEASURED + not-active read before a departure still confirming, since a label needs no corroborating; + and the tracked-node cap - idle entries reclaimed first, departed entries collapsed into a + count so a present broken node is always admitted and reported, at most three of them left + named, the count surviving into the description as content, a node returning after its + entry was collapsed being measured afresh, the newcomer refused only when every entry is + PRESENT and carrying evidence, and saturation reported as a LEVEL on every refused tick with + its edge re-arming when an episode ends - swept at a shrunk cap and again at the real + shipped 512. +2. **Integration** (`test/test_lifecycle_expectation_integration.cpp`): the detector + driven against a fake `ReportFault` service, with a REAL `ReliabilityGate` arming the + global bringup grace and feeding it labels through + `ReliabilityGate::lifecycle_state_of()` (injected via the gate's test seam strictly + after the last `gate.update()` - see the file header for the ordering rationale, and + for the alternation/not-managed cases, which instead toggle whether the matched app + carries lifecycle services and drive the gate's real discovery path, since the + injection seam can only ever SET a tracked value, never remove tracking to produce a + genuine `nullopt`). Covers the `GRAPH_NODE_INACTIVE` raise/clear round trip naming the + node, the active-from-arming positive control, the absence-after-raise clear, bare-name + and full-FQN matching against a namespaced app, the zero-config silence measured as + zero fault_manager requests of any kind, the once-per-configuration unmanaged-entry + warning, the withheld-clear guard's releases and its once-per-episode log line, the + blink-plus-unread-re-seed sequence, the no-match warning, a filler batch sized (from + the real detail-building code) to exceed the 480-char cap aggregating into one fault + with the fresh crossing named first, a PRESENT node crossing on the same tick as a batch + of departed ones being named ahead of them (and not truncated away by them), a required + node appearing mid-run, a re-bind under the same `App::id`, and the + reconfigure/config-validation edge cases. The + unmeasured clock's own split is pinned directly, for BOTH codes symmetrically: a + managed node whose `GetState` genuinely never answers (through `set_managed_app` and a + real, failing seed - proven by asserting `lifecycle_state_of()` actually returns + `optional("")` first, not assumed) is reported under `GRAPH_NODE_UNREADABLE`; a node + with no tracked lifecycle at all is reported under `GRAPH_NODE_NOT_MANAGED` (no longer + released into silence, the deliberate behaviour change this slice makes); either hold + releases into its report on the exact tick past its bound, never a tick early; a node + already reported under one of the two clears once genuinely read, returning to + ordinary `GRAPH_NODE_INACTIVE` tracking from there; a confirmed node healing while an + unreadable OR not-managed sibling stays present clears `GRAPH_NODE_INACTIVE` promptly + without touching either unmeasured fault; content under either code survives for as + long as the node stays that way, with no window and no expiry; 25 nodes of either + cause aggregate into one capped fault, at `SEVERITY_WARN`, never `GRAPH_NODE_INACTIVE`; + a node whose hold expires opens its fault's own description over an already-reported + filler batch, the same new-first ordering; an already-reported node of either cause + KEEPING its own record once it vanishes (and its description switching to say the node + has left the graph), a node returning from that absence staying reported with no + clear/re-raise churn, each of the three restart-loop shapes raising its own code, the + `inactive` <-> `not-managed` alternation across absence gaps, a node measured ACTIVE + that vanishes raising nothing under any of the three, and a withheld `GRAPH_NODE_INACTIVE` + clear releasing when the absent node's clock MATURES into its sibling's content rather + than when the node is given up on; the two wire strings pinned as hand-typed literals + never read from their constants; and the independence claim in both directions - + clearing one unmeasured fault never disturbs a concurrently raised + `GRAPH_NODE_INACTIVE` or the OTHER unmeasured fault. + + Alongside the fixture, the `configure()`-level cases pin the config + contract: the unknown-key warning, a fully-valid config producing zero warnings + (including the documented low endpoints), `grace` and `prune_grace` validation + (negative, non-integer, past the int range, and both range endpoints), `require_active` + validation, `tracked_node_cap` validation at both range endpoints and one value past each + (with the key proven IN FORCE at both ends, not merely accepted), the unclamped prune + horizon reaching idle bookkeeping at exactly the configured `prune_grace` (0, 1 and 4 - + the smallest positive value included, since neither documented endpoint sweeps it) while a + node carrying evidence survives it - including the `grace: 0, prune_grace: 0` corner and a + wide `grace` beside the tightest `prune_grace`, whose instrument is the CONFIRMATION rather + than the map size - boundedness under identity churn at the real 512-node cap by collapsing + the departed rather than refusing the live node, and saturation reported once per EPISODE + with a second episode reported again after the first ends. +3. **E2e** (`test/e2e/test_lifecycle_expectation_e2e.test.py`): the acceptance gate - + one file, twelve CTest targets, each launching its own real gateway + plugin `.so` + + fault_manager stack. Six scenarios drive the `managed_lifecycle` demo node (a real + `rclcpp_lifecycle::LifecycleNode`, which always answers `GetState`); two drive + `unreadable_lifecycle_node.cpp`, a fixture built specifically to never answer it (see + below); two drive `calibration`, a PLAIN demo service node with no lifecycle + interface at all - no purpose-built fixture was needed for `GRAPH_NODE_NOT_MANAGED`, + unlike `GRAPH_NODE_UNREADABLE`; and two drive this package's own + `test/e2e/droppable_lifecycle_node.cpp`, which looks managed, answers a chosen lifecycle + label, and stops advertising its lifecycle services on command - the only way to reach a + node that leaves the managed set without either becoming healthy or leaving the graph. + + The `cap_pressure` scenario runs `tracked_node_cap: 1` against TWO required nodes, so one + is refused on every tick: it proves the refusal is visible on `GET /x-medkit-watchdog`, + that it WITHHOLDS `GRAPH_NODE_INACTIVE`'s clear (measured through `last_passed`, which + catches even a transient clear), and that the entry for a node that then LEAVES is + collapsed so the present, still-broken node is admitted and named. The + `unsettled_departure` scenario runs two healthy managed nodes and drops the lifecycle + services of each: one is killed immediately (a single missed sweep before a clean + shutdown - nothing may ever be reported about it) and the other holds the dropped state + past the settling budget before being killed (genuinely not managed when it left - it must + still be reported), with the first leg's own window MEASURED against the settling budget so + it cannot silently become the second. `wide_grace` configures the value that used to be + the accepted `grace` maximum and proves it is refused and the default applied - under it + the detector could neither raise nor heal for days. `restart_departed` records where "a + departure never heals a fault" ends: it kills the required node while its fault is + outstanding, restarts the gateway, and pins that the record then heals, because a restarted + detector cannot tell an entry for a departed node from a misspelt one. The main scenario drives real + `lifecycle_msgs/srv/ChangeState` transitions: the required node in its launch-default + unconfigured state raises `GRAPH_NODE_INACTIVE` on `GET /api/v1/faults` (and on the + entity-scoped `/apps/graph_watchdog/faults`), a real CONFIGURE leaves the fault raised + (inactive is still not active), a real gateway RESTART - SIGTERM, the port going down, + the relaunched process arming again - must not produce a single PASSED for the + still-inactive node, which is the withheld-clear guard at the only tier that can reach + it, and a real ACTIVATE finally arms the node's per-entity gate and heals it. The + default-config scenario launches with NO `lifecycle_expectation` config at all against + the same inactive node and holds a sustained silence window - the zero-false-positive + default, falsifiable nowhere else. The negative control runs the self-activating + variant of the same executable with the same grace and cadence, its `require_active` + naming that active variant, and holds the same window; the discriminating variable is + the node's actual lifecycle state. Both silence scenarios gate on three facts before + asserting absence and re-pin the last of them afterwards: the plugin is armed, `GET + /faults` answers 200 in this launch (a dead fault_manager answers 503 and polls to the + same `None` the assertion wants), and the target's label was actually READ via the + plugin's own `GET /x-medkit-watchdog` route - an unread label would produce the same + silence for the wrong reason. The `healing_threshold` scenario proves the + withheld-clear guard's pending leg against the REAL fault_manager debounce state + machine, not a fake sink: with `healing_threshold` set to 1 (the README's own + recommended value, see "Closing the loop"), the required node - already reported stuck + - is SIGTERM'd and respawned twice under the same name (a real snapshot blink, not a + lifecycle transition), each one comfortably inside the tracker's absence grace. The + instrument is the fault's `last_passed` and `status` fields read from `GET + /api/v1/faults`, the same instrument test_02b uses for the restart leg above: nothing + below e2e ever exercises the real debounce counter, only a fake `ReportFault` sink, so + only this tier can prove a blink never moves it. The same run is also this package's + only e2e coverage of a node oscillating readable/unreadable producing no + `GRAPH_NODE_INACTIVE` event churn: `GET /faults/stream` is read raw for the whole + blink sequence (each blink briefly re-seeds the required node's label, present but + unread, before it settles back) and checked for a `fault_cleared` frame for + `GRAPH_NODE_INACTIVE` (there must be none) and for every `fault_confirmed`/ + `fault_updated` frame for `GRAPH_NODE_INACTIVE` carrying `severity_label: "ERROR"` - + its only possible value now that severity is fixed per code rather than chosen from a + tick's mixed content. This scenario's blinks are far too short to ever convert the + required node into `GRAPH_NODE_UNREADABLE` content (that needs a REAL, sustained + 60-consecutive-tick `GetState` failure), so no `GRAPH_NODE_UNREADABLE` frame is + expected on this stream either. + + The fifth scenario, `unreadable`, is where that 60-tick failure is actually reached. + It launches `unreadable_lifecycle_node.cpp`: a plain `rclcpp::Node` that advertises + `get_state`/`change_state` with the service TYPES `find_lifecycle_get_state_path` + checks for, but whose `get_state` handler stores every request (via rclcpp's + deferred-response service callback) and never answers until its own + `start_answering` parameter is set true. Against that fixture the scenario proves, + through `GET /api/v1/faults` on the real stack: the node is present and matched with + its lifecycle label read as `""` (never answered, not "no data yet"); `GRAPH_NODE_INACTIVE` + stays silent for it the whole time - impossible by construction, proven live rather + than trusted from the source; `GRAPH_NODE_UNREADABLE` raises once the 60-tick hold + expires, names the node, and carries `SEVERITY_WARN`, with `GRAPH_NODE_INACTIVE` still + silent at that same moment; and once the fixture starts answering `"active"` the + watcher reads it through a real GetState round trip and `GRAPH_NODE_UNREADABLE` clears. + + The sixth scenario, `departure_keeps`, proves what a DEPARTURE does to an + already-reported unmeasured fault: nothing. It drives the same + `unreadable_lifecycle_node.cpp` fixture up to the same raised `GRAPH_NODE_UNREADABLE`, + then SIGTERMs the fixture process - permanently, no respawn - and confirms it is really + gone from `GET /apps` (the same `ThreadSafeEntityCache` the detector's own per-tick + snapshot reads) before asserting anything about the fault. Past every horizon that could + have discarded the node's evidence, the fault is still active, has never once been + reported PASSED (`last_passed`, which catches even a transient clear), and its + description now says the node has left the graph. `GRAPH_NODE_INACTIVE` never raises for + it at any point, before or after the kill, because the fixture is never told to start + answering anywhere in this scenario. Only this tier can show that the real discovery + layer noticing a process leave DDS does not walk the real fault_manager's debounce + counter toward HEALED. + + The seventh scenario, `not_managed`, is `GRAPH_NODE_NOT_MANAGED`'s own acceptance + gate, proving the sibling cause the unmeasured clock is blind to on the real stack: + `calibration` (`DEMO_NODE_REGISTRY`'s plain `demo_calibration_service`, no lifecycle + interface whatsoever) is present and matched from launch, `GRAPH_NODE_NOT_MANAGED` + raises once the 60-tick hold expires naming the node at `SEVERITY_WARN`, and neither + `GRAPH_NODE_INACTIVE` nor `GRAPH_NODE_UNREADABLE` ever raises for it - a node is + content of at most one of the three, and this is the one live proof that the + NOT-MANAGED cause specifically never bleeds into the UNREADABLE code it shares a clock + with. The departure leg mirrors `departure_keeps`'s own shape: SIGTERM the fixture, + confirm it is gone from `GET /apps`, and the fault is still there afterwards, never + PASSED, now describing a node that has left the graph. + + The eighth scenario, `restart_loop`, is the acceptance gate for the whole + evidence-retention model: the same plain `calibration` node, but respawning, SIGTERM'd + over and over on a cadence that never lets it accumulate the 60 consecutive PRESENT + ticks the unmeasured hold would otherwise need. Every cycle's absence is proven rather + than assumed - `GET /apps` is polled until the node is gone and again until it is back, + and the measured gap must exceed the 3-tick absence grace, on top of launch's own + enforced `respawn_delay` floor - so every cycle genuinely crosses the horizon past which + evidence used to be discarded. `GRAPH_NODE_NOT_MANAGED` must raise anyway, naming the + node at `SEVERITY_WARN`, and must survive the restarts that follow. A required node in a + crash loop is the case this detector most exists to catch, and it is the one that a + design discarding evidence on absence makes permanently silent. + ## Reliability (bringup-quiesce) Silent-fault detectors are prone to bringup noise: a node joining the graph, a @@ -615,10 +1229,13 @@ bringup-quiesce centrally so no individual detector has to reimplement it. discovery is briefly re-seeded, so an `active` transition lost during the subscription's endpoint-matching window self-heals instead of suppressing the node forever. Seeds are bounded per tick so a batch bringup cannot stall the - tick loop. Those `~/transition_event` subscriptions are kept out of the - gateway's ROS executor (own callback group, own single-threaded executor that - the plugin's tick thread drains between ticks), so they are only ever created, - run and destroyed on that one thread. Non-managed nodes are never gated. + tick loop. A tracked id whose binding moved (a different node behind the same + app id) is dropped and re-seeded from scratch rather than keeping the old + node's label. Those `~/transition_event` + subscriptions are kept out of the gateway's ROS executor (own callback group, + own single-threaded executor that the plugin's tick thread drains between + ticks), so they are only ever created, run and destroyed on that one thread. + Non-managed nodes are never gated. - **Clock validity.** `ctx.clock->time_is_valid()` flags a paused or absent `/clock` (e.g. a bag pauses or a sim crashes under `use_sim_time`). This is **detector-consulted, not centrally enforced**: a time-based detector @@ -645,7 +1262,14 @@ apart from a dead watchdog (503 if the gate was never initialized): "state": "armed", "lifecycle": "active" } - ] + ], + "detectors": { + "lifecycle_expectation": { + "tracking_saturated": false, + "tracked_nodes": 12, + "tracked_node_cap": 512 + } + } } } ``` @@ -654,6 +1278,16 @@ apart from a dead watchdog (503 if the gate was never initialized): is `active`; otherwise it is `"warming_up"`. `lifecycle` is the raw lifecycle state label, or `null` for entities with no tracked lifecycle state. +`detectors` carries one block per detector that has something of its own to report, and is +omitted entirely when none does - for a condition that belongs to a single detector and +would otherwise exist only in the gateway log. `lifecycle_expectation` reports whether its +tracked-node cap is SATURATED, i.e. whether it has refused to track a required node because +every slot is held by a present node carrying evidence. `tracking_saturated: true` means a +required node is going unchecked and `GRAPH_NODE_INACTIVE`'s clear is withheld for as long +as it lasts; the fix is either a `require_active` entry matching fewer identities (a bare +name on a graph whose nodes respawn under ever-new namespaces matches an unbounded set) or a +larger `tracked_node_cap`, which is why the cap and the live count are reported beside it. + **What detectors see.** Nothing changes in how a detector raises or clears a fault - the gate is applied transparently inside `ctx.raise_fault()` itself. The two things a detector opts into explicitly are @@ -681,5 +1315,25 @@ adding a per-detector unit test is the one shared touch - it adds an Frozen in `include/ros2_medkit_graph_watchdog/graph_fault_codes.hpp`: `GRAPH_QOS_MISMATCH`, `GRAPH_ORPHAN`, `GRAPH_NODE_DISAPPEARED`, `GRAPH_TF_STALE`, -`GRAPH_PARAM_DRIFT`, `GRAPH_LATENCY_BUDGET`, plus one extension of the frozen -namespace for a new capability beyond the original six: `GRAPH_NODE_INACTIVE`. +`GRAPH_PARAM_DRIFT`, `GRAPH_LATENCY_BUDGET`, plus three extensions of the frozen +namespace beyond the original six, all raised by `lifecycle_expectation`: +`GRAPH_NODE_INACTIVE` (a required node CONFIRMED non-active), `GRAPH_NODE_UNREADABLE` +(a required, managed node whose lifecycle label has never been read), and +`GRAPH_NODE_NOT_MANAGED` (a required node with no tracked lifecycle at all). The latter +two share one cause-blind unmeasured clock internally (see the detector section above) +but are always two DISTINCT codes on the wire - the clock's blindness to which cause it +is seeing never leaks into which fault code a node ends up reported under. + +Splitting the unmeasured cases out of `GRAPH_NODE_INACTIVE` has two consequences nothing +else states. Anything downstream that filters or correlates on `GRAPH_NODE_INACTIVE` +alone no longer sees either unmeasured case at all - they live under +`GRAPH_NODE_UNREADABLE`/`GRAPH_NODE_NOT_MANAGED` now, different codes, not a +WARN-severity instance of the first. And a node moving from confirmed-inactive to +unmeasured (or back), or from one unmeasured cause to the other while its clock is still +climbing (before maturity), produces a HEAL on one code and a RAISE on another for the +SAME node, on adjacent ticks - separate fault_manager events, not one transition, +because none of the three faults share a record to transition within. A node whose +unmeasured clock has already MATURED under one cause does not flip fault codes merely +because the LIVE cause changes later, though (the sticky-cause design decision, see +above) - only a real measurement resetting the clock entirely can move it off the code +it matured under. diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst index 256eae2ff..b51f8dc3b 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst @@ -75,6 +75,11 @@ Reliability core lost during the subscription's DDS endpoint-matching window self-heals rather than suppressing the node for the process lifetime; the blocking seeds are bounded per tick so a batch bringup cannot stall the tick loop. + An entry's identity is its BINDING - the pair (fqn, ``GetState`` path) - not + its ``App::id``, which can survive a graph sweep while pointing at a + different node. When either half moves, the entry is dropped (the old + binding recorded as departed under its own fqn) and re-seeded from scratch, + so a moved binding can never keep enforcing the old node's label. The subscription callback holds only a ``weak_ptr`` to the watcher's shared state, so a callback in flight during teardown bails instead of touching freed memory. Non-managed nodes are never gated - ``node_ok`` returns true @@ -88,6 +93,7 @@ Reliability core would otherwise be delayed by up to a whole tick interval, and a bringup burst could overrun the subscription's queue and lose the intermediate transitions the departure classification reads. + - **Central enforcement.** ``DetectorContext::raise_fault`` runs every raise through ``reliability_allows(gate, source_id)`` before the fault client sends anything; a detector raising about a still-warming-up entity or a @@ -114,6 +120,16 @@ Reliability core entry per known entity (``id``, ``first_seen_tick``, ``armed``, ``state``, ``lifecycle``). Returns 503 with ``ERR_SERVICE_UNAVAILABLE`` if the gate has not been constructed yet or has already been torn down by ``shutdown()``. + + Beside the gate's own state the payload carries a ``detectors`` object, one block per + detector whose ``Detector::status_json()`` returns something, omitted entirely when none + does. It exists for a condition that belongs to a SINGLE detector and would otherwise live + only in the gateway log, which no HTTP client and no e2e assertion can read - + ``lifecycle_expectation`` reports ``tracking_saturated`` (it has refused to track a + required node) beside the live ``tracked_nodes`` count and the ``tracked_node_cap`` in + force. The handler runs on an HTTP thread while ``tick()`` runs on the tick thread and + holds no lock a detector takes, so a ``status_json()`` implementation must build its + payload from atomics and must not block. - **Build note.** ``LifecycleWatcher`` reuses the gateway's own lifecycle-state helpers (``lifecycle_status_helpers.cpp``, ``ros2_lifecycle_state_reader.cpp``) compiled in via ``GATEWAY_SRC_DIR`` - @@ -122,9 +138,9 @@ Reliability core Detectors --------- -``qos_mismatch``, ``orphan`` and ``param_drift`` are the detectors this package ships so -far. The remaining silent-fault classes land in follow-up changes, each against its own -issue. +``qos_mismatch``, ``orphan``, ``param_drift`` and ``lifecycle_expectation`` are the +detectors this package ships so far. The remaining silent-fault classes land in +follow-up changes, each against its own issue. ``qos_mismatch`` raises ``GRAPH_QOS_MISMATCH``. It watches every topic's publisher/subscriber QoS pairs rather than parameter values: each @@ -349,9 +365,510 @@ ROS parser types as a YAML 1.1 boolean; those three prove configuration delivery suppression only, and assert no clear. +``lifecycle_expectation`` raises three independent faults, ``GRAPH_NODE_INACTIVE``, +``GRAPH_NODE_UNREADABLE`` and ``GRAPH_NODE_NOT_MANAGED``. It watches an operator-declared +set of ``require_active`` node names rather than topics, parameters, or presence. Each +tick it matches every configured entry against the live apps by ``App::id`` OR the +stable ``effective_fqn()`` OR that fqn's bare leaf name - ``App::id`` alone is +recomputed each sweep and gets namespaced on a same-bare-name collision, so an id-only +match would silently stop covering a multi-robot graph; a bare name therefore matches +that node in every namespace (use a full FQN to pin one). For each matched present node +it reads the node's raw lifecycle label via ``ReliabilityGate::lifecycle_state_of()`` +and hands the matches - a ``std::vector`` of (entry, node fqn, state) - +to ``lifecycle_expectation_tracker.hpp``'s ``LifecycleExpectationTracker``, the +detector's pure, ROS-free core, independently unit-tested before the detector ever +touches a live gate. + +**The model: one observed state per node per tick, two clocks - not three fault codes +each running their own bookkeeping.** All three faults are outputs of a SINGLE per-node +state machine inside the tracker. Every tick, a matched node is classified into exactly +one of ``ACTIVE``, ``INACTIVE`` (non-empty label, not ``active``), ``UNREADABLE`` (a +managed node whose label has never been read - ``optional("")``), or ``NOT_MANAGED`` +(``nullopt`` - no tracked lifecycle at all); a node not matched at all this tick is a +fifth case, ``ABSENT``, handled by a wholly separate path. Two clocks track history: + +- The **violation streak** advances on ``INACTIVE``, resets ONLY on ``ACTIVE``. Past + ``grace`` the node is ``GRAPH_NODE_INACTIVE``'s content. +- The **unmeasured clock** advances on ``UNREADABLE`` OR ``NOT_MANAGED``, resets ONLY on + a real measurement (``ACTIVE`` or ``INACTIVE``). It is deliberately BLIND to which of + the two causes it is seeing on any given tick - a node alternating between "matched but + never read" and "not a tracked lifecycle node at all" keeps this ONE clock climbing + instead of each cause resetting a separate counter of its own. That blindness closes an + entire class of alternation, not one more special case for it: three review rounds + each found a pair of "I cannot measure this" causes whose separate counters could erase + each other's progress, leaving a node invisible to every code that existed. Past + ``unmeasured_hold_ticks`` (a fixed 60 ticks, not configurable), the clock MATURES and + takes ownership of the node. + +**Ownership is exclusive.** A node is content of at most one of the three faults at a +time. The moment the unmeasured clock matures, the violation streak is RELEASED - +actually reset to 0, not merely excluded from ``GRAPH_NODE_INACTIVE``'s content this +tick - so that fault's clear is free to flow immediately; a node returning to +``INACTIVE`` afterwards re-earns ``grace`` from zero. Which of the two unmeasured codes +a matured node reports under is STICKY: it keeps reporting under whichever code the +clock matured under even if the LIVE cause later flips, and only a real measurement +(resetting the whole clock) changes it. The rejected alternative, current-cause-wins, +is more literal but makes a node that flaps between the two causes flap the fault +surface too - a raise/clear/raise churn across two codes for a node whose actual +situation never changed. + +**Absence continues the last CORROBORATED observation, it never erases.** A node not matched +at all this tick is ABSENT, a fact separate from any observed state (there is no label to +classify). For up to ``absence_grace`` (a fixed 3 ticks) consecutive absent ticks a node's +whole state is held unchanged - the blink tolerance. Past ``absence_grace`` absence ADVANCES +whichever clock the node's SETTLED observation had started, and resets nothing: a +settled-``INACTIVE`` node keeps climbing its violation streak, a +settled-``UNREADABLE``/``NOT_MANAGED`` node keeps climbing its unmeasured clock under that +same cause, and a settled-``ACTIVE`` node advances nothing at all - anything it had started +but not corroborated is released, so the entry becomes idle and is reclaimed silently. So a +departure never heals a fault and never starts one; what it changes is the detail phrase, +which then says the node has since left the graph. + +**What "settled" means, and why an unmeasured reading needs corroborating.** A real +measurement (``ACTIVE`` or a non-active label) settles at once - a label is a fact about the +node, and no discovery artifact invents one. An UNMEASURED reading settles at once too on a +node nothing has ever measured, since it is then the only thing known about it (this is what +keeps a crash-looping node accumulating toward its report rather than being released on every +absence). Only when an unmeasured reading would OVERRIDE a real measurement must it first +hold for ``kDefaultObservationSettleTicks`` (6) consecutive matched ticks. The transient it +guards against is real and cheap to hit: ``LifecycleWatcher::update()`` drops a tracked id +whose ``get_state`` path is absent from the current sweep, and ``discover_apps()`` can yield +an app with no services when a sweep races service enumeration - so a present, healthy, +managed node reads ``NOT_MANAGED`` for a tick. If that tick is the last one before a clean +shutdown, continuing "the last observation" literally would mature a healthy departure into a +permanent ``GRAPH_NODE_NOT_MANAGED``. Six is the same bar the typo warning already sets +against the same transient, and a tenth of the 60-tick unmeasured hold, so R30 is untouched: a +node that is genuinely unmeasurable when it leaves corroborated that long before the hold that +reports it. + +**Where "a departure never heals a fault" ends.** It holds within one gateway lifetime. Across +a RESTART it does not: the restarted tracker has no measurements, the departed node is not in +the graph, and its ``require_active`` entry matches nothing - which this detector cannot tell +apart from a misspelt entry, because the only component that knows the difference is the fault +store and it is not read at startup. The never-matched hold is deliberately bounded (a typo +must not block healing forever), so once it lapses the level-triggered clear flows and the +record heals with nothing having been measured. Re-seeding the tracker from the fault store at +startup would change that and is not implemented; the boundary is pinned by the +``restart_departed`` e2e scenario rather than left to be discovered. + +Why the erasure went away rather than moving: every erasure horizon is an evasion for a +node that touches it periodically. A node in a restart loop - start, crash, respawn delay, +start - touches absence by construction, and that is the node this detector most exists to +catch, so discarding its evidence let it alternate ``(UNREADABLE, ABSENT x N)``, +``(NOT_MANAGED, ABSENT x N)`` or ``(INACTIVE, ABSENT x N)`` forever without ever +accumulating enough of anything to be reported. Reporting a HEALTHY node that left remains +the presence class's job (``GRAPH_NODE_DISAPPEARED``, still no detector of its own) - the +single gap this class keeps, and a far narrower one than "a node absent past the grace is +invisible whichever clock it was on". + +**Content follows the clocks, not the snapshot.** Whether a node was in this tick's matches +decides nothing about what it reports: a node past ``grace`` stays in +``GRAPH_NODE_INACTIVE``'s content while it blinks, and a matured node stays in its own +code's content. A fault's content is what has been MEASURED about the node, and a missing +snapshot entry measures nothing either way. + +**Presence is not enough here - and absence is someone else's fault.** A +present-but-non-active managed node (``inactive``, ``unconfigured``, ``finalized``) is +exactly the case ``GRAPH_NODE_INACTIVE`` exists to catch: on a Nav2 stack, a +``controller_server`` stuck ``inactive`` means the robot silently will not act, with +no crash and no signal on ``/diagnostics`` or ``/rosout``. A node that vanishes having +only ever been measured HEALTHY is the presence fault class's business, and this detector +raises nothing for it; a node that vanishes while already reported under any of the three +faults KEEPS that fault, because leaving the graph answers nothing the operator asked - +the integration test's ``AbsenceAfterRaiseKeepsTheFaultAndSaysTheNodeIsGone`` and +``HealthyNodeThatVanishesRaisesNothingAtAll`` pin the two halves of that split. +**Safe default: off.** ``require_active`` is +empty by default, so the detector emits nothing at all - not even clears - until an +operator opts specific nodes in (zero false positives, the same config-scoped posture +``param_drift``'s ``expect`` uses). + +**Keyed by node, not by config entry.** The bare-name form of ``require_active`` is +deliberately fleet-wide ("all ``controller_server``\ s must be active"), so one entry +legitimately covers N nodes. The tracker therefore keys every fact by the node's +stable fqn: two namesakes are two report entries (one cannot silently replace the +other, and a healthy one cannot mask a broken one), and the fault description names +each offending node's FQN with the entry that demanded it as context - the +description is the only carrier, since every ``GRAPH_*`` fault shares one +``source_id``. Two entries naming the SAME node advance its clocks once per tick, not +once per matching entry. + +**Why grace, not the reliability gate (the design decision this detector turns on).** +Every detector's raise is subject to the central ``reliability_allows(gate, +source_id)`` gate inside ``ctx.raise_fault()``, which ANDs the entity's warmup state +with ``LifecycleWatcher::node_ok()`` - true only when a managed node's lifecycle is +``active`` (or it is not tracked at all). Gating THIS detector's raise the same way on +the required node's OWN lifecycle would be self-defeating: ``node_ok()`` is exactly +FALSE for the inactive node this detector exists to report, so the central gate would +suppress the signal forever. Instead the tracker counts consecutive not-active ticks +itself, per node, entirely independent of ``reliability_allows``. This does not bypass +bringup-quiesce entirely: the aggregated fault's own outer ``ctx.raise_fault`` call is +still subject to the central gate for the aggregate's own ``source_id`` +(``graph_watchdog``) - only the per-node lifecycle check bypasses it, because that check +IS what this detector reports on. + +**Bounded by evidence, not by age.** The map is NOT simply bounded by the +operator-declared ``require_active`` set: keys are node fqns, and one bare-name entry +matches a node of that name in every namespace, so identity churn (nodes reappearing under +ever-new fqns) would grow it without a bound. Since absence no longer erases anything, an +entry carrying a live clock is not reclaimed by age either - those are the same defect seen +twice, and moving a horizon rather than removing it leaves the evasion in place at a +different N. So ``prune_ticks`` (the operator's ``prune_grace``, used as written - there is +no ``grace + 1`` clamp any more, because there is nothing left for one to protect) reclaims +IDLE entries only: both clocks at zero and no matured ownership, i.e. nothing to lose, and +still atomically - ONE map entry per node, gone in the same tick, never partially. A +non-idle entry is never pruned by age, and cannot grow without bound in time either: past +the absence grace its clock advances every tick, so it matures within at most +``grace + absence_grace + 1`` ticks (a violation streak) or ``60 + absence_grace + 1`` (an +unmeasured clock) and is reported. Those are also the longest ``GRAPH_NODE_INACTIVE``'s clear +can be withheld by one departed node, which is why ``grace`` is capped at 300 instead of +being accepted up to ``INT_MAX - 1``: at the old maximum the bound was roughly 24 days at the +shipped cadence, during which the fault could neither raise nor heal for ANY node - silence +indistinguishable from a working detector finding nothing. + +What bounds the map is ``tracked_node_cap`` (default 512, ``kDefaultTrackedNodeCap``, +accepted range 1..16384): at the cap idle entries are reclaimed first, then entries for +DEPARTED nodes are collapsed into per-code COUNTS - lexicographically last first, keeping at +most ``kMaxNamedDepartedEntries`` (3) named, since three maximally-long details are all one +480-character description holds - and only if every tracked node is PRESENT and carrying +evidence is the NEWCOMER refused, never a live violation evicted. + +Collapsing follows from the fault being keyed by CODE rather than by node: five hundred +entries for dead identities keep the same one fault raised that a single entry would, so +holding them buys nothing while the slots they occupy can cost total blindness. Under +identity churn a cap full of the dead would refuse a genuinely broken PRESENT node - which +then never reaches ``affected`` or ``pending``, is not covered by the never-matched hold +either (its entry HAS matched), and so ``GRAPH_NODE_INACTIVE`` would emit a level-triggered +CLEAR every tick while that node sat there not-active. The count is content, so collapsing an +entry heals nothing; it is ordered ahead of the individually named entries but behind +anything crossing on this tick, so a node that just broke is never displaced by it. It only +grows within one tracker lifetime: a collapsed entry's fqn is no longer known, so a node +returning under it is tracked and measured afresh. + +A refused node is a required node going unchecked, so refusing one is never silent: it +withholds ``GRAPH_NODE_INACTIVE``'s clear for as long as it lasts, is logged once per +saturation EPISODE (the latch re-arms when the episode ends, so a later, real saturation is +not silenced by an earlier one), and is reported on ``GET /x-medkit-watchdog`` under +``detectors.lifecycle_expectation``. That withhold is deliberately UNBOUNDED, unlike the +never-matched hold beside it: the never-matched hold is bounded because a typo must not block +healing forever, while saturation - once departed entries can no longer crowd out present +ones - means genuinely more required PRESENT nodes than the cap allows, a capacity condition +the operator resolves rather than a transient that resolves itself. + +**The clear is withheld until the required set has actually been measured - and this is +entirely about** ``GRAPH_NODE_INACTIVE``. The other two faults have no withheld-clear +guard of their own; see "Three independent faults, not one shared record" below for why +they don't need one. A clear asserts that every required node is free of a CONFIRMED +violation. That is the restart-heal hazard: a gateway restart brings every detector +counter back to zero while the ``GRAPH_NODE_INACTIVE`` raised before it is still in the +fault_manager's store (a separate process), so a clear emitted on the strength of an +empty affected map heals a fault that is still real. TWO things produce that empty map +without the assertion being true, both the ordinary state of bookkeeping that just +started over (a restarted gateway, a reconfigure - ``configure()`` rebuilds the tracker +- or a node that respawned stuck): + +- **Not matched yet.** Before the entity snapshot catches up with the graph a + ``require_active`` entry matches nothing at all, so the clear is about a node the + detector has never once looked at. The plugin ticks as soon as it is loaded, so every + restart passes through this window. Bounded the same way node-keyed state is (60 + ticks), so a misspelt entry cannot block healing for the process lifetime. +- **A node's status is UNSETTLED** (the tracker's own ``pending`` set). A violation streak + that has not yet passed ``grace``, an unmeasured clock still climbing under EITHER cause, + or a streak HELD while the node is inside an unmeasured spell. Absence never puts a node + here on its own - content follows the clocks, so a node already past ``grace`` stays in + the fault's content through a blink rather than dropping into a withheld limbo. A node whose + unmeasured clock has MATURED is deliberately NOT in this set - ownership passed to its + own fault code and the violation streak was released, so it stops counting toward this + withhold the exact tick it stops being uncertain, rather than continuing to poison + ``GRAPH_NODE_INACTIVE``'s withhold decision the way a shared record used to. + +Either reason withholds the emission entirely, neither raise nor clear. A raise is never +withheld: a violation read from the nodes that DID answer is real regardless of the +unmeasured ones. Every hold is bounded: the never-matched leg and both unmeasured +causes all release after 60 consecutive ticks (a minute at the shipped cadence, +mirroring ``param_drift``'s frozen hold), whether the node is present or gone - the pending +leg releases as soon as the node reads ``active``, or its streak passes grace and the fault +is raised again, which past the absence grace happens while the node is absent too. Every +hold releases by SETTLING the node's status, never by giving up on it. Because a correctly +withheld clear and a detector with nothing to report look identical from outside, a hold +that lives past 10 consecutive ticks is explained in the log once per episode, naming +every reason in force and, for the node-keyed ones, the count behind each and one node +by name - the not-managed and unreadable reasons are still named separately even though +both now release the same way (into their own fault code), since an operator reading the +log wants to know WHICH of the two is happening. + +**Three independent faults, not one shared record.** ``GRAPH_NODE_INACTIVE``, +``GRAPH_NODE_UNREADABLE`` and ``GRAPH_NODE_NOT_MANAGED`` are each raised through the +shared ``AggregatedFault`` helper every ``GRAPH_*`` detector uses (one graph-level +record per code, since the fault_manager identifies a fault by ``fault_code`` alone), +but as three SEPARATE, fixed-severity class members - ``GRAPH_NODE_INACTIVE`` always +``SEVERITY_ERROR``, the other two always ``SEVERITY_WARN`` - the same shape +``orphan_detector`` and ``param_drift_detector`` use, rather than one record whose +severity is chosen from that tick's mixed content the way +``qos_mismatch_detector``'s ``any_starved ? kStarvedSeverity : kPartialSeverity`` does +for its own single code. Raises are fully independent: each fault's content comes from +its own measurement, and one raising, healing, or changing severity never forces, +blocks, or reflects onto another. ``GRAPH_NODE_INACTIVE``'s own clear is not simply +"nothing CONFIRMED non-active this tick" though - it is withheld exactly as described +above; that is the ORIGINAL withheld-clear guarantee this detector always gave, scoped +to ``GRAPH_NODE_INACTIVE`` alone. The other two have no such guard: each one's own +clear needs nothing beyond its own content going empty, because once the unmeasured +clock has matured "still cannot be measured" is a settled fact, not a pending one. A +node is content of at most one of the three at a time, and healing one never forces, +blocks, or changes the severity of another. Content under either unmeasured code +survives for as long as a node stays that way, with no further bound past the initial +hold. + +**A re-bind is a fresh binding.** ``LifecycleWatcher`` keys its tracked map by +``App::id``, and an id can survive a graph sweep while pointing at a DIFFERENT node +(id assignment shifts under bare-name collisions) - an entry kept across such a move +would keep enforcing the old node's label, and its old ``~/transition_event`` +subscription, against the new binding. An entry's binding identity is its fqn plus +its ``GetState`` service path, both captured at first sighting, and ``update()`` +re-checks that identity every tick. A moved binding is two events at once: the OLD +binding departed (recorded under ITS OWN fqn in ``recently_departed_``, same record +and retention as a vanish), and the id is new again (erased and re-seeded through the +ordinary new-node path: fresh ``GetState``, fresh subscription, fresh self-heal +budget). For this detector the consequence is that a re-bind is never enforced with the +departed node's label: the new binding starts unknown (benign) until its own label is +read. + +No straggler from the old subscription has to be filtered out of the new entry. Erasing +the entry destroys that subscription, on the tick thread: the private executor that +runs these callbacks is pumped by the same thread that runs ``update()`` (see the +plugin-shell bullet above), so nothing else holds a reference to it and no message it +had queued is delivered afterwards. The same property removes the other ordering +question this seam used to carry - a re-seed's blocking ``GetState`` cannot be +overtaken by a ``~/transition_event``, because while ``update()`` blocks nothing is +pumping the events at all. + +**New violations are named first, not alphabetically - in EACH fault's own description +independently.** The description used to list affected nodes in fqn order +(``AggregatedFault::emit``'s default), which is fine when every entry is equally +interesting but not once the cap is full: a fleet sharing +``require_active: ["controller_server"]`` across a dozen robots fills the 480-char cap +from the alphabetically-earliest ones, and a THIRTEENTH robot going inactive afterward +would be silently invisible forever - one shared ``fault_code``, one record, no way to +tell the operator which of thirteen actually broke. The tracker reports which fqns +entered EACH fault's content on THIS tick - ``newly_affected``, ``newly_unreadable``, +``newly_not_managed`` - and each list orders only its OWN fault's +``AggregatedFault::emit_ordered`` call, since the three faults never share a +description: a fresh entry is named FIRST in whichever fault it belongs to; every other +affected node in that same fault still appears, in the same fqn order as before, once +the fresh ones are placed. When every node crosses on the same tick (``grace: 0`` +during a bringup burst, for instance), there is nothing to distinguish them by and the +order degrades to fqn order. Two budgets protect each description, applied before it +ever reaches the 480-char cap: the lifecycle label - which arrives verbatim off a +remote ``~/transition_event`` and is therefore untrusted and unbounded, and only ever +appears in ``GRAPH_NODE_INACTIVE``'s own detail, never the two unmeasured faults' - +is trimmed to 32 characters before it is interpolated (more than double the longest +label a conforming implementation produces, ``errorprocessing`` at 15 characters), and +the whole per-node detail is then capped at 150 characters as a backstop against a +pathological fqn or a long ``require_active`` "required by" list, sized so at least +three worst-case details still fit inside the 480-char cap +(``3 * 150 + 2 * 2 = 454 <= 480``). + +**Test tiers.** Three tiers each prove a different layer, deliberately not +overlapping - plus the shared-watcher seam the re-bind behaviour lives in: + +1. **Unit** (``test_lifecycle_expectation_tracker.cpp``): pure + ``LifecycleExpectationTracker`` logic over hand-built matches. Covers the violation + streak (stuck-inactive past grace raising keyed by the node with the entry as + context, active never raising, reaching active within grace resetting the streak, + both namesakes reported separately, two entries not halving the grace, a violating + read winning a duplicate match's tie-break); the unmeasured clock shared by + UNREADABLE and NOT_MANAGED (neither ever confirms a violation while climbing; each + matures into its OWN fault at the exact hold boundary; the clock resets ONLY on a + real measurement, in both directions; maturity RELEASES the violation streak + entirely, so a node returning to inactive re-earns grace from zero; the sticky-cause + decision, pinned directly against the rejected current-cause-wins alternative); the + alternation this redesign closes - a node alternating between unreadable and + not-managed, indefinitely and on every single tick, still matures the shared clock; + the ``INACTIVE``/``NOT_MANAGED`` alternation across absence gaps longer than the + absence grace; the violation streak surviving non-maturing unmeasured ticks and + RESUMING rather than restarting, counted exactly and read through + ``pending_violation``; absence CONTINUING whichever clock the last real observation + started - a blink holds it unchanged, past the blink tolerance it advances, a matured + fault survives a departure, a node measured ACTIVE that vanishes raises nothing and is + reclaimed, and the three ``(X, ABSENT x N)`` restart-loop shapes are swept at the + absence grace and past it; the ``pending`` set and its per-reason breakdown; new-first + ordering for all three fault-shaped maps; the remote-supplied label's own trim budget + ahead of the whole-detail backstop, reapplied to a matured unreadable node's detail + (which carries no label, only a fqn and a "required by" list); the age horizon + reclaiming IDLE entries only, atomically, and never one that carries evidence even + when it undercuts the absence grace; the SETTLING rule that decides what absence may + continue - an uncorroborated unmeasured run before a healthy departure raising nothing, + swept across every run length below the bound and with the entry released rather than + left holding the clear hostage, the same run one tick longer still being reported, and a + single MEASURED not-active read before a departure still confirming; and the tracked-node + cap - idle entries reclaimed first, departed entries collapsed into a count so a present + broken node is always admitted and reported, at most three left named, the count + surviving into the description as content, a node returning after its entry was collapsed + measured afresh, the newcomer refused only when every entry is PRESENT and carrying + evidence, and saturation reported as a LEVEL on every refused tick with its edge + re-arming when an episode ends - swept at a shrunk cap and again at the real shipped 512. + The re-bind seam is shared infrastructure and is pinned separately in + ``test_lifecycle_watcher.cpp``. +2. **Integration** (``test_lifecycle_expectation_integration.cpp``): the detector + driven against a fake ``ReportFault`` service, with a REAL ``ReliabilityGate`` + arming the global bringup grace and feeding the detector its labels through + ``lifecycle_state_of()`` - injected via ``set_lifecycle_state_for_test()`` strictly + AFTER the last ``gate.update()`` call for most cases, or, for the alternation and + not-managed cases (which need a genuine ``nullopt`` the injection seam cannot + produce - it can only ever SET a tracked value, never remove tracking), by toggling + whether the matched app carries lifecycle services and driving the gate's real + discovery path instead. The fixture cases cover the ``GRAPH_NODE_INACTIVE`` + raise/clear round trip naming the stuck node; the active-from-arming positive + control; the fault SURVIVING the required node's departure while a healthy node's + departure raises nothing at all; bare-name and full-FQN matching against a + namespaced app; the zero-config default measured as zero fault_manager requests of + any kind; the unmanaged-entry and no-match warnings; the withheld-clear guard's + releases and its once-per-episode log line; the blink-plus-unread-re-seed sequence; + a filler batch sized (from the real detail-building code) to exceed the 480-char cap + aggregating into one fault with the fresh crossing named first; a PRESENT node crossing + on the same tick as a batch of departed ones being named ahead of them rather than + truncated away by them; a required node appearing mid-run; a re-bind under the same + ``App::id``; and the reconfigure/config-validation edge cases. The unmeasured clock's own split is pinned directly, + for BOTH codes symmetrically: a managed node whose ``GetState`` genuinely never + answers (through ``set_managed_app`` and a real, failing seed - proven by asserting + ``lifecycle_state_of()`` actually returns ``optional("")`` first, not assumed) is + reported under ``GRAPH_NODE_UNREADABLE``; a node with no tracked lifecycle at all is + reported under ``GRAPH_NODE_NOT_MANAGED`` (no longer released into silence, the + deliberate behaviour change this redesign makes); either hold releases into its + report on the exact tick past its bound; a node already reported under one of the + two clears once genuinely read, returning to ordinary ``GRAPH_NODE_INACTIVE`` + tracking; a confirmed node healing while an unreadable OR not-managed sibling stays + present clears ``GRAPH_NODE_INACTIVE`` promptly without touching either unmeasured + fault; content under either code survives with no window and no expiry; 25 nodes of + either cause aggregate into one capped fault at ``SEVERITY_WARN``; a node whose hold + expires opens its fault's own description over an already-reported filler batch with + new-first ordering; an already-reported node of either cause KEEPING its own record + once it vanishes, with its description switching to say the node has left the graph; a + node returning from that absence staying reported with no clear/re-raise churn; each of + the three restart-loop shapes raising its own code, and the ``inactive``/``not-managed`` + alternation across absence gaps; a withheld ``GRAPH_NODE_INACTIVE`` clear releasing when + the absent node's clock MATURES into its sibling's content rather than when the node is + given up on; the two wire strings pinned as hand-typed literals; and the independence + claim in both directions. + + The ``configure()``-level cases pin the config contract: the unknown-key warning for + ``require_activ`` (the worst-case typo - it also leaves the detector unconfigured, so + the warning must precede the zero-config early return), a fully-valid config + producing zero warnings (including ``grace: 0`` and ``prune_grace: 0``, the + documented low endpoints), negative, non-integer, past-the-int-range and + exact-``kMaxGrace``-boundary ``grace`` (both sides), ``prune_grace`` out of 0..3600 + rejected on the WIDE integer (never truncated into a hair-trigger prune) plus both + range endpoints accepted, non-array ``require_active`` and empty-string and + non-string entries each warning, the unclamped prune horizon reaching idle bookkeeping + at exactly the configured ``prune_grace`` (0, 1 and 4 - the smallest positive value + included, since neither documented endpoint sweeps it) while a node carrying evidence + survives it - including the ``grace: 0, prune_grace: 0`` corner and a wide ``grace`` + beside the tightest ``prune_grace``, whose instrument is the CONFIRMATION rather than the + map size - ``tracked_node_cap`` validated at both range endpoints and one value past each + with the key proven IN FORCE at both ends, boundedness under identity churn at the real + 512-node cap by collapsing the departed rather than refusing the live node, and saturation + reported once per EPISODE with a second episode reported again after the first ends. +3. **E2e** (``test/e2e/test_lifecycle_expectation_e2e.test.py``): the acceptance + gate - one source file, TWELVE CTest targets (the config-plumbing pattern: the + plugin reads its config once at ``set_context()``, so different configs need + different gateway launches), each bringing up a real gateway with the plugin + ``.so`` and a real fault_manager, asserting on the operator-visible + ``GET /api/v1/faults`` surface. Six scenarios drive the ``managed_lifecycle`` + demo node (a real ``rclcpp_lifecycle::LifecycleNode``, which always answers + ``GetState``); two, ``unreadable`` and ``departure_keeps``, drive a fixture built to + never answer it (see below); the last two, ``not_managed`` and ``restart_loop``, drive + ``calibration`` - a PLAIN demo service node with no lifecycle interface at all, so no + purpose-built fixture was needed for ``GRAPH_NODE_NOT_MANAGED``, unlike + ``GRAPH_NODE_UNREADABLE``. + The main scenario proves raise-survives-CONFIGURE-survives-RESTART-heals-on-ACTIVATE + through real ``lifecycle_msgs/srv/ChangeState`` transitions, including the + entity-scoped ``/apps/graph_watchdog/faults`` surface. The restart leg is the + withheld-clear guard at the only tier that can reach it: the gateway is SIGTERMed, + its port is waited down, the relaunched process is gated on being armed again, and + the fault about the still-inactive node must come back with ``last_passed`` unset. + The default-config scenario launches with no ``lifecycle_expectation`` config at all + against the same inactive node and holds a sustained silence window, and the + negative control does the same with the self-activating variant of the same + executable; the discriminating variable is the node's actual lifecycle state. Both + silence scenarios gate on three facts before asserting absence: the plugin is + armed; ``GET /faults`` answers 200 in THIS launch; and the target's label was + actually READ, pinned through the plugin's own ``GET /x-medkit-watchdog`` route. + + The fourth scenario, ``healing_threshold``, is the withheld-clear guard's pending + leg at the only tier that runs the REAL fault_manager debounce state machine at + all. The required node, already reported stuck, is SIGTERM'd and respawned twice + under the same name (a real snapshot blink, not a lifecycle transition), each one + comfortably inside the tracker's fixed absence grace, against a + ``healing_threshold`` of 1. This same run is also this package's only e2e coverage + for a node oscillating readable/unreadable producing no ``GRAPH_NODE_INACTIVE`` + event churn, read raw off ``GET /faults/stream``: no ``fault_cleared`` frame for + ``GRAPH_NODE_INACTIVE`` at any point, and every ``fault_confirmed``/ + ``fault_updated`` frame carrying ``severity_label: "ERROR"`` - its only possible + value now that severity is fixed per code. + + The fifth scenario, ``unreadable``, is where the 60-consecutive-tick failure that + matures ``GRAPH_NODE_UNREADABLE`` is actually reached. It launches + ``unreadable_lifecycle_node.cpp``: a plain ``rclcpp::Node`` that advertises + ``get_state``/``change_state`` with the service TYPES ``find_lifecycle_get_state_path`` + actually checks for, whose ``get_state`` handler stores every request and never + answers until its own ``start_answering`` parameter is set true. Against that + fixture the scenario proves the node is present and matched with its lifecycle + label read as ``""``; ``GRAPH_NODE_INACTIVE`` stays silent for it throughout; + ``GRAPH_NODE_UNREADABLE`` raises once the hold expires, names the node, and carries + ``SEVERITY_WARN`` - a node is content of at most one of the three, never more than + one; and once the fixture starts answering ``"active"`` the watcher reads it through + a real ``GetState`` round trip and ``GRAPH_NODE_UNREADABLE`` clears. + + The sixth scenario, ``departure_keeps``, proves what a DEPARTURE does to an + already-reported unmeasured fault: nothing. The same fixture is SIGTERM'd permanently + once ``GRAPH_NODE_UNREADABLE`` has raised, confirmed gone from ``GET /apps``, and past + every horizon that could have discarded its evidence the fault is still active, has + never once been reported PASSED (``last_passed``, which catches even a transient + clear), and its description now says the node has left the graph. The seventh, + ``not_managed``, is the sibling proof for the OTHER cause: ``calibration`` is present + and matched from launch, ``GRAPH_NODE_NOT_MANAGED`` raises once its hold expires naming + the node at ``SEVERITY_WARN``, and neither ``GRAPH_NODE_INACTIVE`` nor + ``GRAPH_NODE_UNREADABLE`` ever raises for it - the one live proof that the NOT-MANAGED + cause specifically never bleeds into the UNREADABLE code it shares a clock with. Its + departure leg mirrors ``departure_keeps``'s own shape. + + The eighth scenario, ``restart_loop``, is the acceptance gate for the whole + evidence-retention model. The same plain ``calibration`` node, but respawning, is + SIGTERM'd over and over on a cadence that never lets it accumulate the 60 consecutive + PRESENT ticks the unmeasured hold would otherwise need. Every cycle's absence is proven + rather than assumed - ``GET /apps`` is polled until the node is gone and again until it + is back, and the measured gap must EXCEED the 3-tick absence grace, on top of launch's + own enforced ``respawn_delay`` floor - so every cycle genuinely crosses the horizon past + which evidence used to be discarded. ``GRAPH_NODE_NOT_MANAGED`` must raise anyway and + survive the restarts that follow. A required node in a crash loop is the case this + detector most exists to catch, and the one a design that discarded evidence on absence + made permanently silent. + + The last four scenarios are about the bounds themselves. ``cap_pressure`` runs + ``tracked_node_cap: 1`` against TWO required nodes, so one is refused on every tick - + reachable only because the cap is a config key; against a compile-time 512 it would need + 513 real lifecycle nodes. It proves the refusal is visible on ``GET /x-medkit-watchdog``, + that it WITHHOLDS ``GRAPH_NODE_INACTIVE``'s clear (measured through ``last_passed``, which + catches even a transient clear, at the moment the tracked node's unmeasured clock matures + and that fault's content goes empty), and that the entry for a node that then LEAVES is + collapsed so the present, still-broken node is admitted and named. + ``unsettled_departure`` runs two healthy managed nodes built from this package's own + ``droppable_lifecycle_node.cpp`` - which looks managed, answers a chosen label, and stops + advertising its lifecycle services on command - and drops the services of each: one is + killed immediately (a single missed sweep before a clean shutdown, about which nothing may + ever be reported) and the other holds the dropped state past the settling budget before + being killed (genuinely not managed when it left, so it must still be reported). The first + leg's window is MEASURED against the settling budget rather than assumed, so it cannot + silently turn into the second. ``wide_grace`` configures the value that used to be the + accepted ``grace`` maximum and proves it is refused and the documented default applied - + under it the detector could neither raise nor heal for days. ``restart_departed`` records + where "a departure never heals a fault" ends: the required node is killed while its fault + is outstanding, the gateway is restarted, and the record then heals - because the restarted + detector cannot tell an entry for a departed node from a misspelt one. + + Status --------------- The plugin loads, ticks the graph, and shuts down cleanly. The reliability core is real -and already ticking. Three silent-fault detector classes raise through it today, -``qos_mismatch``, ``orphan`` and ``param_drift``. The remaining classes land in follow-up -changes, each against its own issue. +and already ticking. Four silent-fault detector classes raise through it today, +``qos_mismatch``, ``orphan``, ``param_drift`` and ``lifecycle_expectation``. The +remaining classes land in follow-up changes, each against its own issue. diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp index 2db5b83de..b16e7df8b 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp @@ -135,6 +135,18 @@ class Detector { return 0; } + /// Detector-scoped status, surfaced under `detectors.` on GET /x-medkit-watchdog next + /// to the reliability gate's own. For a condition that belongs to ONE detector and would + /// otherwise exist only in the gateway log - which no HTTP client and no e2e assertion can + /// read. Null (the default) means the detector has nothing to say and no key is added. + /// + /// Called on an HTTP handler thread while tick() runs on the plugin's tick thread, and the + /// handler holds no lock a detector takes, so an implementation must build its payload from + /// atomics (or other independently synchronised state) and must not block. + virtual nlohmann::json status_json() const { + return nullptr; + } + /// Test-only injection hook for detectors that read parameters over a transport. Returns false /// when the detector does not use one, so a test can assert it reached the right detector rather /// than silently doing nothing. diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/graph_fault_codes.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/graph_fault_codes.hpp index a918703a4..aba107993 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/graph_fault_codes.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/graph_fault_codes.hpp @@ -26,4 +26,15 @@ inline constexpr const char * kLatencyBudget = "GRAPH_LATENCY_BUDGET"; // Extension of the frozen namespace above (new capability, beyond the original // six classes): a node the operator declared must-be-active is present but not active. inline constexpr const char * kNodeInactive = "GRAPH_NODE_INACTIVE"; +// Second extension, alongside kNodeInactive: a node the operator declared must-be-active +// is present and matched, but its lifecycle label has never been read - an unverified +// promise, not a confirmed violation, and deliberately its own record rather than content +// folded into kNodeInactive. +inline constexpr const char * kNodeUnreadable = "GRAPH_NODE_UNREADABLE"; +// Third extension, sibling of kNodeUnreadable: a node the operator declared must-be-active +// is present and matched, but carries no tracked lifecycle at all (`lifecycle_state_of()` +// returns nullopt - a typo, or a plain non-lifecycle node). Like kNodeUnreadable this is an +// unverified promise, not a confirmed violation, and shares the same cause-blind unmeasured +// clock as kNodeUnreadable - the two differ only in which cause the clock matured under. +inline constexpr const char * kNodeNotManaged = "GRAPH_NODE_NOT_MANAGED"; } // namespace ros2_medkit_graph_watchdog::graph_fault_codes diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/graph_watchdog_plugin.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/graph_watchdog_plugin.hpp index 4e6b38a1b..3228db4c0 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/graph_watchdog_plugin.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/graph_watchdog_plugin.hpp @@ -64,6 +64,13 @@ class GraphWatchdogPlugin : public ros2_medkit_gateway::GatewayPlugin, return "graph_watchdog"; } void configure(const nlohmann::json & config) override; + /// PRECONDITION: called exactly once, before any tick. It builds `gate_` and the + /// detector list and then spawns the tick thread, so everything the tick thread reads is + /// published to it by that thread creation and needs no lock afterwards. A second call + /// while the tick thread is running would rebuild both under a live reader with no + /// synchronisation at all. The gateway honours this (one call site, gateway_node.cpp), + /// which is why nothing here defends against the other case - stated rather than + /// enforced, because a guard would be untestable through any caller that respects it. void set_context(ros2_medkit_gateway::PluginContext & context) override; std::vector get_routes() override; void shutdown() override; diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/lifecycle_expectation_tracker.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/lifecycle_expectation_tracker.hpp new file mode 100644 index 000000000..de1a3593d --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/lifecycle_expectation_tracker.hpp @@ -0,0 +1,929 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ros2_medkit_graph_watchdog { + +/// Consecutive ticks a require_active entry may match NO node at all before it is worth +/// warning about. A required node that never came up is invisible to the presence class +/// (GRAPH_NODE_DISAPPEARED needs the node present and online first), so nothing else +/// reports it. +inline constexpr int kDefaultNoMatchWarnTicks = 10; +/// Consecutive ticks a matched node may be genuinely ABSENT from the snapshot before its +/// clocks start moving again. Inside this budget a node's whole state is simply HELD: +/// zeroing it on the first blink would mean a node dropping out of one snapshot in every +/// few never accumulates enough consecutive present ticks on either clock below to be +/// reported at all. PAST this budget absence CONTINUES whatever the node's last real +/// observation had started - it never restarts a clock and never discards one. See +/// "Absence continues, it never erases" in LifecycleExpectationTracker's class doc. +inline constexpr int kDefaultAbsenceGrace = 3; + +/// Default bound on how many nodes this tracker keeps state for at once - the operator's +/// `tracked_node_cap`. Evidence is never reclaimed by AGE (an age horizon is itself an +/// evasion for a node that touches it periodically), so this is what bounds the map: at the +/// cap, IDLE entries - both clocks at zero and no matured ownership, so nothing to lose - +/// are reclaimed first, then DEPARTED entries are collapsed into a count (see +/// kMaxNamedDepartedEntries), and only if every tracked node is PRESENT and carrying +/// evidence is a newly-seen node refused and the saturation surfaced to the operator. 512 +/// against a realistic graph: this map only ever holds nodes a require_active entry MATCHED, +/// and a large single-robot ROS 2 graph (a full Nav2 stack plus perception) runs to roughly +/// a hundred nodes while a ten-robot fleet sharing one domain runs to a few hundred - so +/// even an operator whose entries match every node in the graph stays well under the cap, at +/// a cost of a few hundred KB to hold it. Growth past it can only come from identity CHURN +/// (respawns under ever-new fqns), which is exactly what the cap exists to bound. Same +/// value, for the same reason, as ros2_medkit_log_bridge's own max_tracked_nodes default. +inline constexpr int kDefaultTrackedNodeCap = 512; + +/// The most DEPARTED entries kept individually NAMED when the cap has to make room for a +/// present node; the rest are collapsed into a per-code count (see "A present node always +/// wins a slot" in the class doc). Three, and the number falls straight out of the +/// description budget: a detail is capped at kMaxLifecycleDetailChars and +/// AggregatedFault::kMaxDescriptionChars holds at most three of them +/// (3 * 150 + 2 * 2 = 454 <= 480, while a fourth would not fit - the arithmetic +/// lifecycle_expectation_detector.cpp already static_asserts). So a fourth named departed +/// entry could never appear in any description however the ordering fell out; past three +/// they are pure cost, and the cost they impose is a slot a PRESENT node needs. +inline constexpr int kMaxNamedDepartedEntries = 3; + +/// Consecutive matched ticks an UNMEASURED observation (kUnreadable or kNotManaged) must +/// hold before ABSENCE is allowed to continue it - see "Absence continues the last +/// CORROBORATED observation" in the class doc. A present, healthy, managed node reads +/// kNotManaged for a tick whenever its get_state path is missing from one sweep +/// (LifecycleWatcher::update() drops a tracked id whose path is absent from the current +/// sweep, and discover_apps() can yield an app with no services when a sweep races service +/// enumeration), so a single such reading must never be what decides that a node departed +/// unmeasurable. Six: no single racing sweep produces six consecutive readings, it is the +/// same bar the detector's own typo warning already sets (kUnmanagedWarnTicks fires on the +/// sixth consecutive unmanaged tick, guarding against this exact transient), and it is a +/// tenth of kDefaultUnmeasuredHoldTicks - so a node that is genuinely unmeasurable when it +/// leaves has corroborated that long before the hold which would report it. +inline constexpr int kDefaultObservationSettleTicks = 6; + +/// Consecutive ticks a node's UNMEASURED clock (see LifecycleObservedState below) may +/// climb before the cause it is currently climbing under takes ownership of the node and +/// gets its own fault code. "Frozen one past itself" like every other bounded clock here: +/// past the bound the stored counter stops incrementing, so it only needs to distinguish +/// "still climbing" from "matured", never how far past. Not configurable via detector +/// config (mirrors the previous kUnmeasuredHoldTicks, which was a detector-file-local +/// constant) - exposed here as a constructor default so a test can shrink it without +/// waiting out 60 real ticks. +inline constexpr int kDefaultUnmeasuredHoldTicks = 60; + +/// Longest label a REAL lifecycle implementation can produce on `~/transition_event`: the +/// four primary states this detector enforces against (`unconfigured`, `inactive`, +/// `active`, `finalized`) plus the six transition states lifecycle_msgs/msg/State.msg +/// defines (`configuring`, `cleaningup`, `shuttingdown`, `activating`, `deactivating`, +/// `errorprocessing`). The longest of the ten is `errorprocessing` at 15 characters. +/// kMaxLifecycleLabelChars is more than double that, so no conforming label is ever +/// affected by the trim below - only a value from a non-conforming remote publisher (the +/// label arrives verbatim off a remote `~/transition_event`, see lifecycle_watcher.cpp) +/// spends the budget. This is the PRIMARY guard (R13): the label is the untrusted, +/// unbounded part of the detail, so it is trimmed on its own rather than as part of a +/// whole-detail head-and-tail that would also mangle the node name. +inline constexpr std::size_t kMaxLifecycleLabelChars = 32; + +/// Backstop on the WHOLE per-node detail string (after the label above is already +/// trimmed), for a pathological fqn or a long `require_active` "required by" list rather +/// than the label - R13's secondary guard. Sized so at least 3 maximally-long details +/// still fit inside one description under AggregatedFault::kMaxDescriptionChars (480, +/// joined by "; "): 3 * 150 + 2 * 2 = 454 <= 480, while 4 maximally-long details would not: +/// 4 * 150 + 3 * 2 = 606 > 480. lifecycle_expectation_detector.cpp static_asserts this +/// arithmetic against the real AggregatedFault::kMaxDescriptionChars constant, since this +/// header does not include aggregated_fault.hpp (see its own "Pure." class doc). +inline constexpr std::size_t kMaxLifecycleDetailChars = 150; + +/// Truncates `text` to at most `max_chars`, replacing the tail with "..." when it does not +/// fit whole - the same suffix-ellipsis convention AggregatedFault::describe_ordered uses +/// for the overall description, so a trimmed label or detail reads the same way a trimmed +/// description does. Shared by every detail builder below. +inline std::string trim_to(const std::string & text, std::size_t max_chars) { + if (text.size() <= max_chars) { + return text; + } + constexpr std::size_t kMarkerChars = 3; // "..." + const std::size_t keep = max_chars > kMarkerChars ? max_chars - kMarkerChars : 0; + return text.substr(0, keep) + "..."; +} + +/// One live node matched by a require_active entry, with its lifecycle state. +struct LifecycleMatch { + std::string entry; ///< the require_active entry that matched + std::string fqn; ///< the node's stable App::effective_fqn() + std::optional state; +}; + +/// The one observed fact this whole state machine runs on: what a MATCHED node's read +/// tells us this tick. A node that did not match anything this tick never reaches +/// classify_observed_state() at all - it is ABSENT, and absence is handled by a wholly +/// separate path (the per-node absence budget in update()), not by this enum, because +/// absence carries no label to classify. +enum class LifecycleObservedState { + kActive, ///< label == "active" + kInactive, ///< label non-empty and != "active" - the one state that ever counts + ///< toward the violation clock + kUnreadable, ///< optional("") - a MANAGED node whose label has never been read + kNotManaged, ///< nullopt - no tracked lifecycle at all this tick +}; + +/// Pure function of the label alone. +inline LifecycleObservedState classify_observed_state(const std::optional & state) { + if (!state.has_value()) { + return LifecycleObservedState::kNotManaged; + } + if (state->empty()) { + return LifecycleObservedState::kUnreadable; + } + return *state == "active" ? LifecycleObservedState::kActive : LifecycleObservedState::kInactive; +} + +/// Which of the two "I cannot measure this" causes a node's unmeasured clock most +/// recently climbed under. Meaningful in two different ways depending on +/// `NodeState::unmeasured_matured` (private below): while the clock is still climbing it +/// is simply "what did the last unmeasured tick look like" (refreshed every such tick, for +/// the withheld-clear log's own bookkeeping); once matured it is frozen - see "Sticky vs. +/// current-cause-wins" in update() for why. +enum class LifecycleUnmeasuredCause { + kUnreadable, + kNotManaged, +}; + +/// What one update() sweep found. Ownership across the three fault-shaped maps below is +/// exclusive: a node appears in at most one of `affected`, `unreadable_affected` and +/// `not_managed_affected` on any given tick - see "Ownership is exclusive" in update(). +struct LifecycleExpectationReport { + /// fqn -> detail phrase, for GRAPH_NODE_INACTIVE: a node CONFIRMED non-active (its + /// violation clock crossed `grace`) and not currently owned by the unmeasured clock. + std::map affected; + /// fqns whose violation clock crossed `grace` on THIS update() - i.e. were ADDED to + /// `affected` this tick, not merely still in it from an earlier one. Lexicographic by + /// fqn (this tracker's own node map is a std::map) when several cross together, so the + /// caller's new-first ordering is deterministic even in the degenerate `grace: 0` case. + std::vector newly_affected; + + /// fqn -> detail phrase, for GRAPH_NODE_UNREADABLE: the unmeasured clock matured with + /// cause kUnreadable and still holds that cause (see the sticky rule in update()). + std::map unreadable_affected; + std::vector newly_unreadable; + + /// fqn -> detail phrase, for the not-managed fault code: the unmeasured clock matured + /// with cause kNotManaged and still holds that cause. + std::map not_managed_affected; + std::vector newly_not_managed; + + /// fqns whose GRAPH_NODE_INACTIVE status is UNSETTLED this tick - a violation streak that + /// has not yet passed `grace`, or an unmeasured clock still climbing (not yet matured), + /// or a streak HELD while the node is inside an unmeasured spell. The caller must not + /// assert "nothing is inactive" while any of these exist: a level-triggered clear + /// whenever `affected` is empty would otherwise heal a fault whose node's status this + /// tracker simply has not settled yet. Once a node's unmeasured clock MATURES it leaves + /// this set entirely (ownership passes to its own fault code, and GRAPH_NODE_INACTIVE has + /// nothing left to say about it either way - see "the violation streak is RELEASED" in + /// advance_unmeasured_clock()). Absence never puts a node here on its own: content + /// follows the clocks, not the snapshot, so a node already past `grace` stays in + /// `affected` through a blink rather than dropping into a withheld limbo. + std::set pending; + /// Breakdown of `pending`, for the withheld-clear LOG LINE only - never the withhold + /// decision itself, which is `!pending.empty()`. A node can appear in more than one + /// (e.g. a held violation streak while ALSO climbing the unmeasured clock), so these are + /// not a partition and must not be summed against `pending.size()`. + std::set pending_violation; ///< below-grace violation streak + std::set pending_unreadable; ///< unmeasured clock climbing, cause kUnreadable + std::set pending_not_managed; ///< unmeasured clock climbing, cause kNotManaged + + /// Entries that have matched nothing for more CONSECUTIVE ticks than the no-match + /// threshold and have not been reported yet. Consecutive, not "never": an entry whose node + /// matched and later left the graph surfaces here too, and for that one the caller's + /// "never came up, and the presence class cannot see it either" framing is false - so the + /// caller filters on whether the entry has ever matched before it says anything. + std::vector entries_matching_nothing; + + /// LEVEL, not an edge: true on every tick a newly matched node had to be refused because + /// the tracked-node cap is full of PRESENT nodes all carrying evidence, and nothing could + /// be reclaimed or collapsed. It stays true for as long as the condition lasts, because a + /// refused node is still matched on the following tick and refused again. Refusing the + /// newcomer rather than evicting a live violation is the safe direction, but it means a + /// required node is going unchecked, so the caller must NOT clear GRAPH_NODE_INACTIVE + /// while it holds - a detector that declined to check a required node cannot assert that + /// every required node is healthy. + bool tracking_saturated = false; + /// EDGE: the first tick of a saturation episode, for the log line. Re-arms when the + /// episode ends, so a later, real saturation is not silent because an earlier one spent + /// the one warning. + bool saturation_started = false; +}; + +/// Enforces "these nodes must be active" and, inseparably from it, "I could not tell +/// whether these nodes are active" - one state machine per matched node rather than two +/// systems that have to agree. +/// +/// **The model.** Per node, per tick, exactly one LifecycleObservedState (or ABSENT, its +/// own separate case - see below). Two clocks, and only two: +/// +/// - **Violation streak** (`NodeState::violation_streak`). Advances on kInactive. Resets +/// ONLY on kActive. Past `grace`, the node is in `affected` (GRAPH_NODE_INACTIVE). +/// - **Unmeasured clock** (`NodeState::unmeasured_clock`). Advances on kUnreadable OR +/// kNotManaged. Resets ONLY on a real measurement - kActive or kInactive. It is BLIND to +/// which of the two causes it saw on any given tick: a node alternating between +/// "matched but never read" and "not a tracked lifecycle node at all" keeps this single +/// clock climbing instead of each cause resetting the other's own counter - which is +/// what let such a node evade both codes forever in the design this replaces (a +/// MANAGED-node-whose-label-never-arrives counter and a NOT-MANAGED counter, each zeroed +/// by the other's tick). Past `unmeasured_hold_ticks`, the node's UNMEASURED clock +/// matures and takes ownership - see "Ownership is exclusive" below. +/// +/// **Ownership is exclusive.** A node is content of at most one of `affected`, +/// `unreadable_affected` or `not_managed_affected` at a time. The moment the unmeasured +/// clock matures, the violation streak is RELEASED (reset to 0) - not merely excluded from +/// `affected` THIS tick, but actually zeroed - so GRAPH_NODE_INACTIVE has nothing left to +/// hold and its clear is free to flow. A node returning to kInactive afterwards therefore +/// re-earns `grace` from zero, exactly as a node that had never gone unmeasured would. +/// Below maturity the two still do not overlap: a node inside an unmeasured spell +/// (`unmeasured_clock > 0`) is never `affected` content however far its streak had already +/// climbed - "I cannot measure this right now" is not a violation - but the streak is HELD, +/// not reset, and resumes the instant a real read comes back. +/// +/// **Content follows the clocks, not the snapshot.** Whether a node happened to be in this +/// tick's matches decides nothing about what it reports. A node whose streak is past +/// `grace` stays in `affected` while it blinks, and a matured node stays in its own map, +/// because a fault's content is what the tracker has MEASURED about the node, and a missing +/// snapshot entry measures nothing either way. What absence does change is the detail +/// phrase: past `absence_grace` it says the node has left the graph, so an operator is +/// never sent to look at a node that is no longer there. +/// +/// **Sticky vs. current-cause-wins (the one design choice this class makes on its own).** +/// The unmeasured clock is cause-blind, but the two causes map to two different fault +/// codes. What should a node report once its clock has MATURED, if the cause it is +/// observed under later changes (unreadable this tick, not-managed the next, without ever +/// crossing back to a real measurement)? This class is STICKY: `NodeState::cause` freezes +/// at whatever it was on the tick the clock matured, and nothing but a real measurement +/// (kActive/kInactive, which resets the whole clock) ever changes it again - so a node that +/// matured as GRAPH_NODE_UNREADABLE keeps reporting under that code even if it is later +/// observed NOT_MANAGED for a while, and vice versa. The rejected alternative is +/// current-cause-wins: report whichever cause the LIVE tick shows, so the fault code can +/// flip between the two on every alternating tick. Current-cause-wins is the more literal +/// reading of "exactly what is true right now", but it makes a node that flaps between the +/// two causes flap the FAULT SURFACE too - a raise/clear/raise churn on two different fault +/// codes for a node whose actual situation (still cannot be measured, whichever way you +/// slice it) never changed. Sticky keeps the surface calm: once "I cannot measure this" +/// has been reported, which specific unmeasured-flavor it is stays fixed until something +/// REAL is learned about the node, matching how `grace` and `unmeasured_hold_ticks` +/// already treat a real measurement as the only thing that resets a clock. +/// +/// **Absence continues the last CORROBORATED observation, it never erases.** A node not +/// matched at all this tick is ABSENT - a wholly separate fact from any +/// LifecycleObservedState, since there is no label to classify. For up to `absence_grace` +/// consecutive absent ticks a node's whole state (both clocks, whichever fault - if any - +/// currently owns it) is simply held, unchanged, so a node that blinks out of one snapshot +/// in every few is not indistinguishable from one that never stayed present long enough to +/// be measured at all. PAST `absence_grace` absence advances whichever clock the node's +/// `NodeState::settled_observed` says it was on, and resets nothing: +/// +/// - settled kInactive: the violation streak keeps climbing, so a node measured not-active +/// that then vanishes eventually raises GRAPH_NODE_INACTIVE about a node that is no +/// longer there. That is the honest reading of the expectation the operator wrote: the +/// node must be ACTIVE, it was not, and now it is gone. Any UNCORROBORATED unmeasured +/// spell it had also started is dropped here rather than held: nothing is left to read, +/// and a clock that can never mature would keep the entry non-idle - and therefore +/// `pending` - for the life of the process. +/// - settled kUnreadable or kNotManaged: the unmeasured clock keeps climbing, under the +/// cause that observation set - absence carries no label, so it cannot change one. +/// - settled kActive: nothing advances, and anything the node had started but not +/// corroborated is released, so the entry becomes idle and is reclaimed silently. An +/// entry that is already CONTENT (a matured unmeasured clock, or a streak past `grace`) +/// is never released - a departure heals nothing. Starting a NEW violation from a healthy +/// departure is the presence class's job (GRAPH_NODE_DISAPPEARED), which still has no +/// detector in this package and is still out of scope - the single remaining gap here, +/// and a far narrower one than "a node absent past the grace is invisible whichever clock +/// it was on". +/// +/// **What "settled" means, and the alternatives rejected for it.** A real measurement +/// (kActive or kInactive) settles IMMEDIATELY: a lifecycle label is a fact about the node, +/// and no sweep artifact can invent one. An UNMEASURED observation settles immediately too +/// on a node NOTHING HAS EVER MEASURED, because it is then the only thing known about it - +/// which is what keeps a node that is unreadable or unmanaged whenever it is present, and +/// absent the rest of the time, accumulating toward its report instead of being released on +/// every absence. It is only when an unmeasured reading would OVERRIDE a real measurement +/// that it has to be corroborated first, over `kDefaultObservationSettleTicks` consecutive +/// matched ticks. The reason is that kNotManaged and kUnreadable are BOTH producible by the +/// discovery layer missing a node's service path for one sweep, on a node that is present and +/// perfectly healthy - and if that sweep is the last one before a clean shutdown, "absence +/// continues the last observation" would mature a healthy departure into a permanent +/// GRAPH_NODE_NOT_MANAGED. +/// +/// The rejected alternative is the literal one this replaces: continue whatever the single +/// last observation was. It is simpler and it is wrong for exactly the reading it cannot tell +/// apart from a real one. Rejected as well: continuing the MOST INFORMATIVE observation of +/// the recent window, which is worse still here - an unmeasured reading outranks a healthy +/// one (see observed_rank), so a single bad sweep would decide the carve-out by design rather +/// than by accident. And rejected as a shortcut: requiring corroboration unconditionally, +/// which reopens the restart-loop evasion above for any node whose uptime is shorter than the +/// settling budget. Corroboration does not weaken the promise it guards: a node that is +/// genuinely unmeasurable when it leaves has been so for far more than six ticks, since the +/// hold that would report it is sixty. +/// +/// The design this replaces reset both clocks and released ownership past `absence_grace`. +/// That made every erasure horizon an evasion: a node in a restart loop - start, crash, +/// respawn delay, start - touches absence periodically BY CONSTRUCTION, so it could +/// alternate `(UNREADABLE, ABSENT x N)`, `(NOT_MANAGED, ABSENT x N)` or `(INACTIVE, +/// ABSENT x N)` forever and never accumulate enough of anything to be reported, which is +/// precisely the node this detector most exists to catch. +/// +/// **Bounded by evidence, not by age.** Since absence no longer erases anything, an entry +/// carrying a live clock is never reclaimed by age either - the two are the same defect +/// seen twice, and moving the horizon rather than removing it would leave the evasion in +/// place at a different N. `prune_ticks` therefore reclaims only IDLE entries (both clocks +/// zero, no matured ownership), which carry nothing to lose. What bounds the map instead is +/// `tracked_node_cap`. A non-idle entry cannot grow without bound in time either way: past +/// `absence_grace` its clock advances every tick, so it matures within at most +/// `grace` + `absence_grace` + 1 ticks (violation) or `unmeasured_hold_ticks` + +/// `absence_grace` + 1 (unmeasured) and is reported. Both bounds are why the detector caps +/// the `grace` it accepts: they are also the longest GRAPH_NODE_INACTIVE's clear can be +/// withheld by one departed node. +/// +/// **A present node always wins a slot.** At the cap the order is: reclaim IDLE entries +/// (they carry nothing); then collapse DEPARTED entries - lexicographically last first, +/// keeping at most `kMaxNamedDepartedEntries` of them named - into per-code COUNTS; and only +/// if every remaining entry is PRESENT and carrying evidence is the newcomer refused, with +/// `LifecycleExpectationReport::tracking_saturated` saying so on every such tick. +/// +/// Collapsing rather than holding follows from something easy to miss: **the fault is keyed +/// by CODE, not by node.** Five hundred entries for dead identities keep exactly the same +/// one fault raised that a single entry would, and the description budget can only ever name +/// a handful of them - so holding them buys nothing, while the slots they occupy can cost +/// total blindness. Under identity churn (each robot's nodes appearing under their own +/// namespace and leaving for good) a cap full of the dead would refuse a genuinely broken +/// PRESENT node: `track()` returns nullptr, the node never enters `affected` or `pending`, +/// its entry has matched so the caller's never-matched hold does not apply either, and +/// GRAPH_NODE_INACTIVE emits a level-triggered CLEAR every tick while that node reads +/// not-active. The detector would report health it had refused to check. +/// +/// **What the operator sees when a count is non-zero.** The collapsed count enters that +/// code's content as one extra line - "and N more required node(s) left the graph ..." - +/// ahead of the individually named entries but behind anything crossing on this tick, so a +/// node that just broke is never displaced by it while the summary is not itself truncated +/// away by a list of names that are all equally uninformative. Its +/// presence means two things at once: those N nodes left the graph carrying evidence and +/// their fault is still correctly raised (a departure heals nothing), and the tracked-node +/// cap is under enough pressure that they could not be kept individually - i.e. `require_active` +/// is matching an unbounded set of identities, or `tracked_node_cap` is too small for the +/// fleet. The count only ever grows within one tracker lifetime: once an entry is collapsed +/// the tracker no longer knows which fqn it was, so a node returning under that same fqn is +/// tracked afresh, measured afresh, and cannot decrement it. That is deliberate rather than +/// overlooked - it is "a departure never heals a fault" applied to an entry that can no +/// longer be named - and it is bounded the same way the promise itself is: a gateway restart +/// re-baselines everything. +/// +/// **Keyed by node, not by config entry.** The bare-name form of require_active is +/// deliberately fleet-wide ("all controller_servers must be active"), so one entry +/// legitimately covers N nodes; two entries (a fleet-wide bare name plus a pinned FQN) can +/// also both name the SAME node, in which case its clocks each advance once per tick, not +/// once per matching entry (counting per match would silently halve whatever budget the +/// operator configured). Every entry that named a node is still carried as "required by" +/// context in that node's detail string. +class LifecycleExpectationTracker { + public: + /// kNoPrune keeps every IDLE entry forever too - the default, so a short-lived unit test + /// is unaffected. Production wiring passes the operator's own `prune_grace`. + static constexpr int kNoPrune = std::numeric_limits::max(); + + /// prune_ticks, unmeasured_hold_ticks and tracked_node_cap are LAST on purpose: every + /// parameter here is an int, so inserting one in the middle keeps existing positional + /// calls compiling while silently rebinding them. + LifecycleExpectationTracker(std::set require_active, int grace, int absence_grace = kDefaultAbsenceGrace, + int no_match_warn_ticks = kDefaultNoMatchWarnTicks, int prune_ticks = kNoPrune, + int unmeasured_hold_ticks = kDefaultUnmeasuredHoldTicks, + int tracked_node_cap = kDefaultTrackedNodeCap) + : require_active_(std::move(require_active)) + , grace_(grace) + , absence_grace_(absence_grace) + , no_match_warn_ticks_(no_match_warn_ticks) + , prune_ticks_(prune_ticks) + , unmeasured_hold_ticks_(unmeasured_hold_ticks) + , tracked_node_cap_(tracked_node_cap < 1 ? 1 : tracked_node_cap) { + } + + /// Nodes whose bookkeeping is still held - for tests asserting the map stays bounded. + /// Every per-node fact (both clocks, absence count, cause, entries) now lives in ONE map, + /// so this is simply its size - no second map to reconcile against. + std::size_t tracked_count() const { + return nodes_.size(); + } + + LifecycleExpectationReport update(const std::vector & matches) { + LifecycleExpectationReport report; + + // Collapse the per-(entry, node) matches to one record per NODE before any clock + // moves - see "Keyed by node, not by config entry" above. + struct NodeTick { + std::vector entries; ///< every entry that named this node, in match order + std::optional state; ///< the label enforced for it this tick + }; + std::set matched_entries; + std::map nodes; + for (const auto & match : matches) { + matched_entries.insert(match.entry); + const auto inserted = nodes.emplace(match.fqn, NodeTick{}); + NodeTick & node = inserted.first->second; + if (std::find(node.entries.begin(), node.entries.end(), match.entry) == node.entries.end()) { + node.entries.push_back(match.entry); + } + // Duplicate matches for one node carry the same label by construction (the caller + // reads the label once per node), but if they ever disagree the tie-break below picks + // the MOST INFORMATIVE reading: a confirmed violation beats an unmeasured read, which + // beats a merely-not-tracked one, which beats a healthy one - a detector for silent + // faults must not let a duplicate match throw information away. + if (inserted.second || + observed_rank(classify_observed_state(match.state)) > observed_rank(classify_observed_state(node.state))) { + node.state = match.state; + } + } + + // Clocks that cross their bound on THIS tick. Collected here rather than pushed + // straight into report.newly_*, so the content pass below - one lexicographic sweep of + // nodes_ - is the only place that decides ordering, whether the node that crossed was + // matched this tick or merely absent past its blink tolerance. + std::set crossed_violation; + std::set crossed_unmeasured; + + std::set seen_fqns; + for (const auto & entry : nodes) { + const std::string & fqn = entry.first; + const NodeTick & node_tick = entry.second; + NodeState * const tracked = track(fqn, report); + if (tracked == nullptr) { + continue; // every tracked node carries evidence and the cap is full - see track() + } + seen_fqns.insert(fqn); + NodeState & node = *tracked; + node.absent_ticks = 0; + node.entries = node_tick.entries; // refreshed on every matched tick, held through a blink + + const LifecycleObservedState state = classify_observed_state(node_tick.state); + switch (state) { + case LifecycleObservedState::kActive: + // A real measurement: both clocks reset, ownership released, nothing left to say. + // A label is a fact about the node, so it settles at once - see "What 'settled' + // means" in the class doc. + node.violation_streak = 0; + node.unmeasured_clock = 0; + node.unmeasured_matured = false; + node.last_label.clear(); + node.settled_observed = state; + node.ever_measured = true; + break; + case LifecycleObservedState::kInactive: + // A real measurement resets the unmeasured clock and releases unmeasured + // ownership (a node returning to a real read - even a bad one - is no longer + // "cannot be measured"), THEN the violation clock advances under its own rules. + node.unmeasured_clock = 0; + node.unmeasured_matured = false; + // Trimmed on the way IN, so the untrusted remote-supplied label bounds the memory + // this tracker holds as well as the detail it later builds (R13's primary guard). + node.last_label = trim_to(*node_tick.state, kMaxLifecycleLabelChars); + node.settled_observed = state; + node.ever_measured = true; + advance_violation_streak(node, fqn, crossed_violation); + break; + case LifecycleObservedState::kUnreadable: + case LifecycleObservedState::kNotManaged: + // Cause-blind advance: this clock does not care which of the two it is seeing, + // only that a real measurement has not happened. While still climbing, `cause` + // tracks the live cause (for the withheld-clear log, and so that whatever is live + // AT MATURITY is what freezes); once matured it is left untouched - see the class + // doc's "Sticky vs. current-cause-wins". + if (!node.unmeasured_matured) { + node.cause = state == LifecycleObservedState::kUnreadable ? LifecycleUnmeasuredCause::kUnreadable + : LifecycleUnmeasuredCause::kNotManaged; + } + advance_unmeasured_clock(node, fqn, crossed_unmeasured); + // An unmeasured reading has to be CORROBORATED before absence may continue it - but + // only when there is a real measurement for it to override. On a node nothing has + // ever measured, it is the only thing known, so it settles at once. The clock IS + // the run length, since only a real measurement resets it. See "What 'settled' + // means" in the class doc. + if (!node.unmeasured_matured && + (!node.ever_measured || node.unmeasured_clock >= kDefaultObservationSettleTicks)) { + node.settled_observed = state; + } + break; + } + } + + // Absence loop: every tracked node NOT matched this tick. Inside absence_grace_ + // everything is HELD unchanged (a blink). Past it, absence CONTINUES whichever clock + // the node's last REAL observation had started, and resets nothing - see "Absence + // continues, it never erases" in the class doc. A node last measured kActive continues + // nothing: a healthy node shutting down is GRAPH_NODE_DISAPPEARED's business, which + // still has no detector in this package and stays out of scope. Only IDLE bookkeeping + // is reclaimed by age, because only an idle entry has nothing to lose. + for (auto it = nodes_.begin(); it != nodes_.end();) { + const std::string & fqn = it->first; + NodeState & node = it->second; + if (seen_fqns.count(fqn) != 0) { + ++it; + continue; + } + if (node.absent_ticks < std::numeric_limits::max()) { + ++node.absent_ticks; // saturating: past both horizons the exact count stops mattering + } + if (node.absent_ticks > absence_grace_) { + switch (node.settled_observed) { + case LifecycleObservedState::kInactive: + // An unmeasured spell that never lasted long enough to be corroborated dies with + // the node: there is nothing left to read, and a clock that can never mature + // would keep this entry non-idle - and therefore `pending` - forever. + node.unmeasured_clock = 0; + advance_violation_streak(node, fqn, crossed_violation); + break; + case LifecycleObservedState::kUnreadable: + case LifecycleObservedState::kNotManaged: + advance_unmeasured_clock(node, fqn, crossed_unmeasured); + break; + case LifecycleObservedState::kActive: + release_uncorroborated(node); // healthy departure: nothing here starts a violation + break; + } + } + if (node.absent_ticks > prune_ticks_ && is_idle(node)) { + it = nodes_.erase(it); // idle: carries no evidence, so reclaiming it loses nothing + } else { + ++it; + } + } + + // Content pass: EVERY tracked node, present or not - content follows the clocks, not + // the snapshot (see the class doc). One lexicographic sweep, so the newly_* orderings + // are deterministic even when several nodes cross together - but PRESENT crossings are + // collected apart from DEPARTED ones and go first in each newly_* list. The caller feeds + // newly_* to AggregatedFault::emit_ordered, which stops once the description budget is + // spent, so on a tick where a departed node and a present, newly-broken one both cross, + // a single lexicographic list could name the one that LEFT and truncate away the one + // that just broke. + std::vector departed_affected, departed_unreadable, departed_not_managed; + for (auto & [fqn, node] : nodes_) { + const bool departed = node.absent_ticks > absence_grace_; + if (node.unmeasured_matured) { + const std::string detail = unmeasured_detail(fqn, node, departed); + const bool crossed = crossed_unmeasured.count(fqn) != 0; + if (node.cause == LifecycleUnmeasuredCause::kUnreadable) { + report.unreadable_affected[fqn] = detail; + if (crossed) { + (departed ? departed_unreadable : report.newly_unreadable).push_back(fqn); + } + } else { + report.not_managed_affected[fqn] = detail; + if (crossed) { + (departed ? departed_not_managed : report.newly_not_managed).push_back(fqn); + } + } + continue; // matured: never pending for GRAPH_NODE_INACTIVE (ownership released it) + } + // A node inside an unmeasured spell is not confirmed content however far its streak + // had climbed - the streak is HELD, not reported, until a real read comes back. + if (node.unmeasured_clock == 0 && node.violation_streak > grace_) { + report.affected[fqn] = inactive_detail(fqn, node, departed); + if (crossed_violation.count(fqn) != 0) { + (departed ? departed_affected : report.newly_affected).push_back(fqn); + } + } + if (node.violation_streak > 0 && report.affected.count(fqn) == 0) { + report.pending.insert(fqn); + report.pending_violation.insert(fqn); + } + if (node.unmeasured_clock > 0) { + report.pending.insert(fqn); + if (node.cause == LifecycleUnmeasuredCause::kUnreadable) { + report.pending_unreadable.insert(fqn); + } else { + report.pending_not_managed.insert(fqn); + } + } + } + append_all(report.newly_affected, departed_affected); + append_all(report.newly_unreadable, departed_unreadable); + append_all(report.newly_not_managed, departed_not_managed); + + // Departed entries collapsed to free slots for present nodes still count as content, or + // freeing the slot would heal a fault a departure must never heal. Sorted after every + // real fqn (see kCollapsedKeyPrefix), so this never takes description budget from a + // node that is still named. + add_collapsed_content(report.affected, collapsed_inactive_, "not active"); + add_collapsed_content(report.unreadable_affected, collapsed_unreadable_, "with an unread lifecycle state"); + add_collapsed_content(report.not_managed_affected, collapsed_not_managed_, "unmanaged"); + + report.saturation_started = report.tracking_saturated && !saturated_last_tick_; + saturated_last_tick_ = report.tracking_saturated; + + // An entry matching nothing is silent in every other path: it never reaches the + // violation branch above, and the presence class never starts tracking a node that was + // never present, so a misspelt entry - or one whose node crashed during launch - + // produces no fault and no log line anywhere. + for (const auto & entry : require_active_) { + if (matched_entries.count(entry) != 0) { + no_match_[entry] = 0; + continue; + } + if (++no_match_[entry] > no_match_warn_ticks_ && reported_no_match_.insert(entry).second) { + report.entries_matching_nothing.push_back(entry); + } + } + return report; + } + + private: + /// Per-node bookkeeping - the entire state machine for one fqn. Everything that used to + /// be spread across two counter maps in this class plus two more in the detector's own + /// guard now lives here. + struct NodeState { + int violation_streak = 0; ///< consecutive kInactive ticks since the last kActive + int unmeasured_clock = 0; ///< consecutive kUnreadable-or-kNotManaged ticks since + ///< the last real measurement (frozen one past the hold) + bool unmeasured_matured = false; ///< the unmeasured clock has crossed the hold and not + ///< yet been reset by a real measurement + LifecycleUnmeasuredCause cause = LifecycleUnmeasuredCause::kUnreadable; ///< live while + ///< climbing, frozen (sticky) once + ///< unmeasured_matured - see the class doc + /// What absence continues past absence_grace: the last observation of this node that + /// COUNTS - a real measurement immediately, an unmeasured reading either immediately (if + /// there is no real measurement for it to override) or once it has held for + /// kDefaultObservationSettleTicks consecutive matched ticks. kActive is the right initial + /// value: it continues nothing, and it is only ever read while `ever_measured` is true. + LifecycleObservedState settled_observed = LifecycleObservedState::kActive; + /// Whether a REAL measurement (kActive or kInactive) has ever been taken of this node. + /// It is what an unmeasured reading has to be corroborated against: with no real + /// measurement on record an unmeasured reading is simply the best - and only - thing + /// known about the node, so it settles at once. Without this distinction a node that is + /// unreadable or unmanaged whenever it is present and absent the rest of the time (a + /// restart loop, the case this detector most exists to catch) would have its clock + /// released on every absence and never mature. + bool ever_measured = false; + /// The label of the last kInactive observation, already trimmed to + /// kMaxLifecycleLabelChars. Kept because the detail for a CONFIRMED violation names the + /// state the node is stuck in, and absence carries no label of its own to name. Cleared + /// by kActive, the one observation that also clears the streak, so it can never outlive + /// the streak it describes. + std::string last_label; + std::vector entries; ///< "required by" entries, refreshed on every matched tick + int absent_ticks = 0; ///< consecutive ticks this fqn was not matched at all (saturating) + }; + + /// An entry with nothing to lose: no streak, no unmeasured clock, no matured ownership. + /// The only kind that may be reclaimed - by age past prune_ticks_, or to make room at the + /// tracked-node cap. + static bool is_idle(const NodeState & node) { + return node.violation_streak == 0 && node.unmeasured_clock == 0 && !node.unmeasured_matured; + } + + /// An entry that is already a fault's CONTENT - a matured unmeasured clock, or a violation + /// streak past `grace`. Nothing may release one: a departure heals no fault. + bool is_content(const NodeState & node) const { + return node.unmeasured_matured || node.violation_streak > grace_; + } + + /// An entry whose absence has outlived the blink tolerance - the node is gone, not + /// flickering. + bool is_departed(const NodeState & node) const { + return node.absent_ticks > absence_grace_; + } + + /// A departed node whose last CORROBORATED observation was healthy has nothing for absence + /// to continue, so anything it had started but not corroborated goes with it - otherwise a + /// single missed sweep before a clean shutdown leaves a clock that can never mature and an + /// entry that is never idle, i.e. `pending` for the life of the process. An entry that is + /// already CONTENT is never touched. + void release_uncorroborated(NodeState & node) const { + if (is_content(node)) { + return; + } + node.violation_streak = 0; + node.unmeasured_clock = 0; + } + + /// The tracked entry for `fqn`, creating it if a slot can be found. Returns nullptr only + /// when every tracked node is PRESENT and carrying evidence - never evicting a live + /// violation for a newcomer - and the report says so on every such tick, so the caller can + /// withhold the clear it must not emit while a required node is going unchecked. + NodeState * track(const std::string & fqn, LifecycleExpectationReport & report) { + const auto existing = nodes_.find(fqn); + if (existing != nodes_.end()) { + return &existing->second; + } + if (nodes_.size() >= static_cast(tracked_node_cap_) && !make_room()) { + report.tracking_saturated = true; + return nullptr; + } + return &nodes_[fqn]; + } + + /// Frees a slot for a PRESENT node - see "A present node always wins a slot" in the class + /// doc. Idle entries first (they carry nothing), then departed entries collapsed into + /// per-code counts, keeping at most kMaxNamedDepartedEntries of them named; if even that + /// leaves no room the named ones go too, because a present required node going unchecked + /// is a worse outcome than a departed one losing its name. False when nothing could be + /// freed, i.e. every entry is present and carrying evidence. + bool make_room() { + for (auto it = nodes_.begin(); it != nodes_.end();) { + it = is_idle(it->second) ? nodes_.erase(it) : std::next(it); + } + if (nodes_.size() < static_cast(tracked_node_cap_)) { + return true; + } + collapse_departed(kMaxNamedDepartedEntries); + if (nodes_.size() < static_cast(tracked_node_cap_)) { + return true; + } + collapse_departed(0); + return nodes_.size() < static_cast(tracked_node_cap_); + } + + /// Collapse departed entries into per-code counts until at most `keep_named` of them are + /// left in the map. Lexicographically LAST first, so the survivors are a stable prefix and + /// the same graph always keeps the same names. + void collapse_departed(std::size_t keep_named) { + std::vector departed; + for (const auto & [fqn, node] : nodes_) { + if (is_departed(node)) { + departed.push_back(fqn); + } + } + for (std::size_t i = departed.size(); i > keep_named; --i) { + const auto it = nodes_.find(departed[i - 1]); + count_collapsed(it->second); + nodes_.erase(it); + } + } + + /// Fold one departed entry into the count for the code its clock was heading for. An + /// unmeasured clock STILL CLIMBING counts under its cause rather than being dropped: + /// absence advances it every tick and nothing can reset it any more, so it would have + /// matured under that cause within a bounded number of ticks anyway, and dropping it + /// instead would let freeing a slot heal a fault. An idle entry contributes nothing - + /// idle entries are reclaimed before this ever runs. + void count_collapsed(const NodeState & node) { + if (node.unmeasured_matured || node.unmeasured_clock > 0) { + if (node.cause == LifecycleUnmeasuredCause::kUnreadable) { + ++collapsed_unreadable_; + } else { + ++collapsed_not_managed_; + } + } else if (node.violation_streak > 0) { + ++collapsed_inactive_; + } + } + + /// Put a non-zero collapsed count into a code's content as one extra line. Keyed by a + /// prefix that sorts below every node fqn (which always begins with '/') - see + /// kCollapsedKeyPrefix for why that is the right end of the list. + static void add_collapsed_content(std::map & affected, int count, const char * left_as) { + if (count <= 0) { + return; + } + affected[std::string(kCollapsedKeyPrefix) + left_as] = + trim_to("and " + std::to_string(count) + " more required node(s) left the graph " + left_as + + "; not named individually (tracked_node_cap is full)", + kMaxLifecycleDetailChars); + } + + static void append_all(std::vector & target, const std::vector & extra) { + target.insert(target.end(), extra.begin(), extra.end()); + } + + /// One tick of the violation streak, recording the tick it CROSSES `grace_` on. Frozen + /// one past the bound like every other clock here: past `grace_ + 1` the stored value + /// only has to distinguish "confirmed" from "still climbing", never how far past, and a + /// counter that only ever increments would eventually overflow on a node that is absent + /// and violating for the life of a long-running process. + void advance_violation_streak(NodeState & node, const std::string & fqn, std::set & crossed) const { + if (node.violation_streak > grace_) { + return; + } + if (++node.violation_streak == grace_ + 1) { + crossed.insert(fqn); + } + } + + /// One tick of the cause-blind unmeasured clock, taking ownership on the tick it matures. + /// Taking it RELEASES the violation streak outright (see "Ownership is exclusive"). + void advance_unmeasured_clock(NodeState & node, const std::string & fqn, std::set & crossed) const { + if (node.unmeasured_matured) { + return; // frozen, and `cause` is STICKY from here - see the class doc + } + if (node.unmeasured_clock <= unmeasured_hold_ticks_) { + ++node.unmeasured_clock; // frozen one past the hold + } + if (node.unmeasured_clock == unmeasured_hold_ticks_ + 1) { + node.unmeasured_matured = true; + node.violation_streak = 0; // RELEASED: GRAPH_NODE_INACTIVE's clear is now free + crossed.insert(fqn); + } + } + + /// Detail phrase for a CONFIRMED violation. `departed` marks a node whose absence has + /// outlived the blink tolerance: its streak is still climbing (absence continues a + /// violation, it does not end one), but "is inactive" on its own would send an operator + /// looking for a node that is no longer in the graph. + static std::string inactive_detail(const std::string & fqn, const NodeState & node, bool departed) { + std::string detail = "node " + fqn + " expected active but is " + node.last_label; + if (departed) { + detail += ", and has since left the graph"; + } + return trim_to(detail + " (required by " + join_quoted(node.entries) + ")", kMaxLifecycleDetailChars); + } + + /// Detail phrase for a matured unmeasured clock, under whichever cause froze at maturity. + /// Carries no live label, so only the whole-detail backstop applies (R13's secondary + /// guard) - the fqn and the "required by" list are the only unbounded parts. + static std::string unmeasured_detail(const std::string & fqn, const NodeState & node, bool departed) { + std::string detail = node.cause == LifecycleUnmeasuredCause::kUnreadable + ? "node " + fqn + " expected active but its lifecycle state could not be read" + : "node " + fqn + " expected active but is not a managed lifecycle node"; + if (departed) { + detail += ", and has since left the graph"; + } + return trim_to(detail + " (required by " + join_quoted(node.entries) + ")", kMaxLifecycleDetailChars); + } + + /// Orders the four observed states by how much they tell the operator, so a duplicate + /// match for one node keeps the MOST informative read: a confirmed violation is the + /// strongest possible signal, an unmeasured read (either flavor) is a weaker "cannot + /// tell" signal, and a healthy read is the least informative (nothing wrong to report). + /// kUnreadable and kNotManaged rank equally - the unmeasured clock is deliberately blind + /// to which of them it is seeing, so a tie between them is broken by iteration order, + /// which never matters: duplicate matches for one node are the same physical app.id read + /// twice, so in practice the two never disagree at all. + static int observed_rank(LifecycleObservedState state) { + switch (state) { + case LifecycleObservedState::kInactive: + return 3; + case LifecycleObservedState::kUnreadable: + return 2; + case LifecycleObservedState::kNotManaged: + return 1; + case LifecycleObservedState::kActive: + return 0; + } + return 0; // unreachable: every enumerator is handled above + } + + /// "'a', '/a'" - every entry that named the node, for a detail's context phrase. + static std::string join_quoted(const std::vector & entries) { + std::string joined; + for (const auto & entry : entries) { + if (!joined.empty()) { + joined += ", "; + } + joined += "'" + entry + "'"; + } + return joined; + } + + /// Key prefix for the collapsed-departed content line. '!' sorts below every character a + /// node fqn can start with ('/'), so among the entries that are NOT crossing on this tick + /// the count is the first thing the description spends its budget on. That is the right + /// order: one line saying N nodes left the graph broken is worth more to an operator than + /// the third name among them, and it cannot displace a node that just broke, because + /// AggregatedFault::describe_ordered emits the caller's `order` list (the newly_* lists, + /// present-first) ahead of any map order at all. + static constexpr const char * kCollapsedKeyPrefix = "!departed: "; + + std::set require_active_; + int grace_; + int absence_grace_; + int no_match_warn_ticks_; + int prune_ticks_; + int unmeasured_hold_ticks_; + int tracked_node_cap_; ///< clamped to at least 1 in the constructor + std::map nodes_; ///< fqn -> the whole per-node state machine + std::map no_match_; ///< entry -> consecutive ticks matching nothing + std::set reported_no_match_; ///< entries already warned about + bool saturated_last_tick_ = false; ///< for the saturation EDGE (see the report) + // Departed entries folded into a count to free slots for present nodes. Monotone within + // one tracker lifetime by design - see "What the operator sees when a count is non-zero". + int collapsed_inactive_ = 0; + int collapsed_unreadable_ = 0; + int collapsed_not_managed_ = 0; +}; + +} // namespace ros2_medkit_graph_watchdog diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/lifecycle_watcher.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/lifecycle_watcher.hpp index 66fcc73a9..1a9c2be84 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/lifecycle_watcher.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/lifecycle_watcher.hpp @@ -77,10 +77,11 @@ class LifecycleWatcher { LifecycleWatcher & operator=(LifecycleWatcher &&) = delete; /// Discover managed nodes from the snapshot (find_lifecycle_get_state_path over - /// App::services): seed+subscribe new ones, drop vanished ones. `tick` is the - /// plugin's own tick counter (threaded through from ReliabilityGate::update()) - - /// used to timestamp a departure into `recently_departed_` and to prune stale - /// entries from it. + /// App::services): seed+subscribe new ones, drop vanished ones, and drop + re-seed + /// tracked ids whose BINDING moved (same App::id, different fqn / get_state path - + /// see Tracked::get_state_path). `tick` is the plugin's own tick counter (threaded + /// through from ReliabilityGate::update()) - used to timestamp a departure into + /// `recently_departed_` and to prune stale entries from it. void update(const ros2_medkit_gateway::IntrospectionInput & snapshot, std::uint64_t tick); /// Execute the ~/transition_event callbacks that are already pending, for at most @@ -132,6 +133,13 @@ class LifecycleWatcher { /// non-active, to recover an `active` transition_event lost during the DDS /// endpoint-matching window right after the (volatile) subscription is created. int reseeds_remaining = 0; + /// The GetState path this entry was created from. Together with `fqn` it is the + /// entry's BINDING identity, which update() re-checks every tick: the subscription + /// topic derives from this path, so if either piece moves under an unchanged + /// App::id, the entry describes a different node than the snapshot now binds and + /// must be dropped + re-seeded (an id kept across such a move would keep enforcing + /// the old node's label against the new binding). + std::string get_state_path; /// Set by the ~/transition_event callback - see DepartedLifecycle for why the last /// label alone is not enough to classify a departure. bool saw_transition = false; @@ -149,8 +157,9 @@ class LifecycleWatcher { std::unordered_map tracked; // key = App::id /// A departed node's observed departure, keyed by its stable fqn (NOT App::id) - /// value = {departure, departed_tick}. Populated on the drop path in update() just - /// before a tracked entry is erased; pruned (in update(), keyed off `tick`) once - /// `tick - departed_tick > retention_ticks_`. + /// before a tracked entry is erased - whether the id left the managed set or its + /// binding moved (either way, the OLD binding departed); pruned (in update(), keyed + /// off `tick`) once `tick - departed_tick > retention_ticks_`. std::map> recently_departed_; bool shutdown_requested = false; }; diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/orphan_policy.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/orphan_policy.hpp index 28c5d0732..d4c944e9a 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/orphan_policy.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/orphan_policy.hpp @@ -197,10 +197,16 @@ inline std::vector find_orphans(const std::vector std::chrono::milliseconds(1); const bool sim_advanced = sim_now.nanoseconds() > last_sim_ns_; - valid_ = !(wall_advanced && !sim_advanced); + // Wall time moving while sim time stands still is the stall. Anything else is fine: + // wall not advancing says nothing, and sim advancing proves the clock is live. + valid_ = !wall_advanced || sim_advanced; } last_sim_ns_ = sim_now.nanoseconds(); last_wall_ = wall_now; diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/lifecycle_expectation_detector.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/lifecycle_expectation_detector.cpp new file mode 100644 index 000000000..477fc25d0 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/lifecycle_expectation_detector.cpp @@ -0,0 +1,560 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "ros2_medkit_gateway/core/providers/introspection_provider.hpp" +#include "ros2_medkit_graph_watchdog/aggregated_fault.hpp" +#include "ros2_medkit_graph_watchdog/detector_config_keys.hpp" +#include "ros2_medkit_graph_watchdog/detector_registry.hpp" +#include "ros2_medkit_graph_watchdog/graph_fault_codes.hpp" +#include "ros2_medkit_graph_watchdog/lifecycle_expectation_tracker.hpp" +#include "ros2_medkit_graph_watchdog/reliability_gate.hpp" + +namespace ros2_medkit_graph_watchdog { + +namespace { +// Fault severity scale: SEVERITY_WARN=1, SEVERITY_ERROR=2 (ros2_medkit_msgs/msg/Fault.msg). +// Three independent fault codes, each fixed at its own severity - mirroring orphan_detector +// and param_drift_detector's fixed-severity AggregatedFault. GRAPH_NODE_INACTIVE (a +// required-active node CONFIRMED stuck inactive) means the robot silently will not act to +// the outside world - as severe as node death. GRAPH_NODE_UNREADABLE and +// GRAPH_NODE_NOT_MANAGED (a required node whose lifecycle promise has never been measured, +// for one of two reasons) are lesser, independent claims: nothing has been measured wrong +// about either, the promise is merely UNVERIFIED - and like every non-CRITICAL severity +// here they settle through the ordinary debounce path rather than bypassing it (see +// ros2_medkit_msgs/msg/Fault.msg's CRITICAL contract). +constexpr std::uint8_t kConfirmedInactiveSeverity = ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR; +constexpr std::uint8_t kUnmeasuredSeverity = ros2_medkit_msgs::msg::Fault::SEVERITY_WARN; +// Consecutive not-active ticks a required node tolerates before being reported inactive. +// Config-overridable via "grace" - gives a managed node bringup time to reach "active" +// (configure + activate) before the operator's expectation is enforced. +constexpr int kDefaultGrace = 5; +// Widest `grace` accepted, in ticks (five minutes at the shipped 1 s cadence). Range-checked +// on the WIDE integer before narrowing for the same reason prune_grace is (see +// kMaxPruneGrace): get() truncates first, so 4294967296 would arrive as 0 and pass the +// >= 0 check - turning the documented "non-negative integer" contract into a hair-trigger +// that reports a node on its very first not-active tick, with no warning and no default. +// +// The upper end used to be INT_MAX - 1, which is not a wide tolerance but an off switch with +// no warning attached. `grace` bounds two things at once: how long a node may read not-active +// before being reported, AND how long a node that LEFT the graph while not-active sits in the +// tracker's pending set - during which GRAPH_NODE_INACTIVE's clear is withheld for every +// node, so the fault can neither raise nor heal for anybody. At INT_MAX - 1 that is roughly +// 24 days at the shipped cadence; five minutes is already an extravagant allowance for a +// managed node to reach `active`, and it makes the worst-case withhold something an operator +// can reason about. A deployment that genuinely needs longer wants a slower tick, not a +// detector that is silent for weeks. +constexpr std::int64_t kMaxGrace = 300; +/// Consecutive ticks an entry must match ONLY unmanaged nodes before the typo warning +/// fires. One transient tick is not evidence: discover_apps() wraps the per-node service +/// enumeration in a try/catch and pushes the app regardless, so a sweep that races service +/// discovery yields an app with no services and no tracked lifecycle state. +constexpr int kUnmanagedWarnTicks = 5; +// Fallback plugin-scope prune_grace (ticks) when the config doesn't carry one - mirrors +// GraphWatchdogPlugin's own default. +constexpr int kDefaultPruneGrace = 60; +// Widest prune_grace accepted, in ticks (one hour at the shipped 1 s cadence). The value is +// range-checked on the WIDE integer before narrowing: get() truncates first, so +// 4294967296 would arrive as 0 and reclaim bookkeeping on the first absent tick, and 2^63-1 +// would arrive as -1 and be dropped in silence. +constexpr std::int64_t kMaxPruneGrace = 3600; +// Widest tracked_node_cap accepted. 32x the shipped default of 512: one tracked node really +// costs on the order of 500 bytes (the NodeState itself, its map node, the fqn key and the +// "required by" entry strings), so a map full at this cap is roughly 8 MB - the most this +// detector may spend on bookkeeping inside a gateway that also holds the entity cache and the +// HTTP server, on hardware that is often an embedded board. A deployment with more than +// sixteen thousand DISTINCT required-node identities alive at once is not a large fleet, it is +// identity churn, and churn is what the cap exists to bound rather than to accommodate. The +// range check runs on the WIDE integer before narrowing, for the same reason grace and +// prune_grace do. Zero is refused rather than clamped: a cap of nothing means the detector +// checks nothing, which is exactly the silence it exists to prevent. +constexpr std::int64_t kMinTrackedNodeCap = 1; +constexpr std::int64_t kMaxTrackedNodeCap = 16384; +/// Consecutive withheld ticks before the reason is put in the log. Withholding is right - +/// health that was never measured is not health - but from the outside it is +/// indistinguishable from a detector that is working and finding nothing, so once per +/// episode the detector says which it is. Ten ticks: label seeding normally completes +/// within a tick or two of the watcher seeing the node, so a hold that lives this long is +/// worth explaining. +constexpr int kWithheldClearReportTicks = 10; +// Ties kMaxLifecycleDetailChars (lifecycle_expectation_tracker.hpp) to the real +// AggregatedFault::kMaxDescriptionChars it is budgeted against - the tracker header does +// not include aggregated_fault.hpp (kept "Pure.", see its class doc), so this is the one +// place that can verify the arithmetic in the tracker's own comment against the actual +// cap: at least 3 maximally-long details must still fit under it. +static_assert(3 * kMaxLifecycleDetailChars + 2 * 2 <= AggregatedFault::kMaxDescriptionChars, + "kMaxLifecycleDetailChars must leave room for at least 3 worst-case details under " + "AggregatedFault::kMaxDescriptionChars"); +} // namespace + +/// Watches operator-declared "must be active" nodes and raises GRAPH_NODE_INACTIVE for +/// one that is present in the graph (alive) but sitting in a non-active lifecycle state +/// for more than `grace` consecutive ticks, GRAPH_NODE_UNREADABLE for one whose lifecycle +/// label this run has never been able to read, and GRAPH_NODE_NOT_MANAGED for one that +/// carries no tracked lifecycle at all. The three are independent, level-triggered +/// aggregates: each one's content comes from its own measurement, and one raising or +/// healing never forces or blocks another. GRAPH_NODE_INACTIVE's own clear is not simply +/// "nothing CONFIRMED inactive this tick", though: it is withheld while any required +/// node's status is UNSETTLED (see the withheld-clear guard in tick()) - the original +/// withheld-clear guarantee this detector always gave, now scoped to GRAPH_NODE_INACTIVE +/// alone. GRAPH_NODE_UNREADABLE and GRAPH_NODE_NOT_MANAGED have no such guard: each one's +/// own clear needs nothing beyond its own content going empty. A node can only ever be +/// content of one of the three at a time - see LifecycleExpectationTracker's own class doc +/// for the state machine (one observed state per node per tick, two clocks) that makes +/// that guarantee structural rather than something this detector has to re-derive. +/// Distinct from the presence class (GRAPH_NODE_DISAPPEARED, pure presence): here the +/// process is ALIVE and in the graph, but a node the operator declared critical-active is +/// either stuck inactive/unconfigured/finalized (GRAPH_NODE_INACTIVE), or its lifecycle +/// promise is simply UNVERIFIED (GRAPH_NODE_UNREADABLE / GRAPH_NODE_NOT_MANAGED) - on a +/// Nav2 stack a controller_server stuck inactive means the robot silently will not act, no +/// crash, no log. Safe default: require_active is empty, so nothing is checked and there +/// are zero false positives until an operator opts specific nodes in (mirrors param_drift +/// being config-scoped). See lifecycle_expectation_tracker.hpp for the pure tracking core +/// and the design doc. +/// +/// Design note (why grace, not the reliability gate): this detector cannot gate its +/// raise on reliability_allows() - it ANDs lifecycle_.node_ok(), which is FALSE for +/// exactly the inactive node this detector exists to catch, so gating on it would +/// suppress the signal forever. Instead the tracker counts consecutive not-active ticks +/// itself and raises only past its own grace, independent of the central gate (which +/// still applies to the aggregated RAISE itself via ctx.raise_fault - see +/// DetectorContext - for warmup/bringup-quiesce, just not for the lifecycle-inactive +/// state this detector is specifically built to report). +class LifecycleExpectationDetector : public Detector { + public: + std::string id() const override { + return "lifecycle_expectation"; + } + + void configure(const nlohmann::json & config) override { + std::vector warnings; + std::set require_active; + if (config.contains("require_active")) { + if (config["require_active"].is_array()) { + for (const auto & entry : config["require_active"]) { + if (entry.is_string() && !entry.get().empty()) { + require_active.insert(entry.get()); + } else { + // Skip "" - it would match every unbound app's empty leaf. Silently dropping the + // entry leaves the operator believing the node is covered, so it has to say so. + warnings.push_back("'require_active' entries must be non-empty node names; skipping one"); + } + } + } else { + warnings.push_back("'require_active' must be a string array of node names; ignoring it"); + } + } + int grace = kDefaultGrace; + if (config.contains("grace")) { + const auto & value = config["grace"]; + const std::int64_t wide = value.is_number_integer() ? value.get() : -1; + if (wide >= 0 && wide <= kMaxGrace) { + grace = static_cast(wide); + } else { + warnings.push_back("'grace' must be an integer in 0.." + std::to_string(kMaxGrace) + "; keeping the default (" + + std::to_string(kDefaultGrace) + ")"); + } + } + // prune_grace is injected into every detector's config by the plugin and is exempt from the + // unknown-key warning, so ignoring a bad value here would accept the key with no warning and + // no effect - the exact silent failure the README says cannot happen. Range-check on the + // WIDE value before narrowing (see kMaxPruneGrace). + int prune_grace = kDefaultPruneGrace; + if (config.contains("prune_grace")) { + const auto & value = config["prune_grace"]; + const std::int64_t wide = value.is_number_integer() ? value.get() : -1; + if (wide >= 0 && wide <= kMaxPruneGrace) { + prune_grace = static_cast(wide); + } else { + warnings.push_back("'prune_grace' must be an integer in 0.." + std::to_string(kMaxPruneGrace) + "; keeping " + + std::to_string(kDefaultPruneGrace)); + } + } + // The hard bound on how many nodes the tracker keeps state for. Configurable for the same + // reason every other bound in this detector is: the README already presents 512 as a number + // an operator reasons about, and a deployment whose require_active entries legitimately + // match more than 512 PRESENT nodes at once otherwise has no lever at all - the detector + // simply stops checking the excess and says so once. Same wide-integer range check as + // grace and prune_grace above. + int tracked_node_cap = kDefaultTrackedNodeCap; + if (config.contains("tracked_node_cap")) { + const auto & value = config["tracked_node_cap"]; + const std::int64_t wide = value.is_number_integer() ? value.get() : -1; + if (wide >= kMinTrackedNodeCap && wide <= kMaxTrackedNodeCap) { + tracked_node_cap = static_cast(wide); + } else { + warnings.push_back("'tracked_node_cap' must be an integer in " + std::to_string(kMinTrackedNodeCap) + ".." + + std::to_string(kMaxTrackedNodeCap) + "; keeping the default (" + + std::to_string(kDefaultTrackedNodeCap) + ")"); + } + } + collect_unknown_detector_keys(config, known_keys(), warnings); + require_active_ = require_active; + // prune_grace goes through unclamped: it is the age horizon for IDLE bookkeeping ONLY + // (both clocks at zero, no matured ownership, so nothing to lose). A node carrying + // evidence is never reclaimed by age at all, so there is no longer anything for a + // grace-derived floor to protect - what bounds the map when nothing is idle is + // kDefaultTrackedNodeCap. See LifecycleExpectationTracker's "Bounded by evidence, not + // by age". + tracker_ = LifecycleExpectationTracker(require_active_, grace, kDefaultAbsenceGrace, kDefaultNoMatchWarnTicks, + prune_grace, kDefaultUnmeasuredHoldTicks, tracked_node_cap); + tracked_node_cap_ = tracked_node_cap; + unmanaged_streak_.clear(); + // The one-time warning is scoped to the CURRENT config. Without this, an entry that + // already warned stays permanently silenced across a reconfigure - including after it + // was removed and re-added, which is exactly when the operator wants to hear that it + // still names nothing managed. + warned_.clear(); + warnings_ = std::move(warnings); + warnings_logged_ = false; + // The withheld-clear guard restarts with the config: a reconfigure rebuilds the + // tracker above (its own map is the ENTIRE per-node state machine now), so the guard + // has nothing left to carry over even without a separate clear() of its own. + ever_matched_.clear(); + unmatched_ticks_ = 0; + withheld_ticks_ = 0; + withheld_reported_ = false; + } + + /// Delegates straight to the tracker: every per-node fact (both clocks, absence, + /// entries, cause) lives in its one map now, so there is no second bookkeeping + /// structure in this class to reconcile against. + std::size_t tracked_count_for_test() const override { + return tracker_.tracked_count(); + } + + /// Surfaced under `detectors.lifecycle_expectation` on GET /x-medkit-watchdog. Built from + /// atomics only: this runs on an HTTP handler thread while tick() writes them on the + /// plugin's tick thread, and the handler holds no lock this detector takes. + /// + /// `tracking_saturated` is the one an operator acts on. It means the detector has REFUSED + /// to track a required node because `tracked_node_cap` is full of present nodes that are + /// all carrying evidence - so that node is going unchecked, and GRAPH_NODE_INACTIVE's + /// clear is withheld for as long as it lasts. The fix is either a `require_active` entry + /// that matches fewer identities (a bare name on a graph whose nodes respawn under + /// ever-new namespaces matches an unbounded set) or a larger `tracked_node_cap`, which is + /// why the cap and the current count are reported beside it. + nlohmann::json status_json() const override { + return nlohmann::json{{"tracking_saturated", saturated_.load()}, + {"tracked_nodes", tracked_nodes_.load()}, + {"tracked_node_cap", tracked_node_cap_.load()}}; + } + + void tick(DetectorContext & ctx) override { + // Before the zero-config early return, deliberately: the misspelt key most worth + // reporting is `require_activ`, and that typo is exactly the config in which + // require_active_ is empty. + log_warnings_once(ctx); + if (require_active_.empty()) { + // require_active_ is empty - a config that never set it, or a reconfigure that + // dropped it - so there is nothing to check and nothing below this line ever runs + // while it stays empty (preserves EmptyConfigNeverContactsTheFaultManager / + // ExplicitEmptyRequireActiveNeverContactsTheFaultManager). A record this detector + // already raised under an OLDER, non-empty config is left exactly where it is: + // GraphWatchdogPlugin::set_context() creates and configures every detector exactly + // once, so no caller can reach this branch after a prior non-empty configure() in + // the same process, and a stale record left behind when an operator removes a + // detector's configuration is a question about who clears a fault whose detector no + // longer exists - true of every detector in this package, and not solved here. + return; + } + if (!ctx.snapshot) { + return; // zero-config safe default handled above; here there is just nothing to check yet + } + + // Match each configured require_active entry against a live app by App::id OR its stable + // effective_fqn() OR the bare leaf name of that fqn. App::id alone is unstable: it is + // recomputed each sweep and gets a namespace prefix once a same-bare-name collision exists + // anywhere in the graph, so a bare-name config would silently stop matching on a + // multi-robot graph - the worst failure mode for a silent-fault detector. A bare name + // therefore matches every namespace's node of that name; use a full FQN to pin one. + std::vector matches; + std::set entry_has_managed_match; // entries matching >=1 app with a tracked lifecycle state + for (const auto & app : ctx.snapshot->apps) { + const std::string fqn = app.effective_fqn(); + if (fqn.empty()) { + continue; // an unnamed entry can never be useful in a fault + } + const std::string leaf = fqn.substr(fqn.find_last_of('/') + 1); + for (const auto & id : require_active_) { + if (id != app.id && id != fqn && id != leaf) { + continue; + } + const std::optional state = ctx.gate ? ctx.gate->lifecycle_state_of(app.id) : std::nullopt; + if (state.has_value()) { + entry_has_managed_match.insert(id); + } + // One match per (entry, node): the tracker keys violations by NODE, so two + // namesakes are both reported instead of one silently replacing the other. + matches.push_back(LifecycleMatch{id, fqn, state}); + } + } + std::set matched_entries; + for (const auto & match : matches) { + matched_entries.insert(match.entry); + } + // Third leg of the withheld-clear guard: an entry that has NEVER matched a node since + // configure(). The tracker's own pending/affected/*_affected maps are all keyed by a + // matched node, so in the window before the first match they are empty, the tracker + // reports nothing settled either way, and the clear flows about a node the detector has + // not once looked at. That window is not exotic - it is every restart: the plugin ticks + // as soon as it is loaded, while the entity snapshot is still catching up with the + // graph. Deliberately NOT the same as an entry whose node VANISHES after having + // matched: that is a presence problem and the clear is correct there (the tracker makes + // the same handoff). Bounded by the same hold as the unmeasured legs, so a misspelt + // entry cannot block healing for the process lifetime. + ever_matched_.insert(matched_entries.begin(), matched_entries.end()); + const bool any_never_matched = ever_matched_.size() < require_active_.size(); + if (!any_never_matched) { + unmatched_ticks_ = 0; + } else if (unmatched_ticks_ <= kDefaultUnmeasuredHoldTicks) { + ++unmatched_ticks_; // frozen one past the hold, like the tracker's own clocks + } + const bool unmatched_blocking = + any_never_matched && unmatched_ticks_ >= 1 && unmatched_ticks_ <= kDefaultUnmeasuredHoldTicks; + + // An entry that matches present nodes but NONE with a tracked lifecycle state is + // probably a typo (or a plain, non-managed node). Requiring N CONSECUTIVE such ticks + // before saying so is what keeps that from latching on a transient: LifecycleWatcher + // only tracks a node whose get_state service was seen on THIS sweep, and + // discover_apps() can yield an app with empty services when a sweep races service + // enumeration - one such tick used to pin the accusation for the process lifetime, + // since warned_ is only cleared by configure() and configure() runs once. + // + // Requires ctx.gate: with no gate wired, lifecycle_state_of() is never consulted and + // EVERY entry looks unmanaged, so this would accuse every correct entry of being a typo. + if (ctx.gateway_node && ctx.gate) { + for (const auto & id : matched_entries) { + if (entry_has_managed_match.count(id) != 0) { + unmanaged_streak_.erase(id); // resolved: it may warn again if it regresses + warned_.erase(id); + continue; + } + if (++unmanaged_streak_[id] > kUnmanagedWarnTicks && warned_.insert(id).second) { + RCLCPP_WARN(ctx.gateway_node->get_logger(), + "graph_watchdog lifecycle_expectation: require_active entry '%s' has matched a present node " + "with no tracked lifecycle state for %d consecutive ticks (not a managed rclcpp_lifecycle " + "node?) - check for a typo", + id.c_str(), kUnmanagedWarnTicks + 1); + } + } + } + + auto report = tracker_.update(matches); + // Published for GET /x-medkit-watchdog. Written here on the plugin's tick thread and + // read by status_json() on an HTTP handler thread, which holds no lock this detector + // takes - hence atomics rather than plain members. + saturated_.store(report.tracking_saturated); + tracked_nodes_.store(tracker_.tracked_count()); + // An entry that matches nothing at all is reported by the tracker, not here: the loop + // above only sees entries that DID match a present node. + if (ctx.gateway_node) { + if (report.saturation_started) { + // Once per EPISODE, not once per process: the latch re-arms when saturation ends, so + // a later, real saturation is not silent because an earlier one spent the one + // warning. Refusing the newcomer is the safe direction (no live violation is evicted + // to make room), but a required node is going unchecked, and that must never be + // silent. + RCLCPP_WARN(ctx.gateway_node->get_logger(), + "graph_watchdog lifecycle_expectation: already tracking %d required nodes - the most this " + "detector keeps state for - and every one of them is carrying evidence, so a newly matched " + "node is NOT being checked. A require_active entry is matching an unbounded set of node " + "identities (a bare name on a graph whose nodes respawn under ever-new namespaces?); pin " + "the nodes you mean with full FQNs, or raise 'tracked_node_cap'", + tracked_node_cap_.load()); + } + for (const auto & entry : report.entries_matching_nothing) { + // The tracker counts CONSECUTIVE no-match ticks, so it also surfaces an entry whose + // node matched and later left the graph. The warning below says the opposite in two + // places ("no node at all since startup", and that the presence class cannot report + // it), and for a node that WAS present both statements are false - a departed node + // is precisely what GRAPH_NODE_DISAPPEARED owns. Same handoff the never-matched + // hold makes above: an entry that has matched once is never treated as missing + // again. + if (ever_matched_.count(entry) != 0) { + continue; + } + RCLCPP_WARN(ctx.gateway_node->get_logger(), + "graph_watchdog lifecycle_expectation: require_active entry '%s' has matched no node at all " + "since startup - a misspelt entry, or a required node that never came up (the presence class " + "GRAPH_NODE_DISAPPEARED cannot report it either: it only tracks nodes that were present at " + "least once)", + entry.c_str()); + } + } + + // GRAPH_NODE_UNREADABLE and GRAPH_NODE_NOT_MANAGED: independent of GRAPH_NODE_INACTIVE + // and never withheld by its guard - a node whose unmeasured clock matured is a settled + // fact for as long as it stays matured, not something to wait on further. Both fixed + // SEVERITY_WARN. Level-triggered every tick, same as every other aggregate here: + // non-empty content raises (or re-raises), empty content clears. + aggregated_unreadable_.emit_ordered(ctx, report.unreadable_affected, report.newly_unreadable); + aggregated_not_managed_.emit_ordered(ctx, report.not_managed_affected, report.newly_not_managed); + + // Withheld-clear guard - for GRAPH_NODE_INACTIVE ONLY. The emitter is level-triggered: + // an empty affected map is a clear, and a clear asserts that every required node is + // healthy. Two things can make `report.affected` empty without that being true: an + // entry that has never matched anything yet (`unmatched_blocking`, entry-keyed - see + // above), and any node whose status the tracker itself reports UNSETTLED this tick + // (`report.pending`, node-keyed - see LifecycleExpectationTracker's own class doc for + // exactly what feeds it: a violation streak that has not yet passed grace, an + // unmeasured clock still climbing, or a streak held while the node is inside an + // unmeasured spell). Absence is NOT one of them any more: content follows the clocks + // rather than the snapshot, so a node already past grace stays in `affected` while it + // blinks instead of emptying the map. A node whose + // unmeasured clock has MATURED is neither: ownership passed to its own fault code and + // its violation streak was released, so it is not pending either - which is exactly + // what lets GRAPH_NODE_INACTIVE's clear flow the moment a node's "cannot measure this" + // status is fully resolved into content elsewhere, rather than continuing to poison + // this fault's withhold decision the way a shared record used to. + // + // Saturation is the third reason, and unlike the other two it is not a transient: once a + // departed entry can no longer crowd out a present one, a refusal means there are + // genuinely more required PRESENT nodes than `tracked_node_cap` allows. That is a + // capacity condition an operator resolves by raising the key, so it gets no bounded hold + // of its own - unlike `unmatched_blocking`, which is bounded precisely because a misspelt + // entry must not block healing forever. A detector that declined to check a required + // node cannot assert that every required node is healthy, for as long as it keeps + // declining. + if (report.affected.empty() && (!report.pending.empty() || unmatched_blocking || report.tracking_saturated)) { + report_withheld_clear(ctx, report.pending_violation, report.pending_unreadable, report.pending_not_managed, + unmatched_blocking, report.tracking_saturated); + return; // GRAPH_NODE_INACTIVE: nothing measured - and a young streak - is not the same as healthy + } + withheld_ticks_ = 0; + withheld_reported_ = false; + // GRAPH_NODE_INACTIVE: fixed SEVERITY_ERROR, like orphan_detector and + // param_drift_detector's AggregatedFault members - `report.affected` can no longer be + // anything but a CONFIRMED violation, so there is nothing left for a per-tick severity + // choice to decide. + aggregated_inactive_.emit_ordered(ctx, report.affected, report.newly_affected); + } + + private: + static const std::set & known_keys() { + static const std::set keys{"require_active", "grace", "tracked_node_cap"}; + return keys; + } + + void log_warnings_once(const DetectorContext & ctx) { + if (warnings_.empty() || warnings_logged_ || !ctx.gateway_node) { + return; + } + for (const auto & warning : warnings_) { + RCLCPP_WARN(ctx.gateway_node->get_logger(), "graph_watchdog lifecycle_expectation: %s", warning.c_str()); + } + warnings_logged_ = true; + } + + /// Say, once per withhold episode, that the clear is being withheld and why. + /// Withholding is correct - health that was never measured is not health, and neither is + /// a violation the counter has not finished counting - but from the outside it is + /// indistinguishable from a detector that is working and finding nothing: no raise, no + /// clear, and a GRAPH_NODE_INACTIVE already in the store simply never heals. Once per + /// episode, past a horizon the normal label seeding and a normal grace both fit inside. + /// Every reason is named separately because each releases on a different condition - + /// `not_managed` and `unreadable` each release into their OWN fault code now, while + /// `violation` releases either into content (GRAPH_NODE_INACTIVE itself) or into health, + /// so the three cannot share one sentence. + void report_withheld_clear(const DetectorContext & ctx, const std::set & pending_violation, + const std::set & pending_unreadable, + const std::set & pending_not_managed, bool unmatched, bool saturated) { + ++withheld_ticks_; + if (withheld_ticks_ <= kWithheldClearReportTicks || withheld_reported_ || !ctx.gateway_node) { + return; + } + withheld_reported_ = true; + std::string reason; + const auto add = [&reason](const std::string & phrase) { + if (!reason.empty()) { + reason += " and "; + } + reason += phrase; + }; + if (saturated) { + add("a required node could not be tracked at all because 'tracked_node_cap' (" + + std::to_string(tracked_node_cap_.load()) + + ") is full of present nodes carrying evidence (that hold releases when a slot frees up, or when the " + "cap is raised)"); + } + if (unmatched) { + add("a require_active entry has not matched any node yet (that hold releases as soon as it matches, " + "or by itself after " + + std::to_string(kDefaultUnmeasuredHoldTicks) + " ticks)"); + } + if (!pending_not_managed.empty()) { + add(std::to_string(pending_not_managed.size()) + + " required node(s) have had no lifecycle label read this run (starting with '" + + *pending_not_managed.begin() + "'; after " + std::to_string(kDefaultUnmeasuredHoldTicks) + + " ticks that hold releases by itself and the node is reported under GRAPH_NODE_NOT_MANAGED instead)"); + } + if (!pending_unreadable.empty()) { + add(std::to_string(pending_unreadable.size()) + + " required node(s) are matched to a managed node whose lifecycle label has never answered " + "(starting with '" + + *pending_unreadable.begin() + "'; after " + std::to_string(kDefaultUnmeasuredHoldTicks) + + " unanswered ticks that hold releases by itself and the node is reported under " + "GRAPH_NODE_UNREADABLE instead)"); + } + if (!pending_violation.empty()) { + add(std::to_string(pending_violation.size()) + + " required node(s) are measured not-active but still within grace " + "(starting with '" + + *pending_violation.begin() + "'; that hold releases as soon as the node reads active or " + + "its streak passes grace)"); + } + RCLCPP_WARN(ctx.gateway_node->get_logger(), + "graph_watchdog lifecycle_expectation: nothing is reported inactive, but GRAPH_NODE_INACTIVE is " + "withheld from clearing because %s", + reason.c_str()); + } + + std::set require_active_; + std::set warned_; // require_active entries already warned about (no lifecycle state) + std::map unmanaged_streak_; // entry -> consecutive ticks matching only unmanaged nodes + std::vector warnings_; // configure()-time findings, surfaced once per config + bool warnings_logged_ = false; + std::set ever_matched_; // require_active entries that have matched >=1 node this config + int unmatched_ticks_ = 0; // consecutive ticks with an entry that has never matched (frozen past the hold) + int withheld_ticks_ = 0; // consecutive ticks the clear has been withheld + bool withheld_reported_ = false; // withhold already explained this episode + /// Written by configure()/tick() on the plugin's tick thread, read by status_json() on an + /// HTTP handler thread - atomics, because that route takes no lock this detector holds. + std::atomic tracked_node_cap_{kDefaultTrackedNodeCap}; + std::atomic saturated_{false}; ///< the cap refused a required node on the last tick + std::atomic tracked_nodes_{0}; ///< tracker map size as of the last tick + LifecycleExpectationTracker tracker_{{}, kDefaultGrace}; + // Fixed-severity AggregatedFault members, one per code - the same shape orphan_detector + // and param_drift_detector use, now that each code's content decides its own emission + // independently. + AggregatedFault aggregated_inactive_{graph_fault_codes::kNodeInactive, kConfirmedInactiveSeverity}; + AggregatedFault aggregated_unreadable_{graph_fault_codes::kNodeUnreadable, kUnmeasuredSeverity}; + AggregatedFault aggregated_not_managed_{graph_fault_codes::kNodeNotManaged, kUnmeasuredSeverity}; +}; + +REGISTER_DETECTOR(LifecycleExpectationDetector, "lifecycle_expectation") + +} // namespace ros2_medkit_graph_watchdog diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/orphan_detector.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/orphan_detector.cpp index 6654236e1..2350efa0a 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/orphan_detector.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/orphan_detector.cpp @@ -77,7 +77,7 @@ constexpr int kDefaultGrace = 10; /// Watches every topic for a one-sided endpoint (pub-only or sub-only) with a near-miss, /// same-type counterpart in the same namespace carrying the complementary side - the signature of /// a remap / topic-name typo. See orphan_policy.hpp for the pure matching core (dedup, -/// system-topic skip, same-namespace guard) and design doc / [[project_graph_watchdog_zero_config]]. +/// system-topic skip, same-namespace guard) and design doc. class OrphanDetector : public Detector { public: std::string id() const override { diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/qos_mismatch_detector.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/qos_mismatch_detector.cpp index 1b61bee33..3b5676cf6 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/qos_mismatch_detector.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/qos_mismatch_detector.cpp @@ -51,8 +51,8 @@ constexpr int kDefaultGrace = 3; /// Human-readable owner of an endpoint, for the fault description. std::string endpoint_name(const rclcpp::TopicEndpointInfo & endpoint) { - const std::string ns = endpoint.node_namespace(); - const std::string name = endpoint.node_name(); + const std::string & ns = endpoint.node_namespace(); + const std::string & name = endpoint.node_name(); if (ns.empty() || ns == "/") { return "/" + name; } @@ -80,7 +80,7 @@ std::string join_names(const std::set & names) { /// else in the system reports it. /// /// A subscriber with zero publishers is an orphan (a different detector), not a QoS fault. -/// See design doc / [[project_graph_watchdog_zero_config]]. +/// See design doc. class QosMismatchDetector : public Detector { public: std::string id() const override { diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/graph_watchdog_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/graph_watchdog_plugin.cpp index 7177162e6..f6a7d54d7 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/graph_watchdog_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/graph_watchdog_plugin.cpp @@ -508,7 +508,23 @@ std::vector GraphWatchdogPlugin::get_routes() "graph_watchdog reliability gate not initialized"); return; } - res.send_json(gate_->status_json()); + auto payload = gate_->status_json(); + // Detector-scoped status, beside the gate's own. `detectors_` is only ever mutated by + // set_context() and by shutdown() (which holds tick_mutex_, taken above), and each + // detector's own status_json() is required to be safe against a concurrent tick - see + // Detector::status_json. + nlohmann::json detectors = nlohmann::json::object(); + for (const auto & [detector, mode] : detectors_) { + (void)mode; + auto status = detector->status_json(); + if (!status.is_null()) { + detectors[detector->id()] = std::move(status); + } + } + if (!detectors.empty()) { + payload["x-medkit-watchdog"]["detectors"] = std::move(detectors); + } + res.send_json(payload); }; std::vector routes; routes.push_back({"GET", R"(x-medkit-watchdog)", std::move(handler)}); diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/lifecycle_watcher.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/lifecycle_watcher.cpp index e47a19fb2..891a29929 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/lifecycle_watcher.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/lifecycle_watcher.cpp @@ -131,9 +131,20 @@ void LifecycleWatcher::update(const ros2_medkit_gateway::IntrospectionInput & sn current_fqns.emplace(app.id, app.effective_fqn()); } - // Drop nodes that are no longer managed. Erasing the entry runs ~Subscription, which - // mutates node_'s entity registry and waitset - the same structures create_subscription - // mutates. Two things make that safe, and BOTH are needed: + // Drop nodes that are no longer managed - and tracked ids whose BINDING moved. An + // entry's binding identity is (fqn, get_state path), both captured at first sighting: + // App::id alone is not it, because an id can survive a graph sweep while pointing at a + // DIFFERENT node (id assignment shifts under bare-name collisions). Keeping such an + // entry would keep enforcing the old node's label - and its old ~/transition_event + // subscription - against the new binding, so a moved binding is two events at once: + // the OLD binding departed (recorded under ITS fqn, same record and retention as a + // vanish), and the id is new again (erased here, so the selection below re-seeds it + // through the ordinary new-node path: fresh GetState, fresh subscription, fresh + // self-heal budget). + // + // Erasing the entry runs ~Subscription, which mutates node_'s entity registry and + // waitset - the same structures create_subscription mutates. Two things make that safe, + // and BOTH are needed: // // 1. The subscription is in lifecycle_group_, which no gateway executor collects, so // the only references to it live on this thread. A subscription in the gateway's @@ -148,7 +159,14 @@ void LifecycleWatcher::update(const ros2_medkit_gateway::IntrospectionInput & sn std::lock_guard node_lock(*node_mutex_); std::lock_guard state_lock(state_->mutex); for (auto it = state_->tracked.begin(); it != state_->tracked.end();) { - if (current_paths.find(it->first) == current_paths.end()) { + const auto path_it = current_paths.find(it->first); + bool drop = (path_it == current_paths.end()); + if (!drop) { + const auto fqn_it = current_fqns.find(it->first); + drop = fqn_it == current_fqns.end() || fqn_it->second != it->second.fqn || + path_it->second != it->second.get_state_path; + } + if (drop) { state_->recently_departed_[it->second.fqn] = { DepartedLifecycle{it->second.state_label, it->second.saw_transition, it->second.error_terminated}, tick}; it = state_->tracked.erase(it); @@ -186,9 +204,10 @@ void LifecycleWatcher::update(const ros2_medkit_gateway::IntrospectionInput & sn // job queued behind it that tick used to be skipped AND charged - two ticks of that // drained both attempts with zero reads. The consequences are permanent for the process: // a node whose seed never ran keeps label "" (never gated, and a require_active - // expectation on it is silently never enforced), and one seeded non-active that then - // activates inside the subscription's matching window keeps that label forever (every - // fault suppressed, plus a permanent false GRAPH_NODE_INACTIVE). + // expectation on it can never be CONFIRMED - GRAPH_NODE_UNREADABLE is what reports that + // instead, once its own hold expires), and one seeded non-active that then activates + // inside the subscription's matching window keeps that label forever (every fault + // suppressed, plus a permanent false GRAPH_NODE_INACTIVE). jobs.push_back({id, path, false}); } } @@ -248,6 +267,12 @@ void LifecycleWatcher::update(const ros2_medkit_gateway::IntrospectionInput & sn if (shared->shutdown_requested) { return; } + // No re-check that this entry is still the binding the subscription was created + // for: a moved binding erases the entry, and erasing it destroys THIS + // subscription, on this very thread (see the class note in the header - the + // private executor that runs this callback is pumped by the same thread that + // runs update()). A message from a binding that is already gone therefore has + // nowhere to arrive from. auto it = shared->tracked.find(id); if (it != shared->tracked.end()) { it->second.state_label = msg->goal_state.label; @@ -269,12 +294,17 @@ void LifecycleWatcher::update(const ros2_medkit_gateway::IntrospectionInput & sn tracked.state_label = (seed_it != seeded.end()) ? seed_it->second : ""; // Captured at first sighting: current_fqns is built from the same snapshot pass // that produced current_paths, so an id here is always present in current_fqns too. + // Together, fqn + get_state_path are the binding identity the drop loop re-checks. const auto fqn_it = current_fqns.find(id); tracked.fqn = (fqn_it != current_fqns.end()) ? fqn_it->second : ""; + tracked.get_state_path = get_state_path; } // Apply re-seed results to already-tracked nodes. Never overwrite a good cached label - // with an empty (failed/timed-out) seed. + // with an empty (failed/timed-out) seed. The read above ran with no lock held, but no + // ~/transition_event can have overtaken it: the private executor that delivers them is + // pumped by this same thread, between ticks, so nothing writes state_label while a seed + // is in flight (see the class note in the header). { std::lock_guard lock(state_->mutex); // Charge the self-heal budget only for reads that actually happened (see the selection diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/droppable_lifecycle_node.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/droppable_lifecycle_node.cpp new file mode 100644 index 000000000..824b18f00 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/droppable_lifecycle_node.cpp @@ -0,0 +1,147 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file droppable_lifecycle_node.cpp + * @brief Demo node that looks managed, answers a chosen lifecycle label, and can STOP + * looking managed on command. + * + * Proof fixture for two claims the lifecycle_expectation detector makes that no other + * fixture in this workspace can reach: + * + * 1. A present, healthy, managed node whose `get_state` path is missing from a sweep reads + * "not a managed lifecycle node" for a tick or two. If that happens to be the last thing + * observed before the node shuts down cleanly, the departure carve-out must NOT turn a + * healthy departure into a fault. Dropping the services on command and then being killed + * is exactly that sequence, produced deliberately instead of waited for. + * 2. The same drop, held for longer, is a node that is GENUINELY not managed when it leaves - + * which must still be reported. The only difference between the two runs is how long the + * dropped state is held, which is what makes the pair discriminating. + * + * It also supplies the one state no other fixture can: a node that reads a NON-active + * lifecycle label and can then leave the managed set WITHOUT becoming healthy and WITHOUT + * leaving the graph. `managed_lifecycle_node.cpp` is a real rclcpp_lifecycle::LifecycleNode + * and cannot un-advertise its lifecycle services; `unreadable_lifecycle_node.cpp` never + * answers at all, so it can never be measured not-active in the first place. + * + * Like `unreadable_lifecycle_node.cpp`, this is a plain rclcpp::Node that only has to look + * managed to the gateway's discovery layer: `find_lifecycle_get_state_path()` matches purely + * on discovered SERVICE TYPE (one `lifecycle_msgs/srv/GetState` plus one + * `lifecycle_msgs/srv/ChangeState`), never on the node's C++ type. Unlike that fixture, this + * one answers `get_state` immediately - the interesting variable here is whether the services + * EXIST, not whether they respond. + * + * Parameters: + * - `state_label` (string, default "active"): the label `get_state` answers with. "active" + * also reports the ACTIVE state id; anything else reports UNCONFIGURED's id with the + * given label, which is what the detector classifies as a violation. + * - `drop_services` (bool, default false): setting it true at runtime - through the node's + * own auto-started `~/set_parameters` service - destroys both lifecycle services, so the + * node stays in the graph but stops being a managed lifecycle node. It is a parameter + * rather than a timer so the test controls exactly WHEN, after it has independently + * confirmed the starting state, instead of racing a fixed delay against the detector's own + * horizons. Setting it back to false does not re-advertise: this fixture models a node + * losing its lifecycle interface, and a re-advertise would need a second, unrelated + * discovery round trip to be meaningful. + */ + +#include +#include +#include + +#include +#include +#include +#include + +class DroppableLifecycleNode : public rclcpp::Node { + public: + DroppableLifecycleNode() : Node("droppable_lifecycle") { + state_label_ = this->declare_parameter("state_label", "active"); + const bool drop_at_start = this->declare_parameter("drop_services", false); + + advertise(); + if (drop_at_start) { + drop_services(); + } + + param_callback_handle_ = + this->add_on_set_parameters_callback([this](const std::vector & params) { + rcl_interfaces::msg::SetParametersResult result; + result.successful = true; + for (const auto & param : params) { + if (param.get_name() == "drop_services" && param.as_bool()) { + drop_services(); + } + } + return result; + }); + + RCLCPP_INFO(get_logger(), + "droppable_lifecycle started: get_state answers '%s'; set drop_services:=true to stop advertising " + "the lifecycle services", + state_label_.c_str()); + } + + ~DroppableLifecycleNode() override { + get_state_service_.reset(); + change_state_service_.reset(); + param_callback_handle_.reset(); + } + DroppableLifecycleNode(const DroppableLifecycleNode &) = delete; + DroppableLifecycleNode & operator=(const DroppableLifecycleNode &) = delete; + DroppableLifecycleNode(DroppableLifecycleNode &&) = delete; + DroppableLifecycleNode & operator=(DroppableLifecycleNode &&) = delete; + + private: + void advertise() { + get_state_service_ = this->create_service( + "~/get_state", [this](const std::shared_ptr & /*request*/, + const std::shared_ptr & response) { + response->current_state.id = state_label_ == "active" + ? lifecycle_msgs::msg::State::PRIMARY_STATE_ACTIVE + : lifecycle_msgs::msg::State::PRIMARY_STATE_UNCONFIGURED; + response->current_state.label = state_label_; + }); + change_state_service_ = this->create_service( + "~/change_state", [](const std::shared_ptr & /*request*/, + const std::shared_ptr & response) { + // Never driven: only find_lifecycle_get_state_path()'s type check needs it to exist. + response->success = true; + }); + } + + void drop_services() { + if (!get_state_service_ && !change_state_service_) { + return; // already dropped - a second drop_services:=true is a no-op + } + get_state_service_.reset(); + change_state_service_.reset(); + RCLCPP_INFO(get_logger(), + "droppable_lifecycle dropped its lifecycle services: still in the graph, no longer a " + "managed lifecycle node"); + } + + std::string state_label_; + rclcpp::Service::SharedPtr get_state_service_; + rclcpp::Service::SharedPtr change_state_service_; + rclcpp::node_interfaces::OnSetParametersCallbackHandle::SharedPtr param_callback_handle_; +}; + +int main(int argc, char ** argv) { + rclcpp::init(argc, argv); + rclcpp::spin(std::make_shared()); + rclcpp::shutdown(); + return 0; +} diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py index 5021d5f5f..7378e2a88 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py @@ -31,8 +31,10 @@ on PYTHONPATH once the workspace's install/setup.bash is sourced. """ +import http.server import json import os +import threading import time from launch import LaunchDescription @@ -57,6 +59,8 @@ def create_watchdog_test_launch( port=None, demo_delay=2.0, healing_enabled=True, + healing_threshold=3, + gateway_respawn=False, ): """Build a ``LaunchDescription`` that loads graph_watchdog into a real gateway. @@ -84,6 +88,23 @@ def create_watchdog_test_launch( Passed to the fault_manager as ``healing_enabled`` so a detector's level-triggered PASSED clears can actually advance the debounce counter to HEALED (see the plugin README, "Closing the loop"). + healing_threshold : int + Passed to the fault_manager as ``healing_threshold`` - the debounce + counter value a fault's consecutive PASSED reports must reach before + it heals. Defaults to 3, the fault_manager's own built-in default + (see ``FaultManagerNode``'s ``declare_parameter``), so passing it + through here changes no existing scenario's behaviour. A scenario + whose subject IS the debounce counter (a low threshold makes it + sensitive to even a single spurious PASSED) overrides this directly + rather than adding a second mechanism. + gateway_respawn : bool + If True, ``launch`` restarts the gateway when it exits. For the one + scenario whose subject is a gateway restart: the fault_manager and the + demo nodes are separate processes, so killing only the gateway leaves + a raised fault in the store and brings the plugin back with every + detector counter at zero - the state no single-process launch can + reach. Off by default so an unexpected gateway death stays a visible + failure in every other scenario. Returns ------- @@ -104,12 +125,13 @@ def create_watchdog_test_launch( if extra_gateway_params: params.update(extra_gateway_params) - gateway_node = create_gateway_node(port=port, extra_params=params) + gateway_node = create_gateway_node(port=port, extra_params=params, respawn=gateway_respawn) delayed_actions = create_demo_nodes(demo_nodes if demo_nodes is not None else []) delayed_actions.append(create_fault_manager_node( extra_params={ 'healing_enabled': healing_enabled, + 'healing_threshold': healing_threshold, # AUTOSAR DEM-style debounce: -2 confirms a fault on its very first # FAILED report instead of requiring several ticks to accumulate, # so the positive-control assertion is fast and deterministic. @@ -257,6 +279,111 @@ def wait_until_watchdog_armed(port, timeout=60.0, interval=0.5, app_id=None): return False +def watchdog_detector_status(port, detector_id, timeout=5.0): + """Read one detector's own status block from ``GET /x-medkit-watchdog``. + + The route's payload carries the reliability gate's state plus, under ``detectors``, a + block per detector that has something to say about itself. A detector-scoped condition + (the lifecycle_expectation tracked-node cap being saturated, say) is otherwise visible + only in the gateway's log, which no assertion at this tier can read. + + Parameters + ---------- + port : int + Gateway HTTP port. + detector_id : str + The detector's own ``id()``, e.g. ``'lifecycle_expectation'``. + timeout : float + Per-request timeout in seconds. + + Returns + ------- + dict or None + The detector's status block, or ``None`` when the route did not answer 200 or + carries no block for that detector. + + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + try: + response = requests.get(f'{base}/x-medkit-watchdog', timeout=timeout) + except requests.exceptions.RequestException: + return None + if response.status_code != 200: + return None + status = response.json().get('x-medkit-watchdog', {}) + return (status.get('detectors') or {}).get(detector_id) + + +def poll_detector_status(port, detector_id, field, expected, timeout=30.0, interval=0.5): + """Poll ``GET /x-medkit-watchdog`` until a detector status field equals `expected`. + + Returns ``True`` once it does, ``False`` on timeout (after printing what was last seen, + which is gone once the launch tears down). + """ + deadline = time.monotonic() + timeout + last_seen = 'no detectors block was ever returned' + while time.monotonic() < deadline: + block = watchdog_detector_status(port, detector_id) + if block is not None: + last_seen = json.dumps(block) + if block.get(field) == expected: + return True + time.sleep(interval) + print(f'poll_detector_status({detector_id!r}, {field!r}=={expected!r}) timed out after ' + f'{timeout}s; last seen: {last_seen}') + return False + + +def wait_until_faults_endpoint_live(port, timeout=30.0, interval=0.5): + """Poll ``GET /faults`` until it answers HTTP 200. ``True`` once it does. + + The second half of the gate every absence assertion needs. + :func:`wait_until_watchdog_armed` proves the plugin side is alive - the .so loaded, + the tick loop ran - but it is served by the plugin INSIDE the gateway process and + says nothing about the fault surface. The gateway answers ``GET /faults`` with 503 + while the fault_manager's service is unavailable, and :func:`poll_faults` inspects + only 200 responses and swallows every transport error, returning ``None`` on timeout - + which is exactly what an ``assertIsNone`` wants to see. So a launch whose + fault_manager crashed, hung, or never DDS-matched turns a silence assertion green for + the one reason it must never be green. + + A 200 (whatever the body) proves the gateway reached the fault_manager in THIS launch. + What it does NOT prove is that the PLUGIN's own ReportFault client matched - only a + raise proves that, and that proof lives in the scenario that raises. + + Parameters + ---------- + port : int + Gateway HTTP port. + timeout : float + Maximum time to wait in seconds. + interval : float + Sleep between retries in seconds. + + Returns + ------- + bool + ``True`` once ``GET /faults`` answers 200, ``False`` on timeout. + + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + last_seen = 'GET /faults was never answered at all' + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/faults', timeout=5) + if response.status_code == 200: + return True + # 503 is the specific shape of "the fault_manager is not reachable" - worth + # naming, because it is the failure this gate exists for. + last_seen = f'HTTP {response.status_code} from GET /faults' + except requests.exceptions.RequestException as exc: + last_seen = f'GET /faults failed: {exc}' + time.sleep(interval) + print(f'wait_until_faults_endpoint_live timed out after {timeout}s; last seen: {last_seen}') + return False + + def poll_faults(port, code, timeout=30.0, interval=0.5): """Poll the GLOBAL ``GET /faults`` endpoint until `code` appears. @@ -398,6 +525,182 @@ def poll_fault_describing(port, code, needles, timeout=60.0, interval=0.5): return False, last_description +def assert_fault_absent_throughout(test_case, port, code, duration, interval=0.5): + """Poll ``GET /faults`` every `interval` across the WHOLE `duration`. + + Fails the moment either the channel or the claim breaks. + ``poll_faults`` is built for "wait until X appears" and is unsuitable for a silence + proof: it swallows every non-200 response and every transport error and returns + ``None`` on timeout - exactly what ``assertIsNone`` wants to see, so a ``/faults`` + that died three seconds into a twenty-second silence window still passes. This + instead asks the whole way through: every single poll must answer 200 ("asked, and + there is no such fault") or the assertion fails naming which poll and why ("could not + ask"), and `code` must never appear in any of them. Use this (not + ``assertIsNone(poll_faults(...))``) for every scenario whose claim is sustained + silence over a window, not merely "absent right now". + + Parameters + ---------- + test_case : unittest.TestCase + Used for the actual assertion calls, so a failure here reports through the normal + unittest failure path rather than a bare ``AssertionError`` from a free function. + port : int + Gateway HTTP port. + code : str + The ``fault_code`` that must never appear. + duration : float + Total seconds to keep polling. + interval : float + Sleep between polls in seconds. + + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + duration + polls = 0 + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/faults', timeout=5) + except requests.exceptions.RequestException as exc: + test_case.fail( + f'/faults became unreachable {polls} poll(s) into a {duration}s silence ' + f'window (could not ask, which is not the same as "asked, and there is no ' + f'such fault"): {exc}') + return + if response.status_code != 200: + test_case.fail( + f'/faults answered HTTP {response.status_code} {polls} poll(s) into a ' + f'{duration}s silence window - the channel died mid-window, which a ' + 'silence assertion must not read as "no such fault"') + return + codes = {item.get('fault_code') for item in response.json().get('items', [])} + test_case.assertNotIn( + code, codes, + f'{code} appeared {polls} poll(s) into a {duration}s window that was supposed ' + 'to stay silent') + polls += 1 + time.sleep(interval) + test_case.assertGreater( + polls, 0, + f'the {duration}s silence window never actually polled /faults - duration must be ' + f'>= interval ({interval}s)') + + +class _FlakyFaultsHandler(http.server.BaseHTTPRequestHandler): + """Stands in for a gateway whose ``GET /faults`` answers normally, then dies. + + Answers 200 with an empty fault list for the first ``healthy_polls`` requests to + ``{API_BASE_PATH}/faults`` (this class's own attribute, set per instantiation via + ``make_handler``), then drops the connection with no response at all for every + request after that - the same "channel gone" shape a crashed or hung fault_manager + produces, distinct from an ordinary 503 (also exercised via ``dead_status``). + """ + + healthy_polls = 2 + dead_status = None # None = drop the connection; an int = answer with that status instead + + def do_GET(self): + if self.path != f'{API_BASE_PATH}/faults': + self.send_response(404) + self.end_headers() + return + type(self).seen = getattr(type(self), 'seen', 0) + 1 + if type(self).seen > type(self).healthy_polls: + if type(self).dead_status is None: + self.close_connection = True # drop it: no response at all + return + self.send_response(type(self).dead_status) + self.end_headers() + return + body = json.dumps({'items': []}).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, log_format, *args): + pass # keep test output quiet - this is expected traffic, not diagnostics + + +class _FlakyFaultsServer: + """Context manager for a local `_FlakyFaultsHandler` HTTP server. + + Starts on a free port in a daemon thread and tears itself down on exit - the + boilerplate every leg of `prove_silence_proof_catches_a_dead_fault_surface` below + needs, factored out once so each leg reads as the claim it is checking rather than + server plumbing. + """ + + def __init__(self, healthy_polls, dead_status): + handler = type( + '_Handler', (_FlakyFaultsHandler,), + {'healthy_polls': healthy_polls, 'dead_status': dead_status}) + self._server = http.server.HTTPServer(('127.0.0.1', 0), handler) + self.port = self._server.server_address[1] + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *exc_info): + self._server.shutdown() + self._server.server_close() + self._thread.join(timeout=5) + + +def prove_silence_proof_catches_a_dead_fault_surface(test_case): + """Prove `assert_fault_absent_throughout` catches a `/faults` that dies mid-window. + + The test of the test - the exact hole `assertIsNone(poll_faults(...))` could not + see (see `assert_fault_absent_throughout`'s own docstring). Self-contained: a real + local HTTP server stands in for the gateway, answering normally for the first + couple of polls and then going dark, so this needs no ROS graph, no real gateway, + no fault_manager - only a real socket, so the failure mode under test (a channel + that goes silent) is real rather than mocked away. + + Three things are proven per dead-channel shape, all required to make the RED + result mean something: (1) the OLD pattern this fix replaces - `poll_faults` + + `assertIsNone` - actually DOES pass against this exact dead server (the concrete + "previously passed" the brief asks this to demonstrate, not merely asserted in + prose); (2) `assert_fault_absent_throughout` raises against the SAME server + (`dead_status=None`, a dropped connection, and again `dead_status=503`, the + specific shape a real `/faults` gives while the fault_manager is unreachable - a + helper that only caught one dead-channel shape would leave the other exactly as + blind as `poll_faults` always was); (3), after the loop below, that the fixed + helper does NOT raise while the channel never dies - a helper wired to always fail + would make the RED result meaningless. + + Raises whatever `test_case`'s own assertions raise on failure; callers use it from + inside a test method exactly like any other assertion helper. + """ + for dead_status, label in ((None, 'a dropped connection'), (503, 'a 503 response')): + with _FlakyFaultsServer(healthy_polls=2, dead_status=dead_status) as fake: + # (1) The OLD pattern this fix replaces: prove it actually passes on this + # exact dead channel - the concrete "previously passed" this test exists + # to show, not merely asserted in prose. + test_case.assertIsNone( + poll_faults(fake.port, 'GRAPH_NODE_INACTIVE', timeout=2.0, interval=0.2), + f'the OLD pattern (poll_faults + assertIsNone) did NOT pass against a ' + f'/faults that died via {label} - this self-test no longer ' + 'demonstrates the hole the fix closes') + # (2) The fixed helper must fail against the identical dead channel. + with test_case.assertRaises( + AssertionError, + msg=f'assert_fault_absent_throughout did not fail when /faults ' + f'died mid-window via {label} - it would pass on the exact ' + f'channel-death this fix exists to catch'): + assert_fault_absent_throughout( + test_case, fake.port, 'GRAPH_NODE_INACTIVE', duration=2.0, interval=0.2) + + # (3) The companion proof: a channel that never dies must not trip the assertion, + # or the RED result above would be meaningless (a helper wired to always fail + # "catches" everything by never being usable). + with _FlakyFaultsServer(healthy_polls=10_000, dead_status=None) as fake: + assert_fault_absent_throughout( + test_case, fake.port, 'GRAPH_NODE_INACTIVE', duration=1.0, interval=0.2) + + def poll_cleared(port, code, timeout=30.0, interval=0.5): """Poll the GLOBAL ``GET /faults`` endpoint until `code` is absent. diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_config_plumbing_e2e.test.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_config_plumbing_e2e.test.py index bb3cb116b..fb32ee0a1 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_config_plumbing_e2e.test.py +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_config_plumbing_e2e.test.py @@ -66,8 +66,10 @@ # I100 as well as E402: `harness` is only importable because of the sys.path line above, so this # import cannot be moved up to where the alphabetical order would put it. from harness import ( # noqa: E402, I100 + assert_fault_absent_throughout, create_watchdog_test_launch, poll_faults, + wait_until_faults_endpoint_live, wait_until_watchdog_armed, ) @@ -146,13 +148,22 @@ def test_mode_off_suppresses_detector(self): ) # Proving absence means waiting out the full window: a detector that # (regression) ignored `mode` would have raised well within it. - fault = poll_faults(PORT, 'GRAPH_PARAM_DRIFT', timeout=20.0) - self.assertIsNone( - fault, - 'GRAPH_PARAM_DRIFT raised despite ' - 'detectors.param_drift.mode="off" - the nested `mode` config was ' - 'not delivered to (or not honored by) the plugin', + # assert_fault_absent_throughout, not assertIsNone(poll_faults(...)): the latter + # swallows every non-200 response and transport error into the same None a genuine + # absence produces, so a /faults that died partway through this window would still + # pass - assert_fault_absent_throughout instead fails naming which poll could not + # even ask. + # First contact with GET /faults happens HERE, and the window below is strict: one + # transport error fails it. On the distro running default FastDDS that first call also + # pays service discovery to the fault_manager, which can outlast the window's own 5 s + # per-poll timeout - so prove the channel is up first, with a budget that tolerates + # discovery, and let the window police only what it is for. + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=60.0), + 'GET /faults never answered 200 - the fault surface is not up, so a silence ' + 'assertion below could not tell "no such fault" from "could not ask"', ) + assert_fault_absent_throughout(self, PORT, 'GRAPH_PARAM_DRIFT', 20.0) class TestConfigPlumbingModeOffYamlBool(unittest.TestCase): @@ -170,13 +181,17 @@ def test_yaml_boolean_off_suppresses_detector(self): 'graph_watchdog never reported an armed gate - the plugin did not load ' 'or its tick loop never ran, so an absent fault proves nothing', ) - fault = poll_faults(PORT, 'GRAPH_PARAM_DRIFT', timeout=20.0) - self.assertIsNone( - fault, - 'GRAPH_PARAM_DRIFT raised despite a bare ' - 'detectors.param_drift.mode: off - the YAML boolean form of "off" ' - 'was not honored by the plugin', + # assert_fault_absent_throughout, not assertIsNone(poll_faults(...)) - see + # TestConfigPlumbingModeOff's identical rationale above. + # Same first-contact gate as TestConfigPlumbingModeOff above: this class is its own + # CTest target in its own process, so it makes first contact with GET /faults itself + # and needs its own proof that the channel is up before the strict window. + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=60.0), + 'GET /faults never answered 200 - the fault surface is not up, so the silence ' + 'assertion below could not tell "no such fault" from "could not ask"', ) + assert_fault_absent_throughout(self, PORT, 'GRAPH_PARAM_DRIFT', 20.0) # Each CTest target launches this file with one scenario, so only that scenario's case may diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_lifecycle_expectation_e2e.test.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_lifecycle_expectation_e2e.test.py new file mode 100644 index 000000000..0684f9de6 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_lifecycle_expectation_e2e.test.py @@ -0,0 +1,2627 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Lifecycle-expectation e2e: GRAPH_NODE_INACTIVE raises and heals through the REAL stack. + +Unlike test_lifecycle_expectation_integration.cpp (which drives the detector against a +fake ReportFault service, a fixture-owned snapshot, and lifecycle labels injected via +ReliabilityGate::set_lifecycle_state_for_test() - it can only puppet the label, never +prove the real lifecycle machinery feeds it), this launches the REAL gateway process +with the graph_watchdog plugin (.so) loaded and a real fault_manager, and polls the +operator-visible ``GET /api/v1/faults`` surface while driving the demo +``managed_lifecycle`` node (``ros2_medkit_test_utils``' ``DEMO_NODE_REGISTRY``) through +REAL ``lifecycle_msgs/srv/ChangeState`` transitions. + +Runs as TWELVE separate CTest targets (see CMakeLists.txt), each launching its OWN +gateway+fault_manager+demo-node stack: the plugin reads its config once at +set_context() time, so different configs need different gateway launches, not test +methods sharing one. WATCHDOG_E2E_SCENARIO selects which launch + assertions run: + +- "main": ``require_active: ['managed_lifecycle']`` against the node in its default + unconfigured state -> the fault raises (globally AND on the entity-scoped surface, + naming the node), survives a real CONFIGURE (inactive is still not active), survives a + real gateway RESTART (the withheld-clear guard: the fault_manager is a separate process + and keeps the fault, while the restarted plugin starts with every detector counter at + zero), and heals after a real ACTIVATE. This is the only scenario launched with + ``gateway_respawn``. +- "default_config": NO ``detectors.lifecycle_expectation`` config at all, same demo + node. The shipped default is an empty ``require_active``, documented as "nothing is + checked, zero false positives" - this is the only test at any tier that could falsify + that claim against a live unconfigured lifecycle node. +- "negative_control": ``require_active: ['managed_lifecycle_active']`` with the + self-activating variant of the same executable. A required node that IS active must + never raise, over a sustained window, under the same grace and cadence as "main" - + the entry names the active variant, so the discriminating variable between this + launch and "main" is the node's actual lifecycle state. +- "unreadable": ``require_active: ['unreadable_lifecycle']`` against + ``unreadable_lifecycle_node.cpp`` - a fixture that advertises get_state/change_state + like a managed lifecycle node but never answers get_state until told to. Proves + GRAPH_NODE_UNREADABLE's own raise and clear through a REAL, sustained GetState + failure (the class the other scenarios above cannot reach - see + TestLifecycleExpectationUnreadable's docstring below for the full story) and that + GRAPH_NODE_INACTIVE stays silent for a node whose state was never read at all. +- "departure_keeps": the same ``unreadable_lifecycle_node.cpp`` fixture, but this + scenario SIGTERMs it once GRAPH_NODE_UNREADABLE has raised and proves the fault + SURVIVES the departure - a node whose lifecycle promise was never verified does not + become verified by leaving, so its evidence is retained and its description switches to + saying the node is gone. See TestLifecycleExpectationDepartureKeeps's docstring below. +- "not_managed": ``require_active: ['calibration']`` against a PLAIN demo node with no + lifecycle interface at all (``DEMO_NODE_REGISTRY``'s ``calibration``, a bare service + server) - proves GRAPH_NODE_NOT_MANAGED's own raise (naming the node, WARN severity, + mutually exclusive with the other two codes) and the same retention across a departure, + the sibling proof to "unreadable"/"departure_keeps" for the OTHER cause the unmeasured + clock is blind to. Needed no purpose-built fixture, unlike UNREADABLE. See + TestLifecycleExpectationNotManaged's docstring below for the full story. +- "restart_loop": the same ``calibration`` node, but respawning, SIGTERM'd over and over + on a cadence that never lets it accumulate the 60 consecutive PRESENT ticks the hold + needs. Proves the point of the whole model on a real stack: a required node in a crash + loop is REPORTED rather than silent, because the evidence it accumulates while present + is not discarded every time it goes away. See TestLifecycleExpectationRestartLoop. +- "cap_pressure": ``tracked_node_cap: 1`` against TWO required nodes, both instances of + ``droppable_lifecycle_node.cpp`` (this package's own fixture - looks managed, answers a + chosen label, stops advertising its lifecycle services on command). Proves that a refused + node is visible on ``GET /x-medkit-watchdog``, that a refusal WITHHOLDS + GRAPH_NODE_INACTIVE's clear, and that an entry for a DEPARTED node is collapsed so a + present, genuinely broken node is checked instead. See + TestLifecycleExpectationCapPressure. +- "unsettled_departure": two more ``droppable_lifecycle_node.cpp`` instances, both healthy + and ACTIVE. One drops its lifecycle services and is killed immediately (a single missed + sweep before a clean shutdown - nothing may be reported); the other holds the dropped + state long past the settling budget before being killed (genuinely not managed when it + left - it must still be reported). See TestLifecycleExpectationUnsettledDeparture. +- "wide_grace": ``grace`` at the value that used to be the accepted maximum + (``INT_MAX - 1``), against the stuck ``managed_lifecycle`` node. Under such a value the + detector can neither raise nor heal GRAPH_NODE_INACTIVE for days; the value must be + refused and the documented default applied. See TestLifecycleExpectationWideGrace. +- "restart_departed": the stuck ``managed_lifecycle`` node is killed while its fault is + outstanding, then the gateway itself is restarted. RECORDS the boundary of "a departure + never heals a fault" - it holds within a gateway lifetime, and a restart re-baselines. + See TestLifecycleExpectationRestartDeparted. + +The silence scenarios gate on three facts BEFORE +asserting absence, because absence is +the default outcome of a stack that never came up and every bringup failure mode on this +launch still exits 0 (see the harness docstring): (1) the plugin is live and armed, via +its own GET /x-medkit-watchdog route (harness.wait_until_watchdog_armed); (2) the +gateway reaches the fault_manager in THIS launch, via GET /faults answering 200 +(harness.wait_until_faults_endpoint_live) - the watchdog route is served inside the +gateway process and proves nothing about the fault surface the assertion reads, which +answers 503 and polls to None when the fault_manager is gone; (3) the target node's +lifecycle label was actually READ, via the same watchdog route - the tracker treats an +unread label as benign, so without it the silence could just as well mean the label +never arrived. Fact (3) is re-checked AFTER the window as well, since a trigger that +exits mid-window would leave most of the window measuring an empty graph. + +The "main" scenario gates on the GLOBAL armed state, not on +``app_id='managed_lifecycle'``: the gate reports a tracked node with a KNOWN non-active +label as ``warming_up`` by design (ReliabilityGate::status_json's suppressed branch ANDs +LifecycleWatcher::node_ok, which is false for exactly the inactive node this detector +exists to catch), so a per-entity armed gate on the target would wait for the fault to +be impossible. The raise itself is gated on the SOURCE entity's arming (the aggregate is +raised under source 'graph_watchdog', see AggregatedFault), for which the global armed +state is the precise precondition. Once the node reaches "active" the per-entity gate +IS reachable, and the heal leg uses it to prove the real lifecycle machinery fed the +watcher before measuring the clear. +""" + +import json +import os +import signal +import sys +import threading +import time +import unittest + +from launch.actions import TimerAction +import launch_ros.actions +import launch_testing +from lifecycle_msgs.msg import Transition +from lifecycle_msgs.srv import ChangeState +from rcl_interfaces.msg import Parameter, ParameterType, ParameterValue +from rcl_interfaces.srv import SetParameters +import rclpy +from rclpy.node import Node +import requests + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +# I100 as well as E402: `harness` is only importable because of the sys.path line above, +# so this import cannot be moved up to where the alphabetical order would put it. +from harness import ( # noqa: E402, I100 + API_BASE_PATH, + assert_fault_absent_throughout, + create_watchdog_test_launch, + poll_cleared, + poll_detector_status, + poll_entity_faults, + poll_faults, + prove_silence_proof_catches_a_dead_fault_surface, + wait_until_faults_endpoint_live, + wait_until_watchdog_armed, + watchdog_detector_status, +) + +from ros2_medkit_test_utils.constants import ALLOWED_EXIT_CODES, get_test_port # noqa: E402 +from ros2_medkit_test_utils.coverage import get_coverage_env # noqa: E402 +from ros2_medkit_test_utils.launch_helpers import DEMO_NODE_REGISTRY # noqa: E402 + +# No default on purpose - a default makes this file FAIL OPEN (see +# test_config_plumbing_e2e.test.py's identical rationale): if the CTest ENVIRONMENT +# property never reaches the process, the launch and the assertions would degrade +# together into one scenario and still report 1/1 passed. A KeyError is loud. +SCENARIO = os.environ['WATCHDOG_E2E_SCENARIO'] +PORT = get_test_port() + +# Fast tick cadence + short warmup so the whole story (gateway/fault_manager startup, +# demo-node discovery, lifecycle label seeding, global bringup grace) comfortably fits +# inside the poll timeouts below - same rationale as the other e2e files here. grace is +# a few ticks so the raise is not instantaneous (the detector counts CONSECUTIVE +# not-active ticks) while staying well inside every poll timeout below. +TICK_INTERVAL_MS = 200 +WARMUP_CYCLES = 3 +GRACE = 3 + +FAULT_CODE = 'GRAPH_NODE_INACTIVE' +# Node names from DEMO_NODE_REGISTRY, which are also the App ids the gateway derives +# for nodes in the root namespace - so each constant is at once the launch key, the +# gate entity id, and what the fault description must name. +TARGET_NODE = 'managed_lifecycle' +ACTIVE_NODE = 'managed_lifecycle_active' +CHANGE_STATE_SERVICE = f'/{TARGET_NODE}/change_state' + +_DETECTOR_PREFIX = 'plugins.graph_watchdog.detectors.lifecycle_expectation' + +# How long a fault must stay ABSENT for the two silence scenarios. Counted from the +# arming gate, not process start, so bringup cannot eat it. At the 200 ms tick this is +# ~100 ticks against a grace of 3 (default 5): a detector that wrongly counted the +# node would have raised - and been CONFIRMED by the harness's +# confirmation_threshold: -2 fault_manager - dozens of times over. +SILENT_WINDOW_SEC = 20.0 + +# The "healing_threshold" scenario needs real WALL-CLOCK margin, not just tick count: it +# kills and respawns a real OS process (the target node) and needs the gateway's entity +# cache to actually notice the departure and the return, all comfortably inside the +# tracker's own absence_grace (a fixed 3 ticks - kDefaultAbsenceGrace in +# lifecycle_expectation_tracker.hpp, not configurable). A much slower cadence than the +# other scenarios buys that margin: at 1 s/tick (the shipped production default) the +# absence budget is a full 3 s, comfortably longer than a SIGTERM + respawn_delay + +# rediscovery round trip ever needs. +HEALING_TICK_INTERVAL_MS = 1000 +# kDefaultAbsenceGrace, lifecycle_expectation_tracker.hpp - fixed, not configurable. +# Mirrored here (not imported - this is a separate Python process) so _blink() can +# compute the exact wall-clock budget a blink is supposed to stay inside. +HEALING_ABSENCE_GRACE_TICKS = 3 +# Sped up from the gateway's own 1000 ms default so the entity cache reflects a kill or a +# respawn quickly relative to the absence budget above, not eat most of it. +HEALING_REFRESH_DEBOUNCE_MS = 300 +# launch will not even ATTEMPT to restart the process before this elapses, so it is a +# real, enforced floor under each blink's duration - not just a hint. It has to exceed ONE +# tick interval: a node that leaves and returns inside a single tick is never sampled +# absent at all, so the absence grace this scenario is named for is not exercised and the +# blink proves nothing. The ceiling is the other side of the same window - the whole +# absence must stay under HEALING_ABSENCE_GRACE_TICKS ticks, or what is being exercised is +# the ordinary sustained-absence path instead. Both edges are asserted per blink rather +# than assumed from this constant. +HEALING_RESPAWN_DELAY_SEC = 1.4 +# Slept after the SIGTERM before polling for the node's return: gives the respawn_delay +# above, the new process's own startup, and DDS rediscovery room to finish before the +# poll below starts, without itself risking the 3 s absence budget. +HEALING_BLINK_SLEEP_SEC = 1.5 +# Two separate blink episodes, matching the scenario this detector's design doc walks +# through: a node the detector has already reported vanishing and returning twice must +# never be healed by either one. +HEALING_BLINK_COUNT = 2 + +# The "unreadable" scenario needs to actually cross kUnmeasuredHoldTicks (60, +# lifecycle_expectation_detector.cpp - fixed, not configurable) consecutive +# matched-and-unread ticks before GRAPH_NODE_UNREADABLE raises, unlike every other +# scenario here which only needs a handful of `grace` ticks. A much faster cadence +# than TICK_INTERVAL_MS above keeps that affordable: 60 ticks pass in single-digit +# seconds rather than the ~4 minutes the shipped 1 s default would cost. +UNREADABLE_TICK_INTERVAL_MS = 100 +UNREADABLE_NODE = 'unreadable_lifecycle' +FAULT_CODE_UNREADABLE = 'GRAPH_NODE_UNREADABLE' +# Same 100-tick shape as SILENT_WINDOW_SEC above (~100 ticks there too, at that +# scenario's 200 ms cadence), scaled to this scenario's faster tick: 100 * +# UNREADABLE_TICK_INTERVAL_MS. Proving GRAPH_NODE_INACTIVE's absence does not +# actually need this many ticks - only a MEASURED not-active read ever advances the +# violation streak (lifecycle_expectation_tracker.hpp), so an unread label raising here +# would be a real bug, not a late one - but a window this wide leaves room to let the hold below +# start accumulating in the background while this one runs. +UNREADABLE_INACTIVE_SILENCE_SEC = 10.0 +# The raise poll's budget. Real expected cost: (kUnmeasuredHoldTicks + 1) consecutive +# matched ticks - the hold is "frozen one past itself", so the raise lands on the SAME +# tick the counter first exceeds 60, not one tick later - at UNREADABLE_TICK_INTERVAL_MS +# is 61 * 0.1 = 6.1 s of pure tick cadence. On top of that, THREE of those ticks (the +# initial LifecycleWatcher seed plus its two re-seed attempts - kReseedAttempts in +# lifecycle_watcher.cpp, spent once and never replenished) each run a BLOCKING GetState +# call that only returns once the reader's own timeout expires (500 ms, +# Ros2LifecycleStateReader's default) because this fixture never answers - and that +# block happens INSIDE tick() itself, so it adds to the gap before the NEXT tick rather +# than eating into UNREADABLE_TICK_INTERVAL_MS. Budget: 6.1 + 3 * 0.5 = 7.6 s of +# tick-driven work, plus the same generous bringup margin (gateway/fault_manager start, +# demo_delay, DDS discovery, entity-cache debounce) the sibling scenarios' 60 s "raise +# poll" budgets already carry for a MUCH smaller expected cost - kept here rather than +# tightened, since CI slowness is the variable this margin exists for, not the tick math. +UNREADABLE_RAISE_TIMEOUT_SEC = 60.0 + +# The "departure_keeps" scenario's own budgets, for what a DEPARTURE does to an +# already-reported unmeasured fault: nothing. A SIGTERM's DDS "participant left" propagates +# in well under a second and HEALING_REFRESH_DEBOUNCE_MS (reused below for the exact reason +# healing_threshold defines it: giving the entity cache room to reflect a kill quickly) +# bounds the entity-cache catch-up. The generous margin below is the same CI-slowness +# allowance every other poll budget in this file carries, not a measurement of that cost. +ABSENCE_DEPARTURE_TIMEOUT_SEC = 30.0 +ABSENCE_CLEAR_TIMEOUT_SEC = 30.0 +# Slept after a departure has been CONFIRMED on GET /apps, before asserting the fault +# survived it. Long enough to cover every horizon that could have discarded the node's +# evidence at this scenario's 100 ms cadence: the tracker's absence grace (3 ticks), +# its unmeasured hold (60 ticks) and a level-triggered clear reaching the fault_manager's +# healing threshold - so "the fault is still here" is a measurement, not a race won. +DEPARTURE_SETTLE_SEC = 10.0 +# A `prune_grace` at the widest accepted value (kMaxPruneGrace, 3600 ticks - 360 s at this +# cadence), which also exercises that key's upper endpoint through the real config-delivery +# path. It can no longer change any outcome in these scenarios: the age horizon reaches +# IDLE bookkeeping only, and a node carrying a matured unmeasured clock is never idle. +ABSENCE_PRUNE_GRACE_TICKS = 3600 +# How long a mutually-exclusive sibling code must stay absent. Short (a few ticks at every +# cadence in this file) because the claim is "not at the same time as", not "never" - but a +# WINDOW with the channel checked on every poll, never a single poll that timed out +# silently, which cannot tell "no such fault" from "could not ask". +MUTUAL_EXCLUSION_WINDOW_SEC = 3.0 + +# The "restart_loop" scenario: the whole point of holding evidence across absence. A node +# that crash-loops - up for a while, gone for a while, forever - touches absence +# periodically BY CONSTRUCTION, so any horizon that discards its evidence on absence makes +# exactly that node permanently invisible. Reuses `calibration` (a plain node with no +# lifecycle interface, so the NOT-MANAGED cause) with launch's own respawn, and SIGTERMs it +# on a cadence that never lets it accumulate the 60 consecutive PRESENT ticks the hold would +# otherwise need. +RESTART_LOOP_TICK_INTERVAL_MS = 100 +# Seconds the node is left alive between kills. At the 100 ms tick that is at most ~25 +# ticks present per cycle - comfortably under kUnmeasuredHoldTicks (60), so a run in which +# the clock restarted on every absence could never mature however many cycles it ran. +RESTART_LOOP_UPTIME_SEC = 2.0 +# launch will not even ATTEMPT to restart the process before this elapses, so it is a real, +# enforced floor under each absence: 1.0 s is 10 ticks, well past the tracker's 3-tick +# absence grace, i.e. every cycle genuinely crosses the horizon under test. +RESTART_LOOP_RESPAWN_DELAY_SEC = 1.0 +# Wall-clock budget for the kill loop. The clock advances on every tick once the node has +# been seen, so ~61 ticks (6.1 s) plus the held blink ticks is the real expected cost; this +# is several times that, and a detector that discarded evidence on absence stays silent for +# all of it however many cycles fit. +RESTART_LOOP_WINDOW_SEC = 45.0 +# Kills the loop must have completed before a sighting of the fault is accepted. Three, so +# the run is unambiguously a LOOP rather than one departure, and so the fault is proven to +# survive the restarts that follow its first appearance. At ~3.5 s per cycle that is ~11 s, +# comfortably inside the window above. +RESTART_LOOP_MIN_CYCLES = 3 + +# The "not_managed" scenario's own fixture: unlike "unreadable" (which needs a purpose-built +# node that advertises get_state/change_state but never answers), NOT_MANAGED only needs a +# node with NO lifecycle interface at all - and a plain one already exists in this package's +# demo fixtures (DEMO_NODE_REGISTRY's 'calibration', a plain rclcpp::Node service server with +# no lifecycle_msgs services whatsoever), so nothing new had to be built for this. Reuses the +# same fast cadence as "unreadable"/"departure_keeps": kUnmeasuredHoldTicks (60, fixed) is far +# more ticks than the shipped 1 s default affords inside any reasonable poll budget. +NOT_MANAGED_TICK_INTERVAL_MS = 100 +NOT_MANAGED_NODE = 'calibration' +FAULT_CODE_NOT_MANAGED = 'GRAPH_NODE_NOT_MANAGED' +# Same shape and derivation as UNREADABLE_RAISE_TIMEOUT_SEC above - the expected cost is the +# same (kUnmeasuredHoldTicks + 1) consecutive matched ticks at this scenario's own cadence, +# and NOT_MANAGED needs no blocking GetState round trips at all (there is no lifecycle +# service to call), so if anything this scenario's real cost is lower, not higher - the same +# generous CI-slowness margin is kept rather than tightened. +NOT_MANAGED_RAISE_TIMEOUT_SEC = 60.0 +# Same margins as ABSENCE_DEPARTURE_TIMEOUT_SEC / ABSENCE_CLEAR_TIMEOUT_SEC above, for the +# same OTHER way an unmeasured code clears: an already-reported node leaving the graph and +# staying away past the absence grace, never by being read (there is nothing to read here). +NOT_MANAGED_DEPARTURE_TIMEOUT_SEC = 30.0 +NOT_MANAGED_CLEAR_TIMEOUT_SEC = 30.0 + +# The detector id its own status block is filed under on GET /x-medkit-watchdog. +DETECTOR_ID = 'lifecycle_expectation' + +# ---- the "cap_pressure" scenario: what a FULL tracked-node cap does ------------------- +# +# Two required nodes and a cap of ONE, so exactly one of them can be tracked and the other +# is refused every tick. Reachable only because `tracked_node_cap` is a config key; with a +# compile-time 512 this whole scenario would need 513 real lifecycle nodes. +CAP_TICK_INTERVAL_MS = 100 +# The lexicographically FIRST of the two required nodes, so it is the one that wins the +# single slot (the tracker keys its per-tick map by fqn, a std::map, so the sweep is +# lexicographic). It answers "unconfigured", i.e. a measured violation. +CAP_TRACKED_NODE = 'cap_droppable' +# The one that is REFUSED: present, required, matched, and never checked. +CAP_REFUSED_NODE = 'cap_stuck' +# One node, one slot. +CAP_NODE_CAP = 1 +# When the REFUSED node is started, against the tracked one's own 2.0 s. The slot goes to +# whichever required node is matched first, so the two must not race: eight seconds is 80 +# ticks at this scenario's cadence, far longer than discovery plus the grace the tracked node +# needs to be carrying evidence by the time the second node exists at all. +CAP_REFUSED_NODE_DELAY_SEC = 10.0 +# Long enough for the unmeasured hold (60 ticks, fixed) to run out after the tracked node's +# lifecycle services are dropped, plus the usual CI-slowness margin every raise poll here +# carries: 61 * 0.1 = 6.1 s of tick cadence. +CAP_NOT_MANAGED_RAISE_TIMEOUT_SEC = 60.0 +# How long GRAPH_NODE_INACTIVE's clear must stay withheld while a required node is refused. +# ~100 ticks at this cadence, and the fault_manager runs at healing_threshold 1 here, so a +# single spurious PASSED anywhere in the window is enough to fail the assertion. +CAP_WITHHOLD_WINDOW_SEC = 10.0 +# Budget for the refused node to be admitted and confirmed once the departed entry is +# collapsed: absence grace (3 ticks) + grace (3 ticks) + the report reaching the store. +CAP_ADMISSION_TIMEOUT_SEC = 60.0 + +# ---- the "unsettled_departure" scenario: what ONE bad sweep may decide ---------------- +# +# A deliberately SLOW cadence. The claim is about how many consecutive ticks an unmeasured +# observation is corroborated for before a departure is allowed to continue it, so the test +# has to place its actions between ticks rather than race them: at 500 ms a single deliberate +# hold is one tick, and the entity cache catching up with a SIGTERM costs well under the +# settling budget rather than most of it. +UNSETTLED_TICK_INTERVAL_MS = 500 +# kDefaultObservationSettleTicks (lifecycle_expectation_tracker.hpp) - consecutive matched +# ticks an UNMEASURED observation must hold before absence may continue it. Mirrored here +# (this is a separate Python process) so both legs below can be paced against the real bound. +UNSETTLED_SETTLE_TICKS = 6 +# The node whose lifecycle services vanish for a SHORT spell and which then exits cleanly. +UNSETTLED_BLINK_NODE = 'blink_departer' +# The node whose lifecycle services vanish and STAY gone long enough to be corroborated +# before it exits - the positive control that makes the blinker's silence mean something. +UNSETTLED_SETTLED_NODE = 'settled_departer' +# Ticks the settled leg holds its dropped state for: comfortably past the settle budget, so +# its departure genuinely continues a corroborated not-managed observation. +UNSETTLED_SETTLED_HOLD_TICKS = 10 +# The unmeasured hold (60 ticks, fixed) plus the absence grace, at this cadence: 63 * 0.5 = +# 31.5 s, plus the usual margin. Both legs are measured against this same budget - the +# settled one must raise inside it, the blinking one must never raise at all. +UNSETTLED_RAISE_TIMEOUT_SEC = 90.0 +# Sped up from the gateway's own 1000 ms default so the entity cache reflects a kill quickly +# relative to the settling budget the blink leg has to stay inside. +UNSETTLED_REFRESH_DEBOUNCE_MS = 300 + +# ---- the "wide_grace" scenario: a `grace` past the accepted maximum ------------------- +# +# The value that used to be the accepted maximum (INT_MAX - 1). A streak that has to reach +# it advances one per tick, so a node under it is never CONFIRMED and never CLEARED either: +# it sits in the tracker's pending set, the withheld-clear guard returns early every tick, +# and GRAPH_NODE_INACTIVE is silent for the life of the process - about 24 days at the +# shipped 1 s cadence, and about 2.5 days at this scenario's own. +WIDE_GRACE_TICK_INTERVAL_MS = 100 +WIDE_GRACE_VALUE = 2147483646 +# The raise poll's budget. With the value refused and the documented default (5) in force, +# the node is confirmed within a handful of ticks; the margin is the usual bringup allowance. +WIDE_GRACE_RAISE_TIMEOUT_SEC = 60.0 + +# ---- the "restart_departed" scenario: what a gateway RESTART does --------------------- +# +# Characterisation, not a fix: it records where "a departure never heals a fault" actually +# ends. Inside one gateway process it holds; across a restart the tracker starts empty, the +# departed node is not in the graph, and its require_active entry matches nothing - which the +# detector cannot tell apart from a misspelt entry, so the deliberately bounded never-matched +# hold expires and the clear flows. +RESTART_TICK_INTERVAL_MS = 100 +# kDefaultUnmeasuredHoldTicks (60, fixed): the never-matched hold's own bound, after which +# the clear is released. 60 * 0.1 = 6 s, plus healing at healing_threshold 1 and the usual +# CI margin. +RESTART_HEAL_TIMEOUT_SEC = 90.0 + + +def generate_test_description(): + detector_params = { + 'plugins.graph_watchdog.tick_interval_ms': TICK_INTERVAL_MS, + 'plugins.graph_watchdog.warmup_cycles': WARMUP_CYCLES, + } + demo_node = TARGET_NODE + extra_gateway_params = None + healing_threshold = 3 + if SCENARIO == 'main': + detector_params[f'{_DETECTOR_PREFIX}.require_active'] = [TARGET_NODE] + detector_params[f'{_DETECTOR_PREFIX}.grace'] = GRACE + elif SCENARIO == 'negative_control': + detector_params[f'{_DETECTOR_PREFIX}.require_active'] = [ACTIVE_NODE] + detector_params[f'{_DETECTOR_PREFIX}.grace'] = GRACE + demo_node = ACTIVE_NODE + elif SCENARIO == 'healing_threshold': + detector_params[f'{_DETECTOR_PREFIX}.require_active'] = [TARGET_NODE] + detector_params[f'{_DETECTOR_PREFIX}.grace'] = GRACE + detector_params['plugins.graph_watchdog.tick_interval_ms'] = HEALING_TICK_INTERVAL_MS + extra_gateway_params = {'discovery.refresh_debounce_ms': HEALING_REFRESH_DEBOUNCE_MS} + # A low healing_threshold is the whole point of this scenario: it makes even ONE + # spurious PASSED (from a blink the withheld-clear guard fails to hold) enough to + # move the debounce counter noticeably - the README's own recommended value. + healing_threshold = 1 + # Launched separately below, with a handle this scenario can SIGTERM and respawn + # under its own control - create_demo_nodes() gives no such handle back. + demo_node = None + elif SCENARIO == 'unreadable': + detector_params[f'{_DETECTOR_PREFIX}.require_active'] = [UNREADABLE_NODE] + detector_params[f'{_DETECTOR_PREFIX}.grace'] = GRACE + detector_params['plugins.graph_watchdog.tick_interval_ms'] = UNREADABLE_TICK_INTERVAL_MS + demo_node = UNREADABLE_NODE + elif SCENARIO == 'departure_keeps': + detector_params[f'{_DETECTOR_PREFIX}.require_active'] = [UNREADABLE_NODE] + detector_params[f'{_DETECTOR_PREFIX}.grace'] = GRACE + detector_params['plugins.graph_watchdog.tick_interval_ms'] = UNREADABLE_TICK_INTERVAL_MS + # The upper endpoint of prune_grace's documented range, through the real + # config-delivery path - see ABSENCE_PRUNE_GRACE_TICKS's own comment for why it can + # no longer change this scenario's outcome either way. + detector_params[f'{_DETECTOR_PREFIX}.prune_grace'] = ABSENCE_PRUNE_GRACE_TICKS + # Same speed-up as healing_threshold, and for the same reason: the entity cache + # needs to reflect this scenario's kill quickly relative to the (fixed, 3-tick) + # absence grace the assertions are timed against. + extra_gateway_params = {'discovery.refresh_debounce_ms': HEALING_REFRESH_DEBOUNCE_MS} + # Launched separately below, with a handle this scenario can SIGTERM - permanently, + # unlike healing_threshold's respawning blink - create_demo_nodes() gives no such + # handle back. + demo_node = None + elif SCENARIO == 'not_managed': + detector_params[f'{_DETECTOR_PREFIX}.require_active'] = [NOT_MANAGED_NODE] + detector_params[f'{_DETECTOR_PREFIX}.grace'] = GRACE + detector_params['plugins.graph_watchdog.tick_interval_ms'] = NOT_MANAGED_TICK_INTERVAL_MS + detector_params[f'{_DETECTOR_PREFIX}.prune_grace'] = ABSENCE_PRUNE_GRACE_TICKS + extra_gateway_params = {'discovery.refresh_debounce_ms': HEALING_REFRESH_DEBOUNCE_MS} + # Launched separately below, with a handle this scenario can SIGTERM - a plain demo + # node has no lifecycle to drive, so a permanent kill is the only interesting thing + # left to do to it once it has raised. + demo_node = None + elif SCENARIO == 'restart_loop': + detector_params[f'{_DETECTOR_PREFIX}.require_active'] = [NOT_MANAGED_NODE] + detector_params[f'{_DETECTOR_PREFIX}.grace'] = GRACE + detector_params['plugins.graph_watchdog.tick_interval_ms'] = RESTART_LOOP_TICK_INTERVAL_MS + extra_gateway_params = {'discovery.refresh_debounce_ms': HEALING_REFRESH_DEBOUNCE_MS} + # Launched separately below with respawn, so the test can SIGTERM it over and over + # and have launch bring it straight back - the crash loop this scenario is about. + demo_node = None + elif SCENARIO == 'cap_pressure': + detector_params[f'{_DETECTOR_PREFIX}.require_active'] = [ + CAP_TRACKED_NODE, CAP_REFUSED_NODE] + detector_params[f'{_DETECTOR_PREFIX}.grace'] = GRACE + detector_params[f'{_DETECTOR_PREFIX}.tracked_node_cap'] = CAP_NODE_CAP + detector_params['plugins.graph_watchdog.tick_interval_ms'] = CAP_TICK_INTERVAL_MS + extra_gateway_params = {'discovery.refresh_debounce_ms': HEALING_REFRESH_DEBOUNCE_MS} + # A sensitive healing_threshold, for the same reason healing_threshold uses one: + # this scenario's central claim is that NO clear is emitted while a required node is + # refused, and at 1 a single spurious PASSED walks the fault straight to HEALED. + healing_threshold = 1 + demo_node = None # both fixtures are launched below, with handles this test can drive + elif SCENARIO == 'unsettled_departure': + detector_params[f'{_DETECTOR_PREFIX}.require_active'] = [ + UNSETTLED_BLINK_NODE, UNSETTLED_SETTLED_NODE] + detector_params[f'{_DETECTOR_PREFIX}.grace'] = GRACE + detector_params['plugins.graph_watchdog.tick_interval_ms'] = UNSETTLED_TICK_INTERVAL_MS + extra_gateway_params = {'discovery.refresh_debounce_ms': UNSETTLED_REFRESH_DEBOUNCE_MS} + demo_node = None + elif SCENARIO == 'wide_grace': + detector_params[f'{_DETECTOR_PREFIX}.require_active'] = [TARGET_NODE] + detector_params[f'{_DETECTOR_PREFIX}.grace'] = WIDE_GRACE_VALUE + detector_params['plugins.graph_watchdog.tick_interval_ms'] = WIDE_GRACE_TICK_INTERVAL_MS + extra_gateway_params = {'discovery.refresh_debounce_ms': HEALING_REFRESH_DEBOUNCE_MS} + demo_node = None # launched below, so the test can SIGTERM it + elif SCENARIO == 'restart_departed': + detector_params[f'{_DETECTOR_PREFIX}.require_active'] = [TARGET_NODE] + detector_params[f'{_DETECTOR_PREFIX}.grace'] = GRACE + detector_params['plugins.graph_watchdog.tick_interval_ms'] = RESTART_TICK_INTERVAL_MS + extra_gateway_params = {'discovery.refresh_debounce_ms': HEALING_REFRESH_DEBOUNCE_MS} + # The heal this scenario RECORDS has to be observable, so the debounce counter must + # reach HEALED promptly once the clear starts flowing. + healing_threshold = 1 + demo_node = None # launched below, so the test can SIGTERM it before the gateway restart + # default_config: deliberately NO detectors.lifecycle_expectation keys at all - + # the launch under test is the shipped default itself. + + launch_description, context = create_watchdog_test_launch( + detector_params=detector_params, + extra_gateway_params=extra_gateway_params, + # 'managed_lifecycle' stays unconfigured (auto_activate defaults to false): + # present in the graph, alive, but not active - exactly the silent state this + # detector exists to catch. 'managed_lifecycle_active' is the same executable + # launched with auto_activate:=true, so it reaches "active" on its own. + demo_nodes=[demo_node] if demo_node else [], + port=PORT, + # The clear must reach HEALED, i.e. leave the default active-fault query, + # which the fault_manager only does when healing is enabled - see harness.py + # and the README's "Closing the loop". + healing_enabled=True, + healing_threshold=healing_threshold, + # Only "main" and "restart_departed" restart the gateway. The other scenarios must + # keep an unexpected gateway (or, for healing_threshold, target-node) death visible + # as a failure everywhere it is not the deliberate subject of the test. + gateway_respawn=(SCENARIO in ('main', 'restart_departed')), + ) + + if SCENARIO == 'healing_threshold': + executable, ros_name, namespace = DEMO_NODE_REGISTRY[TARGET_NODE] + target_node_action = launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable=executable, + name=ros_name, + namespace=namespace, + output='screen', + additional_env=get_coverage_env('ros2_medkit_integration_tests'), + sigterm_timeout='30', + sigkill_timeout='15', + # Respawns under the SAME node name after the test's own SIGTERM - see + # TestLifecycleExpectationHealingThreshold._blink(). This is the "node that + # respawned stuck" case the withheld-clear guard's design doc names. + respawn=True, + respawn_delay=HEALING_RESPAWN_DELAY_SEC, + ) + launch_description.add_action(TimerAction(period=2.0, actions=[target_node_action])) + context['target_node'] = target_node_action + + if SCENARIO == 'departure_keeps': + executable, ros_name, namespace = DEMO_NODE_REGISTRY[UNREADABLE_NODE] + target_node_action = launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable=executable, + name=ros_name, + namespace=namespace, + output='screen', + additional_env=get_coverage_env('ros2_medkit_integration_tests'), + sigterm_timeout='30', + sigkill_timeout='15', + # No respawn=True: unlike healing_threshold's blink, this scenario kills the + # fixture PERMANENTLY - the whole point is proving the clear that follows a + # real, sustained departure, never a bounce back. + ) + launch_description.add_action(TimerAction(period=2.0, actions=[target_node_action])) + context['target_node'] = target_node_action + + if SCENARIO == 'not_managed': + executable, ros_name, namespace = DEMO_NODE_REGISTRY[NOT_MANAGED_NODE] + target_node_action = launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable=executable, + name=ros_name, + namespace=namespace, + output='screen', + additional_env=get_coverage_env('ros2_medkit_integration_tests'), + sigterm_timeout='30', + sigkill_timeout='15', + # No respawn=True: same permanent-kill shape as "departure_keeps", proving the + # fault survives a real, sustained departure rather than a bounce back. + ) + launch_description.add_action(TimerAction(period=2.0, actions=[target_node_action])) + context['target_node'] = target_node_action + + if SCENARIO == 'restart_loop': + executable, ros_name, namespace = DEMO_NODE_REGISTRY[NOT_MANAGED_NODE] + target_node_action = launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable=executable, + name=ros_name, + namespace=namespace, + output='screen', + additional_env=get_coverage_env('ros2_medkit_integration_tests'), + sigterm_timeout='30', + sigkill_timeout='15', + # The crash loop itself: launch brings the same node straight back after every + # SIGTERM the test sends, and respawn_delay is an enforced floor under each + # absence (see RESTART_LOOP_RESPAWN_DELAY_SEC). + respawn=True, + respawn_delay=RESTART_LOOP_RESPAWN_DELAY_SEC, + ) + launch_description.add_action(TimerAction(period=2.0, actions=[target_node_action])) + context['target_node'] = target_node_action + + if SCENARIO in ('wide_grace', 'restart_departed'): + executable, ros_name, namespace = DEMO_NODE_REGISTRY[TARGET_NODE] + target_node_action = launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable=executable, + name=ros_name, + namespace=namespace, + output='screen', + additional_env=get_coverage_env('ros2_medkit_integration_tests'), + sigterm_timeout='30', + sigkill_timeout='15', + # No respawn: both scenarios kill it permanently and then measure what happens + # to the fault it left behind. + ) + launch_description.add_action(TimerAction(period=2.0, actions=[target_node_action])) + context['target_node'] = target_node_action + + if SCENARIO == 'cap_pressure': + # Only the tracked one is ever driven (its lifecycle services are dropped, then it is + # killed); the refused one just has to exist and stay stuck. + # + # STAGGERED, not launched together: with a cap of one, the single slot goes to + # whichever required node the detector matches FIRST, and two nodes started at the + # same instant are discovered in whatever order DDS and the entity cache happen to + # settle on. That would make which node is tracked and which is refused a coin flip, + # and every assertion below names one of them. The gap is many ticks at this + # scenario's 100 ms cadence, so the first node is tracked and carrying evidence long + # before the second one exists to be refused. + tracked = _droppable_node(CAP_TRACKED_NODE, 'unconfigured') + refused = _droppable_node(CAP_REFUSED_NODE, 'unconfigured') + launch_description.add_action(TimerAction(period=2.0, actions=[tracked])) + launch_description.add_action( + TimerAction(period=CAP_REFUSED_NODE_DELAY_SEC, actions=[refused])) + context['tracked_node'] = tracked + context['refused_node'] = refused + + if SCENARIO == 'unsettled_departure': + blink = _droppable_node(UNSETTLED_BLINK_NODE, 'active') + settled = _droppable_node(UNSETTLED_SETTLED_NODE, 'active') + launch_description.add_action(TimerAction(period=2.0, actions=[blink, settled])) + context['blink_node'] = blink + context['settled_node'] = settled + + return launch_description, context + + +def _droppable_node(name, state_label): + """One ``droppable_lifecycle_node`` instance under `name`, answering `state_label`. + + The fixture (``test/e2e/droppable_lifecycle_node.cpp``, built and installed by this + package's own CMakeLists) looks like a managed lifecycle node to the gateway's discovery + layer, answers ``get_state`` with `state_label`, and stops advertising both lifecycle + services when ``drop_services:=true`` is set on it. Launching two instances under + different names is how a scenario gets two independently drivable required nodes out of + one executable. + """ + return launch_ros.actions.Node( + package='ros2_medkit_graph_watchdog', + executable='droppable_lifecycle_node', + name=name, + output='screen', + parameters=[{'state_label': state_label}], + additional_env=get_coverage_env('ros2_medkit_graph_watchdog'), + sigterm_timeout='30', + sigkill_timeout='15', + ) + + +def _fault_record(port, code, timeout=30.0, interval=0.5): + """Poll ``GET /faults?status=all`` until `code` appears, whatever its status. + + ``poll_faults`` uses the default (pending+confirmed) filter, so a fault that + HEALED disappears from it - which is indistinguishable from a fault that was + never raised, and from a gateway that is up but has not reached the + fault_manager yet. The restart test needs the record itself, including the + fields that survive a heal, so it asks for every status. + + Returns the matching item dict, or ``None`` on timeout. + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/faults', params={'status': 'all'}, timeout=5) + if response.status_code == 200: + for item in response.json().get('items', []): + if item.get('fault_code') == code: + return item + except requests.exceptions.RequestException: + pass + time.sleep(interval) + return None + + +def _poll_fault_description_contains(port, code, needle, timeout=30.0, interval=0.5): + """Poll ``GET /faults?status=all`` until `code`'s description contains `needle`. + + A single read of the description is a snapshot of whatever the last emitted tick said, + so a claim about the description CHANGING (a newly admitted node appearing in it, a + departed one starting to say it has left) needs a poll rather than one sample plus a + sleep long enough to be safe. ``True`` once it matches, ``False`` on timeout, after + printing the last description seen - which is gone once the launch tears down. + """ + deadline = time.monotonic() + timeout + last_seen = f'{code} was never in the store at all' + while time.monotonic() < deadline: + record = _fault_record(port, code, timeout=interval) + if record is not None: + last_seen = record.get('description', '') + if needle in last_seen: + return True + time.sleep(interval) + print(f'_poll_fault_description_contains({code!r}, {needle!r}) timed out after ' + f'{timeout}s; last description: {last_seen!r}') + return False + + +def _wait_until_port_is_down(port, timeout=60.0, interval=0.2): + """Wait until the gateway's HTTP port stops answering. True once it does.""" + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + requests.get(f'{base}/health', timeout=2) + except requests.exceptions.RequestException: + return True + time.sleep(interval) + return False + + +def _poll_watchdog_entity(port, app_id, lifecycle, timeout=30.0, interval=0.5): + """Poll GET /x-medkit-watchdog until `app_id` appears with this lifecycle label. + + The absence scenarios need one fact wait_until_watchdog_armed cannot give them: + that the target's lifecycle label was actually READ by the live stack. The armed + gate treats an unread label as ok (node_ok defaults open for unknown state), and + the tracker treats an unread label as benign - so a run in which the label never + arrived produces the same silence the assertion is looking for. Pinning the label + through the plugin's own status route closes that hole: 'unconfigured' proves the + false-positive trigger was fully present, 'active' proves the negative control is + actually exercising an active node rather than an unread one, and '' (empty string) + proves the "unreadable" scenario's fixture was matched and its GetState genuinely + never came back - LifecycleWatcher seeds a tracked entry's label to "" and only + overwrites it once a real read succeeds (lifecycle_watcher.cpp), so '' is not "no + data yet", it is "asked, and still waiting". + + Returns the matching entity dict, or ``None`` on timeout (after printing the + last-seen payload, which is gone once the launch tears down). + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + last_seen = 'GET /x-medkit-watchdog was never answered at all' + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/x-medkit-watchdog', timeout=5) + if response.status_code != 200: + last_seen = f'HTTP {response.status_code} from GET /x-medkit-watchdog' + else: + status = response.json().get('x-medkit-watchdog', {}) + last_seen = str(status) + for entity in status.get('entities') or []: + if entity.get('id') == app_id and entity.get('lifecycle') == lifecycle: + return entity + except requests.exceptions.RequestException as exc: + last_seen = f'GET /x-medkit-watchdog failed: {exc}' + time.sleep(interval) + print(f'_poll_watchdog_entity(app_id={app_id!r}, lifecycle={lifecycle!r}) timed out ' + f'after {timeout}s; last watchdog status: {last_seen}') + return None + + +def _poll_apps_absent(port, app_id, timeout=30.0, interval=0.5): + """Poll ``GET /apps`` until `app_id` is no longer listed. ``True`` once it is gone. + + The "departure_keeps" scenario's own gate: confirms the fixture process it just + SIGTERM'd is really gone from the operator-visible SOVD entity graph before + asserting anything about the fault, or the clear that follows would prove nothing - + it could just as well be racing a graph the entity cache has not caught up with yet. + ``GET /apps`` and the detector's own per-tick snapshot read the exact same + ``ThreadSafeEntityCache`` (``DiscoveryHandlers::get_apps()`` and + ``PluginContextImpl::get_entity_snapshot()`` both call + ``node_->get_thread_safe_cache()``), so this is a poll of the precise input the + guard's absence counting is driven from, not a proxy for it. + + Parameters + ---------- + port : int + Gateway HTTP port. + app_id : str + The App id expected to disappear from ``GET /apps``. + timeout : float + Maximum time to wait in seconds. + interval : float + Sleep between retries in seconds. + + Returns + ------- + bool + ``True`` once `app_id` is absent from ``GET /apps``, ``False`` on timeout (after + printing the last-seen id list). + + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + last_seen = 'GET /apps was never answered at all' + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/apps', timeout=5) + if response.status_code == 200: + ids = [item.get('id') for item in response.json().get('items', [])] + last_seen = str(ids) + if app_id not in ids: + return True + else: + last_seen = f'HTTP {response.status_code} from GET /apps' + except requests.exceptions.RequestException as exc: + last_seen = f'GET /apps failed: {exc}' + time.sleep(interval) + print(f'_poll_apps_absent({app_id!r}) timed out after {timeout}s; last seen: {last_seen}') + return False + + +def _poll_apps_present(port, app_id, timeout=30.0, interval=0.5): + """Poll ``GET /apps`` until `app_id` IS listed. ``True`` once it appears. + + The "not_managed" scenario's own gate: `calibration` is launched via a delayed + ``TimerAction`` (2 s) plus its own process startup and DDS discovery, so at the moment + test_01 runs it may genuinely not have appeared yet. `_poll_apps_absent` above cannot + stand in for this the other way round with a short timeout: it returns True on the + FIRST poll that does not see the node, which on a fresh launch is every early poll + before the node has started - the opposite of what "present" needs. This instead loops + until the node genuinely appears (or the deadline is reached). + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + last_seen = 'GET /apps was never answered at all' + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/apps', timeout=5) + if response.status_code == 200: + ids = [item.get('id') for item in response.json().get('items', [])] + last_seen = str(ids) + if app_id in ids: + return True + else: + last_seen = f'HTTP {response.status_code} from GET /apps' + except requests.exceptions.RequestException as exc: + last_seen = f'GET /apps failed: {exc}' + time.sleep(interval) + print(f'_poll_apps_present({app_id!r}) timed out after {timeout}s; last seen: {last_seen}') + return False + + +def _set_bool_parameter(client_node, service_name, param_name, value, timeout=30.0): + """Set one bool parameter on a REMOTE node via its own ``~/set_parameters`` service. + + Drives ``unreadable_lifecycle_node.cpp``'s ``start_answering`` parameter at a time + the TEST controls - after it has independently confirmed GRAPH_NODE_UNREADABLE + actually raised - rather than racing a wall-clock timer in the fixture against the + 60-tick hold this scenario needs to observe first. Every ``rclcpp::Node`` starts + this service automatically (``start_parameter_services`` defaults to true, and + this fixture never overrides it), so no special wiring is needed on the node side + beyond the ``add_on_set_parameters_callback`` it already registers. + + Returns ``True`` once the remote node accepts the change, ``False`` if the service + never became available or the call did not complete/succeed. + """ + client = client_node.create_client(SetParameters, service_name) + if not client.wait_for_service(timeout_sec=timeout): + return False + request = SetParameters.Request() + request.parameters = [Parameter( + name=param_name, + value=ParameterValue(type=ParameterType.PARAMETER_BOOL, bool_value=value), + )] + future = client.call_async(request) + rclpy.spin_until_future_complete(client_node, future, timeout_sec=timeout) + result = future.result() + if result is None or not result.results: + return False + return bool(result.results[0].successful) + + +def _pump_sse_stream(response, frames, stop_event): + """Collect SSE frames as dicts of field -> value, same shape as data.py's fixture. + + Mirrors test_sse_fault_stream.test.py's ``_pump_stream`` - this file's row-6 test is + the only reader of ``GET /faults/stream`` in this suite, so the parsing lives here + rather than being shared, matching the rest of this module's harness.py-vs-local split. + """ + current = {} + try: + for line in response.iter_lines(decode_unicode=True): + if stop_event.is_set(): + break + if line is None: + continue + if line == '': + if current: + frames.append(current) + current = {} + continue + if line.startswith(':'): + continue # keepalive comment + key, _, value = line.partition(':') + current[key.strip()] = value.strip() + except Exception: # noqa: BLE001 - closed socket on test teardown + pass + + +class TestLifecycleExpectationMain(unittest.TestCase): + """GRAPH_NODE_INACTIVE raises for a required-but-unconfigured node, heals on activate.""" + + @classmethod + def setUpClass(cls): + rclpy.init() + cls._client_node = Node('lifecycle_expectation_e2e_client') + cls._change_state = cls._client_node.create_client(ChangeState, CHANGE_STATE_SERVICE) + + @classmethod + def tearDownClass(cls): + cls._client_node.destroy_node() + rclpy.shutdown() + + def _call_transition(self, transition_id, reached_label, timeout=30.0, attempts=3): + """Drive one real lifecycle transition, retrying a call that never comes back. + + The whole scenario hangs off two of these, and a single-shot call makes one lost + response (or a demo-node executor stalled under a sanitizer at the wrong moment) + fail the entire multi-minute run on one RPC. The retry does NOT widen the budget: + the same total wait is split across `attempts`, so a genuinely dead service still + fails after the same wall time, with the last attempt's outcome named. + + `reached_label` is the lifecycle state this transition ends in. It is needed + because a retry introduces a case a single-shot call does not have: if the FIRST + attempt was applied and only its response was lost, the retried transition is + invalid from the state the node has already reached and comes back rejected. That + is a success, not a failure, and the only way to tell it from a genuinely refused + transition is to look at where the node actually is. + """ + self.assertTrue( + type(self)._change_state.wait_for_service(timeout_sec=timeout), + f'{CHANGE_STATE_SERVICE} never became available', + ) + per_attempt = timeout / attempts + result = None + retried = False + for attempt in range(attempts): + request = ChangeState.Request() + request.transition.id = transition_id + future = type(self)._change_state.call_async(request) + rclpy.spin_until_future_complete( + type(self)._client_node, future, timeout_sec=per_attempt) + result = future.result() + if result is not None: + break + # Drop the request the server may still answer, so the next attempt's future + # cannot be completed by a straggling response to this one. + type(self)._change_state.remove_pending_request(future) + retried = True + print(f'ChangeState (transition {transition_id}) attempt {attempt + 1} of ' + f'{attempts} did not complete within {per_attempt:.1f}s; retrying') + self.assertIsNotNone( + result, + f'ChangeState call (transition {transition_id}) never completed in {attempts} ' + f'attempts over {timeout}s', + ) + if result.success: + return + self.assertTrue( + retried, + f'ChangeState call (transition {transition_id}) was rejected on its first ' + 'attempt - the node refused a transition that must be valid from where it was', + ) + self.assertIsNotNone( + _poll_watchdog_entity(PORT, TARGET_NODE, reached_label, timeout=10.0), + f'ChangeState (transition {transition_id}) was rejected after a retry and ' + f'{TARGET_NODE} is not in "{reached_label}" either - the transition neither ' + 'applied on the lost attempt nor succeeded on the retry', + ) + + def test_01_present_but_unconfigured_raises_naming_it(self): + # Gate on the plugin being live and globally armed BEFORE polling for the raise. + # Not on app_id=TARGET_NODE, deliberately: the gate reports a tracked node with a + # known non-active label as 'warming_up' (its node_ok is false - the exact + # condition under test), so that per-entity state stays suppressed for as long + # as the fault condition exists. What the raise actually needs is the SOURCE + # entity's arming (the aggregate goes out under 'graph_watchdog'), for which + # global armed is the precise precondition - and the gate still proves the .so + # loaded and the tick loop ran, so a bringup failure dies here naming itself + # instead of as a 60 s poll timeout blaming the detector. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed global state - the plugin did not ' + 'load, its tick loop never ran, or the bringup grace never elapsed, so no ' + 'raise below could mean anything', + ) + + # managed_lifecycle launches unconfigured: present in the graph, alive, but not + # active. That alone must raise once it persists past the configured grace. + fault = poll_faults(PORT, FAULT_CODE, timeout=60.0) + self.assertIsNotNone( + fault, + f'{FAULT_CODE} never raised while {TARGET_NODE} stayed unconfigured', + ) + self.assertIn(TARGET_NODE, fault.get('description', '')) + + # The flat /faults list carries a fault whatever its source is; only the + # entity-scoped surface proves an operator can OPEN it somewhere (see + # test_qos_e2e.test.py's identical rationale for the same entity). + self.assertIsNotNone( + poll_entity_faults(PORT, 'apps/graph_watchdog', FAULT_CODE, timeout=30.0), + f'{FAULT_CODE} is not reachable at /apps/graph_watchdog/faults - the ' + 'entity the plugin publishes does not own the fault it raises', + ) + + def test_02_configured_but_inactive_still_raises(self): + # Explicitly reach "inactive" (unconfigured -> configuring -> inactive) and + # deliberately do NOT activate. Inactive is still not active, so the + # level-triggered raise must keep the fault active - a detector that treated + # "configured" as good enough would heal it here. + self._call_transition(Transition.TRANSITION_CONFIGURE, reached_label='inactive') + + fault = poll_faults(PORT, FAULT_CODE, timeout=30.0) + self.assertIsNotNone( + fault, + f'{FAULT_CODE} did not stay raised once {TARGET_NODE} reached "inactive"', + ) + self.assertIn(TARGET_NODE, fault.get('description', '')) + + def test_02b_gateway_restart_never_heals_a_still_inactive_node(self, gateway_node): + """The withheld-clear guard, at the tier that counts: a real restart. + + The guard exists because a gateway restart brings every detector counter back + to zero while the fault it raised is still in the fault_manager's store (a + separate process, so it survives). For a whole `grace` window after the + restart the tracker reports nothing affected even though the detector's own + reads say the node is still not active, and the level-triggered emitter turns + that into a clear - which, with healing enabled, walks a CONFIRMED + GRAPH_NODE_INACTIVE to HEALED on a robot that never moved. + + Nothing below this tier can reach that state: the C++ integration test rebuilds + the detector against a fake ReportFault sink, and the other e2e scenarios run + one gateway process from start to finish. + + The instrument is ``last_passed``, not the fault's current status. A heal here + is TRANSIENT - the re-raise follows within a few ticks - so a status sample + taken after the fact sees CONFIRMED again and proves nothing. ``last_passed`` + is set by the fault_manager on the FIRST PASSED report a fault ever receives + and is never unset, so it catches a single spurious clear whether or not it + ever reached HEALED. + """ + before = _fault_record(PORT, FAULT_CODE, timeout=30.0) + self.assertIsNotNone( + before, + f'{FAULT_CODE} is not in the store before the restart, so there is nothing ' + 'a restart could wrongly heal', + ) + self.assertIsNone( + before.get('last_passed'), + 'the detector had already emitted a clear for this fault BEFORE the restart ' + f'(last_passed={before.get("last_passed")!r}), so the assertion after the ' + 'restart could not attribute anything to it', + ) + + old_pid = gateway_node.process_details['pid'] + os.kill(old_pid, signal.SIGTERM) + self.assertTrue( + _wait_until_port_is_down(PORT, timeout=60.0), + f'the gateway (pid {old_pid}) kept answering after SIGTERM - nothing was ' + 'restarted, so the rest of this test would measure the original process', + ) + + # launch brings the same configuration back. Gate on the plugin being live and + # globally armed again: a raise (or a clear) is only possible past that point, + # so measuring before it would measure a stack that had not started detecting. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=90.0), + 'the gateway never came back armed after the restart', + ) + new_pid = gateway_node.process_details['pid'] + self.assertNotEqual( + new_pid, old_pid, + 'the gateway process id did not change, so this test never restarted anything', + ) + + # Long enough for the whole spurious-heal sequence to have played out: the + # detector needs GRACE ticks to walk its fresh streak back up, and the + # fault_manager needs healing_threshold (default 3) PASSED reports to heal. + time.sleep(max(4.0, (GRACE + 8) * TICK_INTERVAL_MS / 1000.0)) + + after = _fault_record(PORT, FAULT_CODE, timeout=30.0) + self.assertIsNotNone( + after, + f'{FAULT_CODE} vanished from the store entirely after the restart', + ) + self.assertIsNone( + after.get('last_passed'), + 'the restarted gateway reported GRAPH_NODE_INACTIVE as PASSED while ' + f'{TARGET_NODE} was still inactive (last_passed=' + f'{after.get("last_passed")!r}) - the restart healed a fault that is still ' + 'real', + ) + self.assertNotEqual( + after.get('status'), 'healed', + f'{FAULT_CODE} is HEALED while {TARGET_NODE} has never left "inactive"', + ) + + # And the fault is still on the operator-visible active list, i.e. the withhold + # preserved it rather than merely delaying the damage. + self.assertIsNotNone( + poll_faults(PORT, FAULT_CODE, timeout=30.0), + f'{FAULT_CODE} is no longer an active fault after the gateway restart', + ) + + def test_03_activated_arms_the_gate_and_heals(self): + # "Cleared" is an absence, and it is the default state of a fault that was never + # raised: on a stack test_01 just failed against, poll_cleared answers True + # immediately. Confirm the fault is actually THERE first - BEFORE the activate, + # because after it the heal is racing this check - so what poll_cleared measures + # is a heal of a proven-present fault (same pairing as + # test_param_drift_e2e.test.py's heal leg). + self.assertIsNotNone( + poll_faults(PORT, FAULT_CODE, timeout=30.0), + f'{FAULT_CODE} is not active at the start of the heal test, so there is ' + 'nothing here that could heal and the clear below would pass without ' + 'measuring anything', + ) + + self._call_transition(Transition.TRANSITION_ACTIVATE, reached_label='active') + + # Now - and only now - the per-entity armed gate is reachable: the entity leaves + # 'warming_up' exactly when the watcher has read "active" for it. Waiting on it + # splits the two ways the heal could fail: if the REAL lifecycle machinery never + # fed the label to the gate, the failure is here and names the node; if the + # label arrived but the clear never flowed, poll_cleared below is the one that + # fails. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0, app_id=TARGET_NODE), + f'the gate never armed {TARGET_NODE} after a successful ACTIVATE - the ' + 'live lifecycle machinery (transition_event / GetState) never delivered ' + 'the "active" label to the watcher', + ) + + cleared = poll_cleared(PORT, FAULT_CODE, timeout=60.0) + self.assertTrue( + cleared, + f'{FAULT_CODE} did not heal after {TARGET_NODE} reached "active"', + ) + + # The clear above does not by itself prove the READ caused it: a fixture exit + # right after the "active" observation would clear the same fault through the + # absence path (AbsenceAfterRaiseClearsNotNodeDeathsDomain's own mechanism, at + # the tracker tier), which would make this heal test pass for the wrong reason. + # Pin that the node is still present and reading "active" once the clear was + # observed. + self.assertIsNotNone( + _poll_watchdog_entity(PORT, TARGET_NODE, 'active', timeout=15.0), + f'{TARGET_NODE} is no longer present and reading "active" once the clear was ' + 'observed - the clear could have come from the absence path (a fixture exit) ' + 'rather than the ACTIVATE that this test is actually about', + ) + + +class TestLifecycleExpectationDefaultConfig(unittest.TestCase): + """The shipped default (no config at all) raises nothing for an inactive node.""" + + def test_default_config_stays_silent(self): + # Gate BEFORE asserting absence: a stack that never came up produces exactly the + # same "no fault" result, and every bringup failure mode on this launch still + # exits 0 (see the harness docstring). + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed global state - the plugin did not ' + 'load or its tick loop never ran, so an absent fault proves nothing', + ) + # The armed gate is served by the plugin inside the gateway process, so it says + # nothing about the surface the assertion below actually reads. A launch whose + # fault_manager died, hung, or never DDS-matched answers GET /faults with 503, + # poll_faults swallows that and returns None, and the absence assertion passes + # for the one reason it must never pass. + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=30.0), + 'GET /faults never answered 200 - the gateway never reached the ' + 'fault_manager in this launch, so an absent fault proves nothing about the ' + 'detector', + ) + # Prove the false-positive trigger is fully present: the stack discovered the + # node AND read its non-active label. Without this the silence below could just + # as well mean the label never arrived. + self.assertIsNotNone( + _poll_watchdog_entity(PORT, TARGET_NODE, 'unconfigured', timeout=30.0), + f'the gate never reported {TARGET_NODE} with lifecycle "unconfigured" - ' + 'the node was not discovered or its label was never read, so the silence ' + 'below would be vacuous', + ) + # Proving absence means waiting out the full window, with the channel proven + # alive for every poll across it - not merely a poll that timed out silently + # (see assert_fault_absent_throughout's own docstring for why poll_faults + + # assertIsNone cannot tell "no such fault" from "could not ask"): with no + # detectors.lifecycle_expectation config, require_active is empty and the + # detector must check nothing - the README's zero-false-positive default. + assert_fault_absent_throughout(self, PORT, FAULT_CODE, SILENT_WINDOW_SEC) + # The trigger was pinned at the START of the window; nothing observed it after + # that. A demo node that exits 0 two seconds in is tolerated by the exit-code + # check and invisible to the tracker (a departed node is the presence class's + # problem), which would leave 2 s of trigger and 18 s of empty graph. + self.assertIsNotNone( + _poll_watchdog_entity(PORT, TARGET_NODE, 'unconfigured', timeout=15.0), + f'{TARGET_NODE} is no longer reported as "unconfigured" at the END of the ' + 'silence window - the trigger did not survive it, so most of the window ' + 'proved nothing', + ) + + def test_silence_proof_catches_a_dead_fault_surface(self): + """The test of the test. + + `assert_fault_absent_throughout` must fail when the channel it watches dies + mid-window - the exact hole `assertIsNone(poll_faults(...))` could not see (see + the harness docstring and `test_default_config_stays_silent` above, which now + uses the fixed helper). Self-contained - a local HTTP server stands in for + `/faults`, not this launch's real gateway - so it runs alongside the real + assertion above without depending on it, and needs no ROS graph of its own. + """ + prove_silence_proof_catches_a_dead_fault_surface(self) + + +class TestLifecycleExpectationNegativeControl(unittest.TestCase): + """A required node that IS active never raises.""" + + def test_active_required_node_never_raises(self): + # Gate on THIS app being armed: for an active node the per-entity state is + # reachable, and it is the strongest form of the arming gate - it proves the + # .so loaded, the tick loop ran, the node was discovered, its warmup elapsed, + # and the watcher considers it ok. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0, app_id=ACTIVE_NODE), + f'graph_watchdog never reported {ACTIVE_NODE} as an armed entity - the ' + 'plugin did not load, its tick loop never ran, or the gateway never ' + 'discovered the node, so an absent fault proves nothing', + ) + # And that the surface the assertion reads is alive in THIS launch: this is the + # scenario whose job is "a buggy detector raising for an active node must be + # caught", and a raise that is emitted but lost - fault_manager dead, hung, or + # never DDS-matched - leaves GET /faults answering 503, which poll_faults + # swallows into exactly the None the assertion below wants. + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=30.0), + 'GET /faults never answered 200 - the gateway never reached the ' + 'fault_manager in this launch, so a wrong raise would have been lost rather ' + 'than caught', + ) + # Armed alone is not enough for THIS absence: node_ok also holds for an unread + # label, and the tracker treats unread as benign - a run whose label never + # arrived would stay silent for the wrong reason. Pin that "active" was READ. + self.assertIsNotNone( + _poll_watchdog_entity(PORT, ACTIVE_NODE, 'active', timeout=30.0), + f'the gate never reported {ACTIVE_NODE} with lifecycle "active" - its ' + 'label was never read, so the silence below would be vacuous', + ) + # Same grace and cadence as the "main" scenario, so the only variable between + # the launch that must raise and this one is the node's actual lifecycle state. + # The channel is proven alive for every poll across the window (see + # assert_fault_absent_throughout's own docstring), not just a poll that timed + # out silently. + assert_fault_absent_throughout(self, PORT, FAULT_CODE, SILENT_WINDOW_SEC) + # The required node must still be there, and still active, at the END of the + # window: a node that exited early leaves an empty graph, which is silent for a + # reason that has nothing to do with the detector being right. + self.assertIsNotNone( + _poll_watchdog_entity(PORT, ACTIVE_NODE, 'active', timeout=15.0), + f'{ACTIVE_NODE} is no longer reported as "active" at the END of the silence ' + 'window - the required node did not survive it, so most of the window was ' + 'not exercising an active node at all', + ) + + +class TestLifecycleExpectationUnreadable(unittest.TestCase): + """GRAPH_NODE_UNREADABLE raises for a node whose lifecycle state is never read. + + Unlike every scenario above - which all drive ``managed_lifecycle_node.cpp``, a REAL + ``rclcpp_lifecycle::LifecycleNode`` that always answers GetState - this launches + ``unreadable_lifecycle_node.cpp``: a fixture that advertises ``get_state``/ + ``change_state`` with the right service TYPES (what ``find_lifecycle_get_state_path`` + actually checks - see that fixture's file doc) but whose ``get_state`` never responds + until told to. That is the only way to reach GRAPH_NODE_UNREADABLE's raise against a + REAL, sustained GetState failure: nothing else in this package can hold a real node + unreadable for 60 consecutive ticks without either racing DDS timing or a demo + executable built exactly for this. + + Proves, in order, the four facts the split between GRAPH_NODE_INACTIVE and + GRAPH_NODE_UNREADABLE exists to keep separate: + + 1. The required node is present and matched, and its lifecycle state has never been + read (test_01) - asserted BEFORE anything else, or the rest proves nothing. + 2. GRAPH_NODE_INACTIVE is NOT raised for it (test_02) - not merely "not yet": an + unread label feeds the unmeasured clock, never the violation streak + (lifecycle_expectation_tracker.hpp), so this is impossible by construction, and + this poll is what proves that live rather than trusting the source. + 3. GRAPH_NODE_UNREADABLE IS raised once the 60-tick hold expires, names the node, and + carries SEVERITY_WARN on the stored record (test_03) - and GRAPH_NODE_INACTIVE is + STILL silent at that same moment: a node is content of at most one of the two, + never both. + 4. Once the fixture starts answering "active", the watcher reads it and + GRAPH_NODE_UNREADABLE clears (test_04) - the real GetState round trip, not a + label injected through a test seam. + """ + + @classmethod + def setUpClass(cls): + rclpy.init() + cls._client_node = Node('lifecycle_expectation_unreadable_e2e_client') + + @classmethod + def tearDownClass(cls): + cls._client_node.destroy_node() + rclpy.shutdown() + + def test_01_present_and_unread_before_anything_else(self): + # Gate on the plugin being live and globally armed before anything else - a + # bringup failure must die here, naming itself, instead of as a raise-poll + # timeout that blames the detector. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed global state - the plugin did not ' + 'load, its tick loop never ran, or the bringup grace never elapsed, so no ' + 'assertion below could mean anything', + ) + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=30.0), + 'GET /faults never answered 200 - the gateway never reached the ' + 'fault_manager in this launch, so a wrong raise would have been lost rather ' + 'than caught', + ) + + # The fact this whole scenario stands on, asserted first: the node is present, + # matched by the require_active entry, and its lifecycle label reads exactly "" + # - LifecycleWatcher seeded it once and nothing has answered since (see + # _poll_watchdog_entity's docstring for why "" is not "no data yet"). + self.assertIsNotNone( + _poll_watchdog_entity(PORT, UNREADABLE_NODE, '', timeout=30.0), + f'the gate never reported {UNREADABLE_NODE} with an empty (never-read) ' + 'lifecycle label - the fixture was not discovered as a managed node, or its ' + 'GetState answered when it must not have, so nothing below would prove ' + 'anything about an UNREADABLE node', + ) + + def test_02_confirmed_inactive_is_never_raised_for_an_unread_node(self): + # GRAPH_NODE_INACTIVE must stay silent for the whole window, with the channel + # proven alive for every poll across it (see assert_fault_absent_throughout's own + # docstring): an unread label can never become a "confirmed violation" (see the + # class docstring). The ticks this window spends also count toward the 60-tick + # UNREADABLE hold in the background, so nothing here is wasted time. + assert_fault_absent_throughout(self, PORT, FAULT_CODE, UNREADABLE_INACTIVE_SILENCE_SEC) + # And the node must still be genuinely unread at the END of the window, or the + # silence above measured an empty graph rather than a real unreadable node. + self.assertIsNotNone( + _poll_watchdog_entity(PORT, UNREADABLE_NODE, '', timeout=15.0), + f'{UNREADABLE_NODE} no longer reads an empty (never-read) lifecycle label ' + 'at the end of the silence window - the trigger did not survive it, so ' + 'most of the window proved nothing', + ) + + def test_03_unreadable_raises_naming_the_node_at_warn_severity(self): + fault = poll_faults(PORT, FAULT_CODE_UNREADABLE, timeout=UNREADABLE_RAISE_TIMEOUT_SEC) + self.assertIsNotNone( + fault, + f'{FAULT_CODE_UNREADABLE} never raised while {UNREADABLE_NODE} stayed ' + 'unreadable past the 60-tick hold', + ) + self.assertIn(UNREADABLE_NODE, fault.get('description', '')) + self.assertEqual( + fault.get('severity_label'), 'WARN', + f'{FAULT_CODE_UNREADABLE} must carry SEVERITY_WARN on the stored record ' + f'(a merely UNVERIFIED promise, not a confirmed violation), got ' + f'{fault.get("severity_label")!r}', + ) + + # Still true at the exact moment UNREADABLE is raised: a node is content of at + # most one of the two codes, never both. Proven across a whole window with the + # channel checked on every poll, not by a single poll that timed out silently - + # see assert_fault_absent_throughout's own docstring. + assert_fault_absent_throughout(self, PORT, FAULT_CODE, MUTUAL_EXCLUSION_WINDOW_SEC) + + # The flat /faults list carries a fault whatever its source is; only the + # entity-scoped surface proves an operator can OPEN it somewhere (see + # TestLifecycleExpectationMain.test_01's identical rationale for the sibling code). + self.assertIsNotNone( + poll_entity_faults(PORT, 'apps/graph_watchdog', FAULT_CODE_UNREADABLE, timeout=30.0), + f'{FAULT_CODE_UNREADABLE} is not reachable at /apps/graph_watchdog/faults - ' + 'the entity the plugin publishes does not own the fault it raises', + ) + + def test_04_answering_active_clears_it(self): + self.assertIsNotNone( + poll_faults(PORT, FAULT_CODE_UNREADABLE, timeout=30.0), + f'{FAULT_CODE_UNREADABLE} is not active at the start of the heal test, so ' + 'there is nothing here that could heal and the clear below would pass ' + 'without measuring anything', + ) + + self.assertTrue( + _set_bool_parameter( + type(self)._client_node, f'/{UNREADABLE_NODE}/set_parameters', + 'start_answering', True, timeout=30.0), + f'setting start_answering:=true on /{UNREADABLE_NODE}/set_parameters ' + 'never succeeded - the fixture never got the signal to start answering ' + 'GetState, so no heal below could mean anything', + ) + + # Proves the REAL GetState round trip fed the watcher - not merely that the + # parameter was accepted. Only now is the per-entity armed gate meaningful for + # this node: node_ok() was already true while unread (an unread label defaults + # open, see lifecycle_watcher.cpp), so this is the read itself, not the gate. + self.assertIsNotNone( + _poll_watchdog_entity(PORT, UNREADABLE_NODE, 'active', timeout=30.0), + f'the gate never reported {UNREADABLE_NODE} with lifecycle "active" after ' + 'start_answering:=true - the fixture accepted the parameter but its ' + 'GetState still never delivered "active" to the watcher', + ) + + cleared = poll_cleared(PORT, FAULT_CODE_UNREADABLE, timeout=30.0) + self.assertTrue( + cleared, + f'{FAULT_CODE_UNREADABLE} did not heal after {UNREADABLE_NODE} started ' + 'answering "active"', + ) + + +class TestLifecycleExpectationHealingThreshold(unittest.TestCase): + """healing_threshold=1 must not let repeated snapshot blinks heal a still-violating fault. + + Reproduces the exact mechanism the design doc describes: a required node stuck + inactive is raised, then blinks out of the graph - a real SIGTERM + relaunch of the + SAME node under the same name, not a lifecycle transition - twice, each blink well + inside the tracker's own absence grace (a fixed 3 ticks, not configurable). Before + this slice's fix, every absent tick emitted a spurious PASSED regardless of grace, + which at a sensitive healing_threshold of 1 (the value this package's own README + recommends) is enough to walk a still-violating fault to HEALED. + + The instrument is ``last_passed`` (same as test_02b's restart leg above) and the + fault's ``status``, both read from the real ``GET /api/v1/faults`` surface - not + whether a PASSED reached some intercepted service. This is the one tier that runs + the REAL fault_manager debounce state machine at all: nothing below e2e ever touches + the actual debounce counter, only a fake ReportFault sink. + + The same run also carries this package's ONLY e2e coverage of a node oscillating + readable/unreadable producing no fault EVENT churn: each blink's respawn window + briefly re-seeds the required node's lifecycle label (present, but unread) before it + settles back to "unconfigured". GRAPH_NODE_INACTIVE carries a fixed SEVERITY_ERROR + (no per-tick severity choice on this fault - each of GRAPH_NODE_INACTIVE and its + sibling GRAPH_NODE_UNREADABLE is now a separate, fixed-severity record, see the + design doc), so a severity regression would still show up here. ``GET /faults/stream`` + is read raw for the whole blink sequence and checked for two things a poll of + ``GET /faults`` cannot guarantee it would catch: no ``fault_cleared`` frame for + GRAPH_NODE_INACTIVE at any point (the raw-event twin of the ``last_passed`` check), + and every ``fault_confirmed``/``fault_updated`` frame for GRAPH_NODE_INACTIVE carrying + ``severity_label: "ERROR"`` (its only possible value now). This run's blinks are far + too short to ever convert the required node into GRAPH_NODE_UNREADABLE content (that + needs 60 consecutive matched ticks, not a few seconds of respawn), so no GRAPH_NODE_UNREADABLE + frame is expected on this stream either. Reaching GRAPH_NODE_UNREADABLE's own raise + through a real, sustained (60-tick) GetState failure is not this scenario's job: the + demo node used here (``managed_lifecycle``) answers GetState as soon as it is + discovered. See the "unreadable" scenario in this same file + (``TestLifecycleExpectationUnreadable``) for that proof, driven by + ``unreadable_lifecycle_node.cpp``'s deferred-response fixture; the integration tier's + ``ManagedNodeWhoseGetStateNeverAnswersIsReportedUnreadableNotInactive`` and the + injection-seam tests beside it still own the fake-ReportFault-sink version of the same + claim. + """ + + @classmethod + def setUpClass(cls): + rclpy.init() + + @classmethod + def tearDownClass(cls): + rclpy.shutdown() + + def _blink(self, node_action, blink_number): + """SIGTERM the current target-node process, then confirm it came back stuck. + + ``respawn=True`` on the launch action brings a brand-new process back under the + same ROS node name after ``HEALING_RESPAWN_DELAY_SEC``, so the node returns + exactly as unconfigured (still violating) as it left - only its identity as a + DDS participant changed. SIGTERM, not SIGKILL, so the exit code the eventual + gateway-teardown check sees stays inside ``ALLOWED_EXIT_CODES`` (same reason + test_02b restarts the gateway with SIGTERM rather than SIGKILL). + """ + old_pid = node_action.process_details['pid'] + os.kill(old_pid, signal.SIGTERM) + + # This scenario is named for a departure that stays INSIDE the tracker's + # absence grace (kDefaultAbsenceGrace, 3 ticks - fixed, not configurable) - not + # merely "a respawn happened eventually". Sample GET /apps while the respawn + # settles so the blink is PROVEN absent, rather than assumed from + # HEALING_BLINK_SLEEP_SEC's own comment alone. + absence_started = time.monotonic() + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=HEALING_BLINK_SLEEP_SEC), + f'blink {blink_number}: {TARGET_NODE} was never observed absent from GET /apps ' + f'within {HEALING_BLINK_SLEEP_SEC}s of the SIGTERM - this blink may not have ' + 'exercised a real departure at all, so nothing below proves the absence grace ' + 'held', + ) + # The return has to be proven from the DISCOVERY view, not from the watchdog's + # entity list: ReliabilityGate::status_json builds its entities from + # WarmupTracker::entries(), which RETAINS an entry for forget_grace_ ticks after + # the node stops being present. Polling the watchdog for "unconfigured" therefore + # matches the entry left over from before the SIGTERM and returns while the + # replacement process is milliseconds old - so everything measured from it, + # including the absence duration below, describes a node the tracker never saw + # leave. GET /apps carries no such retention. + self.assertTrue( + _poll_apps_present(PORT, TARGET_NODE, timeout=15.0), + f'blink {blink_number}: {TARGET_NODE} never came back into GET /apps - the ' + 'respawn or its rediscovery did not complete, so this blink cannot be trusted ' + 'to have stayed inside the absence grace', + ) + absence_duration = time.monotonic() - absence_started + # kDefaultAbsenceGrace ticks at this scenario's own (sped-up) cadence - the exact + # budget the withheld-clear guard's absence leg holds for. A blink that overran + # it would be exercising the ordinary (and separately proven) sustained-absence + # clear instead of the guard this scenario is actually about. + grace_budget_sec = (HEALING_ABSENCE_GRACE_TICKS * HEALING_TICK_INTERVAL_MS) / 1000.0 + tick_sec = HEALING_TICK_INTERVAL_MS / 1000.0 + # The lower edge of the window, and the one that used to be missing entirely. The + # tracker only counts an absent TICK; an absence shorter than one tick interval can + # fall wholly between two samples, in which case the node was never observed away + # and this blink exercised nothing at all. An absence longer than one interval must + # contain at least one sample. + self.assertGreater( + absence_duration, tick_sec, + f'blink {blink_number}: {TARGET_NODE} was away for only {absence_duration:.2f}s, ' + f'less than the {tick_sec:.1f}s tick interval - the tracker can have sampled it ' + 'as present on every tick, so this blink does not exercise the absence grace', + ) + self.assertLess( + absence_duration, grace_budget_sec, + f'blink {blink_number}: {TARGET_NODE} took {absence_duration:.2f}s to return - ' + f'past the {grace_budget_sec:.1f}s absence-grace budget this scenario depends ' + 'on staying inside, so this blink no longer proves the withheld-clear guard, ' + 'only the ordinary sustained-absence clear', + ) + # Polled, not read once: launch updates process_details from its own event loop, so + # a single read here races the respawn it is meant to confirm - the gateway can see + # the replacement through DDS before launch has recorded it. + pid_deadline = time.monotonic() + 10.0 + new_pid = node_action.process_details['pid'] + while new_pid == old_pid and time.monotonic() < pid_deadline: + time.sleep(0.1) + new_pid = node_action.process_details['pid'] + self.assertNotEqual( + new_pid, old_pid, + f'blink {blink_number}: the node process id did not change, so nothing ' + 'actually blinked', + ) + + def test_repeated_blinks_do_not_heal_a_still_violating_fault(self, target_node): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed global state for the ' + 'healing_threshold scenario - no assertion below could mean anything', + ) + self.assertIsNotNone( + _poll_watchdog_entity(PORT, TARGET_NODE, 'unconfigured', timeout=30.0), + f'{TARGET_NODE} never read "unconfigured" - the false-positive trigger this ' + 'test needs was never actually present', + ) + + fault = poll_faults(PORT, FAULT_CODE, timeout=60.0) + self.assertIsNotNone(fault, f'{FAULT_CODE} never raised for the stuck {TARGET_NODE}') + before = _fault_record(PORT, FAULT_CODE, timeout=30.0) + self.assertIsNotNone(before, f'{FAULT_CODE} is not in the store before the blinks') + self.assertIsNone( + before.get('last_passed'), + f'{FAULT_CODE} already had a PASSED on record before any blink ' + f'(last_passed={before.get("last_passed")!r}), so the assertions below could ' + 'not attribute anything to the blinks below', + ) + + # Row-6 instrument: GET /faults/stream, opened BEFORE the blinks and read raw for + # their whole duration. Polling GET /faults (above and below) can step over a + # transient frame between two samples; the SSE stream cannot - every event the + # fault_manager published is either in the replay buffer or was delivered to this + # live connection. Each blink is a real node oscillating readable (the required + # node's own lifecycle label, unrelated to THIS fault) between present and + # absent/re-seeding, and GRAPH_NODE_INACTIVE is emitted every tick from a + # class-scoped, fixed-severity AggregatedFault member (see the class docstring + # above) - so this is where the level-triggered re-send racing the blink, or a + # stray event this fixed-severity design should make impossible, would show up as + # an extra or wrong event. + base = f'http://127.0.0.1:{PORT}{API_BASE_PATH}' + sse_frames = [] + sse_stop = threading.Event() + sse_response = requests.get(f'{base}/faults/stream', stream=True, timeout=(5, 120)) + self.assertEqual(sse_response.status_code, 200) + sse_pump = threading.Thread( + target=_pump_sse_stream, args=(sse_response, sse_frames, sse_stop), daemon=True) + sse_pump.start() + try: + for i in range(1, HEALING_BLINK_COUNT + 1): + frames_before_blink = len(sse_frames) + self._blink(target_node, i) + # A stream that silently died (connection still open, nothing being + # delivered - the status-code check at open time above cannot catch + # that) would let every "no fault_cleared frame" assertion below pass + # on zero evidence. Prove the reader was alive THROUGH this blink, not + # merely that it connected before the first one. + self.assertTrue( + sse_pump.is_alive(), + f'the SSE reader thread died during blink {i} - a dead stream would ' + 'let every assertion below pass without having observed anything', + ) + # And that it actually delivered something across the blink: GRAPH_NODE_INACTIVE + # is emitted every tick this fault stays raised (see the class docstring + # above), so a real blink at this scenario's 1 s tick cadence must produce + # at least one fresh frame - a settle window here (not merely at the very + # end) is what lets THIS blink's own frame(s) be observed before moving on. + time.sleep(2.0) + self.assertGreater( + len(sse_frames), frames_before_blink, + f'no SSE frame arrived during or after blink {i} - the stream may have ' + 'died silently (still connected, nothing delivered), which the ' + 'status-code check at open time cannot catch', + ) + + # A settle window past the last blink: any spurious PASSED the blinks produced + # has long since reached the fault_manager and been folded into the debounce + # counter by now, and any SSE frame it would have produced has been pumped. + time.sleep(3.0) + finally: + sse_stop.set() + sse_response.close() + sse_pump.join(timeout=5) + + after = _fault_record(PORT, FAULT_CODE, timeout=30.0) + self.assertIsNotNone( + after, f'{FAULT_CODE} vanished from the store entirely after the blinks') + self.assertIsNone( + after.get('last_passed'), + f'a blink produced a spurious PASSED for {FAULT_CODE} while {TARGET_NODE} was ' + f'still stuck inactive (last_passed={after.get("last_passed")!r}) - the ' + "withheld-clear guard's pending leg did not hold through the absence grace", + ) + self.assertNotEqual( + after.get('status'), 'healed', + f'{FAULT_CODE} healed at healing_threshold=1 while {TARGET_NODE} never left ' + 'its stuck lifecycle state - a debounce counter that should never have moved', + ) + + # And the node is still genuinely stuck, so the outcome above measured something + # real rather than an empty graph. + self.assertIsNotNone( + _poll_watchdog_entity(PORT, TARGET_NODE, 'unconfigured', timeout=15.0), + f'{TARGET_NODE} is no longer reading "unconfigured" at the end of the test - ' + 'the trigger did not survive the blinks, so the assertions above proved ' + 'nothing', + ) + + # Row 6: no fault_cleared frame for FAULT_CODE reached the stream across the + # whole blink sequence - the raw-event twin of the last_passed check above, at + # the one tier that cannot miss a transient frame between two polls. + cleared_frames = [ + json.loads(f['data']) for f in sse_frames + if f.get('event') == 'fault_cleared' and 'data' in f + ] + cleared_frames = [ + p for p in cleared_frames if p.get('fault', {}).get('fault_code') == FAULT_CODE] + self.assertEqual( + cleared_frames, [], + f'a fault_cleared event for {FAULT_CODE} reached /faults/stream during the ' + f'blinks - the withheld-clear guard let a spurious clear through, and unlike ' + f'the last_passed check above this instrument cannot have missed it: {cleared_frames}', + ) + + # Every fault_confirmed/fault_updated frame for FAULT_CODE across the run must + # carry the confirmed severity (ERROR), never the unreadable one (WARN). This is + # not merely improbable given this scenario's timing (the required node never + # stays unreadable the 60 consecutive matched ticks GRAPH_NODE_UNREADABLE needs to + # convert) - it is impossible IN PRINCIPLE for FAULT_CODE specifically, since + # GRAPH_NODE_INACTIVE is now a class-scoped, fixed-severity AggregatedFault member + # (SEVERITY_ERROR always; see the class docstring above and "Two independent + # faults, not one shared record" in the design doc): there is no per-tick choice + # left anywhere in the code that could produce WARN on this fault_code, only on + # its sibling GRAPH_NODE_UNREADABLE's own, separate record. A WARN here would mean + # the two records' content had been cross-wired, not that some tick's timing went + # wrong. + content_frames = [ + json.loads(f['data']) for f in sse_frames + if f.get('event') in ('fault_confirmed', 'fault_updated') and 'data' in f + ] + content_frames = [ + p for p in content_frames if p.get('fault', {}).get('fault_code') == FAULT_CODE] + self.assertTrue( + content_frames, + 'no fault_confirmed/fault_updated frame for {FAULT_CODE} reached the stream at ' + f'all - the SSE instrument saw nothing, so the severity check below would prove ' + f'nothing; frames seen: {sse_frames}', + ) + wrong_severity = [ + p['fault'].get('severity_label') for p in content_frames + if p['fault'].get('severity_label') != 'ERROR' + ] + self.assertEqual( + wrong_severity, [], + f'{FAULT_CODE} carried a severity other than ERROR on at least one tick - it is a ' + f'fixed-severity fault (SEVERITY_ERROR, see AggregatedFault in the design doc), so ' + f'nothing on any tick can make it carry anything else: {wrong_severity}', + ) + + +class TestLifecycleExpectationDepartureKeeps(unittest.TestCase): + """GRAPH_NODE_UNREADABLE SURVIVES an already-raised node leaving the graph. + + The evidence-retention half of the model, at the tier that runs the real + fault_manager: a node whose lifecycle promise was never verified does not become + verified by leaving. A sibling launch to TestLifecycleExpectationUnreadable above, + not an extension of it: that class's test_04 proves the clear-by-READ path by + flipping unreadable_lifecycle_node.cpp's start_answering parameter, and unittest + runs a class's test methods, in one gateway process, in alphabetical order. A test + that permanently kills the fixture (no respawn - the whole point here is a real, + sustained departure, never a bounce back) cannot share that process: it would + either race test_04's assumption that the fixture is still alive, or - if ordered + to run first - leave the fixture dead before test_04 ever gets to set the + parameter. So this launches its own gateway + fault_manager + + unreadable_lifecycle_node.cpp stack, holding a handle to the fixture's own + launch_ros.actions.Node action (``context['target_node']``, the same shape + TestLifecycleExpectationHealingThreshold uses for its own SIGTERM'able node) - + create_demo_nodes() gives no such handle back. + + Proves, in order, against the real stack through GET /api/v1/faults: + + 1. The same starting state TestLifecycleExpectationUnreadable reaches: the fixture + present, matched, and unread (test_01), then GRAPH_NODE_UNREADABLE actually + raised past the 60-tick hold (test_02) - asserted before anything else, or the + rest would not be about an ALREADY-RAISED fault at all. + 2. The fixture process SIGTERM'd, and confirmed GONE from the operator-visible + graph via GET /apps - the same ThreadSafeEntityCache the detector's own + per-tick snapshot reads (see _poll_apps_absent's docstring) - before anything is + asserted about the fault (test_03). Without this gate the fault still being + there could not be told apart from a graph the entity cache had not caught up + with yet. + 3. After a settle window longer than every horizon that could have discarded the + node's evidence, the fault is STILL active, has never once been reported PASSED + (``last_passed``, the instrument that catches even a transient clear), and its + description now says the node has left the graph rather than continuing to + describe a present-but-unread one. GRAPH_NODE_INACTIVE stays silent throughout: + an unread label can never become CONFIRMED content, before or after the kill. + + The integration tier's own proof of this claim, + ``UnreadableNodeAlreadyReportedThatVanishesKeepsItsOwnRecord``, drives the detector + directly against a fake ReportFault service and a hand-built snapshot; it cannot + prove the real discovery layer ever notices a process leaving DDS, or that the real + fault_manager's debounce state machine never receives a PASSED. This closes that gap. + """ + + @classmethod + def setUpClass(cls): + rclpy.init() + + @classmethod + def tearDownClass(cls): + rclpy.shutdown() + + def test_01_present_and_unread_before_anything_else(self): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed global state - the plugin did not ' + 'load, its tick loop never ran, or the bringup grace never elapsed, so no ' + 'assertion below could mean anything', + ) + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=30.0), + 'GET /faults never answered 200 - the gateway never reached the ' + 'fault_manager in this launch, so a wrong raise would have been lost rather ' + 'than caught', + ) + self.assertIsNotNone( + _poll_watchdog_entity(PORT, UNREADABLE_NODE, '', timeout=30.0), + f'the gate never reported {UNREADABLE_NODE} with an empty (never-read) ' + 'lifecycle label - the fixture was not discovered as a managed node, or its ' + 'GetState answered when it must not have, so nothing below would prove ' + 'anything about this clear path', + ) + + def test_02_unreadable_raises_before_the_kill(self): + fault = poll_faults(PORT, FAULT_CODE_UNREADABLE, timeout=UNREADABLE_RAISE_TIMEOUT_SEC) + self.assertIsNotNone( + fault, + f'{FAULT_CODE_UNREADABLE} never raised while {UNREADABLE_NODE} stayed ' + 'unreadable past the 60-tick hold - there is nothing ALREADY RAISED for the ' + 'kill below to clear', + ) + self.assertIn(UNREADABLE_NODE, fault.get('description', '')) + assert_fault_absent_throughout(self, PORT, FAULT_CODE, MUTUAL_EXCLUSION_WINDOW_SEC) + + def test_03_killed_node_confirmed_gone_and_the_fault_survives_it(self, target_node): + old_pid = target_node.process_details['pid'] + os.kill(old_pid, signal.SIGTERM) + + self.assertTrue( + _poll_apps_absent(PORT, UNREADABLE_NODE, timeout=ABSENCE_DEPARTURE_TIMEOUT_SEC), + f'{UNREADABLE_NODE} (pid {old_pid}) is still listed on GET /apps after ' + 'SIGTERM - it never actually left the graph, so nothing below would prove ' + 'anything about a DEPARTED node', + ) + + # Well past the absence grace (3 ticks at this scenario's 100 ms cadence) and past + # the detector's own 60-tick hold, so a mechanism that discarded the node's + # evidence on absence has had every opportunity to do so. + time.sleep(DEPARTURE_SETTLE_SEC) + + record = _fault_record(PORT, FAULT_CODE_UNREADABLE, timeout=30.0) + self.assertIsNotNone( + record, + f'{FAULT_CODE_UNREADABLE} vanished from the store entirely after ' + f'{UNREADABLE_NODE} left the graph', + ) + self.assertIsNone( + record.get('last_passed'), + f'{FAULT_CODE_UNREADABLE} was reported PASSED once {UNREADABLE_NODE} left the ' + f'graph (last_passed={record.get("last_passed")!r}) - the node\'s lifecycle ' + 'promise is no more verified now than it was while the node was present, so ' + 'its departure must not heal the fault', + ) + self.assertIsNotNone( + poll_faults(PORT, FAULT_CODE_UNREADABLE, timeout=ABSENCE_CLEAR_TIMEOUT_SEC), + f'{FAULT_CODE_UNREADABLE} is no longer an active fault once {UNREADABLE_NODE} ' + 'stayed gone - a departure erased the evidence against it', + ) + + # And the description now says the node is gone, rather than continuing to + # describe a graph it left. + self.assertIn( + 'has since left the graph', + _fault_record(PORT, FAULT_CODE_UNREADABLE, timeout=10.0).get('description', ''), + f'{FAULT_CODE_UNREADABLE} still describes {UNREADABLE_NODE} as a present ' + 'node whose state cannot be read, after it left the graph', + ) + + # GRAPH_NODE_INACTIVE stays silent throughout: the node was never read at any point + # in this whole scenario, so it has no path to CONFIRMED content whether it is in + # the graph or not. + assert_fault_absent_throughout(self, PORT, FAULT_CODE, MUTUAL_EXCLUSION_WINDOW_SEC) + + +class TestLifecycleExpectationNotManaged(unittest.TestCase): + """GRAPH_NODE_NOT_MANAGED raises for a required node with no tracked lifecycle at all. + + The sibling of TestLifecycleExpectationUnreadable/TestLifecycleExpectationDepartureKeeps, + proving the OTHER "I cannot measure this" cause the unmeasured clock this slice adds is + blind to: unlike ``unreadable_lifecycle_node.cpp`` (which advertises GetState/ChangeState + but never answers), ``calibration`` (``DEMO_NODE_REGISTRY``'s plain + ``demo_calibration_service``) advertises no lifecycle interface whatsoever, so + ``lifecycle_state_of()`` reads ``nullopt`` for it from the moment it is discovered - no + purpose-built fixture was needed for this cause, unlike UNREADABLE's. + + Proves, in order, against the real stack through GET /api/v1/faults: + + 1. The required node is present and matched, and it carries NO tracked lifecycle at all + (test_01) - via GET /apps rather than the watchdog status route's `lifecycle` field, + since a never-tracked node's own presence is the fact under test here, not a label. + 2. GRAPH_NODE_NOT_MANAGED raises once the 60-tick hold expires, names the node, and + carries SEVERITY_WARN - and neither GRAPH_NODE_INACTIVE nor GRAPH_NODE_UNREADABLE + ever raises for it: a node is content of at most one of the three (test_02). + 3. The node leaving the graph and staying away does NOT clear + GRAPH_NODE_NOT_MANAGED - the same evidence retention the unreadable sibling + gets, proven independently rather than assumed from sharing one clock (test_03). + """ + + @classmethod + def setUpClass(cls): + rclpy.init() + + @classmethod + def tearDownClass(cls): + rclpy.shutdown() + + def test_01_present_and_not_managed_before_anything_else(self): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed global state - the plugin did not ' + 'load, its tick loop never ran, or the bringup grace never elapsed, so no ' + 'assertion below could mean anything', + ) + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=30.0), + 'GET /faults never answered 200 - the gateway never reached the ' + 'fault_manager in this launch, so a wrong raise would have been lost rather ' + 'than caught', + ) + # The fact this whole scenario stands on: the node is present in the graph (a + # plain service node, no lifecycle interface) - proven via the same ThreadSafeEntityCache + # the detector's own per-tick snapshot reads (see _poll_apps_present's docstring), + # not the watchdog status route's `lifecycle` field, since "never tracked" is a + # statement about ABSENCE of tracking, which GET /apps + the raise below prove + # together more directly than a nullable JSON field would. + self.assertTrue( + _poll_apps_present(PORT, NOT_MANAGED_NODE, timeout=30.0), + f'{NOT_MANAGED_NODE} never appeared on GET /apps - the fixture never came up, ' + 'so nothing below would prove anything about NOT-MANAGED', + ) + + def test_02_not_managed_raises_naming_the_node_at_warn_severity(self): + fault = poll_faults(PORT, FAULT_CODE_NOT_MANAGED, timeout=NOT_MANAGED_RAISE_TIMEOUT_SEC) + self.assertIsNotNone( + fault, + f'{FAULT_CODE_NOT_MANAGED} never raised while {NOT_MANAGED_NODE} stayed ' + 'not-managed past the 60-tick hold', + ) + self.assertIn(NOT_MANAGED_NODE, fault.get('description', '')) + self.assertEqual( + fault.get('severity_label'), 'WARN', + f'{FAULT_CODE_NOT_MANAGED} must carry SEVERITY_WARN on the stored record ' + f'(an UNVERIFIED promise, not a confirmed violation), got ' + f'{fault.get("severity_label")!r}', + ) + + # A node is content of at most one of the three codes, never more than one - proven + # across a window with the channel checked on every poll, not by a single poll that + # timed out silently. + assert_fault_absent_throughout(self, PORT, FAULT_CODE, MUTUAL_EXCLUSION_WINDOW_SEC) + assert_fault_absent_throughout( + self, PORT, FAULT_CODE_UNREADABLE, MUTUAL_EXCLUSION_WINDOW_SEC) + + self.assertIsNotNone( + poll_entity_faults(PORT, 'apps/graph_watchdog', FAULT_CODE_NOT_MANAGED, timeout=30.0), + f'{FAULT_CODE_NOT_MANAGED} is not reachable at /apps/graph_watchdog/faults - ' + 'the entity the plugin publishes does not own the fault it raises', + ) + + def test_03_departed_node_confirmed_gone_and_the_fault_survives_it(self, target_node): + fault = poll_faults(PORT, FAULT_CODE_NOT_MANAGED, timeout=NOT_MANAGED_RAISE_TIMEOUT_SEC) + self.assertIsNotNone( + fault, + f'{FAULT_CODE_NOT_MANAGED} is not active at the start of the departure leg, so ' + 'there is nothing here for the departure below to preserve', + ) + + old_pid = target_node.process_details['pid'] + os.kill(old_pid, signal.SIGTERM) + + self.assertTrue( + _poll_apps_absent(PORT, NOT_MANAGED_NODE, timeout=NOT_MANAGED_DEPARTURE_TIMEOUT_SEC), + f'{NOT_MANAGED_NODE} (pid {old_pid}) is still listed on GET /apps after ' + 'SIGTERM - it never actually left the graph, so nothing below would prove ' + 'anything about a DEPARTED node', + ) + time.sleep(DEPARTURE_SETTLE_SEC) + + record = _fault_record(PORT, FAULT_CODE_NOT_MANAGED, timeout=30.0) + self.assertIsNotNone( + record, + f'{FAULT_CODE_NOT_MANAGED} vanished from the store entirely after ' + f'{NOT_MANAGED_NODE} left the graph', + ) + self.assertIsNone( + record.get('last_passed'), + f'{FAULT_CODE_NOT_MANAGED} was reported PASSED once {NOT_MANAGED_NODE} left ' + f'the graph (last_passed={record.get("last_passed")!r}) - the node was ' + 'required to be active, never was a managed lifecycle node at all, and is now ' + 'gone, none of which is health', + ) + self.assertIsNotNone( + poll_faults(PORT, FAULT_CODE_NOT_MANAGED, timeout=NOT_MANAGED_CLEAR_TIMEOUT_SEC), + f'{FAULT_CODE_NOT_MANAGED} is no longer an active fault once {NOT_MANAGED_NODE} ' + 'stayed gone - a departure erased the evidence against it', + ) + self.assertIn( + 'has since left the graph', + _fault_record(PORT, FAULT_CODE_NOT_MANAGED, timeout=10.0).get('description', ''), + f'{FAULT_CODE_NOT_MANAGED} still describes {NOT_MANAGED_NODE} as a present ' + 'node, after it left the graph', + ) + assert_fault_absent_throughout(self, PORT, FAULT_CODE, MUTUAL_EXCLUSION_WINDOW_SEC) + + +class TestLifecycleExpectationRestartLoop(unittest.TestCase): + """A required node in a CRASH LOOP is reported, not silent. + + The scenario the whole evidence-retention model exists for, on the real stack. A node + that keeps dying and coming back touches absence periodically by construction, so any + horizon that discards its accumulated evidence when it is absent makes exactly that + node permanently invisible - and a crash-looping required node is the single case this + detector most needs to catch. + + The loop is driven by real SIGTERMs against a real, respawning process, not by a + hand-built snapshot: the node is left alive for RESTART_LOOP_UPTIME_SEC (at most ~25 + ticks at this scenario's 100 ms cadence, comfortably under the 60 consecutive + matched ticks kUnmeasuredHoldTicks would otherwise need), then killed. Each absence + is proven, not assumed: GET /apps - the same ThreadSafeEntityCache the detector's own + per-tick snapshot reads - is polled until the node is gone and again until it is back, + and the measured gap must exceed the tracker's 3-tick absence grace, on top of + launch's own enforced respawn_delay floor. So every cycle genuinely crosses the + horizon under test, and a detector that reset its clock there would stay silent for + the whole window however many cycles it ran. + + `calibration` (a plain service node with no lifecycle interface at all) supplies the + NOT-MANAGED cause, so no purpose-built fixture is needed and no blocking GetState + round trip is paid per tick. + """ + + @classmethod + def setUpClass(cls): + rclpy.init() + + @classmethod + def tearDownClass(cls): + rclpy.shutdown() + + def _kill_and_wait_for_the_return(self, node_action, cycle): + """SIGTERM the node, prove it left the graph past the absence grace, wait it back.""" + old_pid = node_action.process_details['pid'] + os.kill(old_pid, signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, NOT_MANAGED_NODE, timeout=30.0, interval=0.05), + f'cycle {cycle}: {NOT_MANAGED_NODE} (pid {old_pid}) was never observed absent ' + 'from GET /apps after the SIGTERM - this cycle never crossed the absence ' + 'horizon it exists to cross', + ) + absence_observed = time.monotonic() + self.assertTrue( + _poll_apps_present(PORT, NOT_MANAGED_NODE, timeout=30.0, interval=0.05), + f'cycle {cycle}: {NOT_MANAGED_NODE} never came back after the SIGTERM - launch ' + 'did not respawn it, so this is a single departure, not a restart loop', + ) + # Measured between two observations of the SAME cache the detector reads, so this + # is a real lower bound on how long the detector saw the node absent for. + absent_for = time.monotonic() - absence_observed + grace_budget_sec = (HEALING_ABSENCE_GRACE_TICKS * RESTART_LOOP_TICK_INTERVAL_MS) / 1000.0 + self.assertGreater( + absent_for, grace_budget_sec, + f'cycle {cycle}: {NOT_MANAGED_NODE} was only observed absent for ' + f'{absent_for:.2f}s, inside the {grace_budget_sec:.1f}s absence grace - this ' + 'cycle was a blink, which was always tolerated, so it does not exercise the ' + 'horizon past which evidence used to be discarded', + ) + + def test_crash_looping_node_is_reported(self, target_node): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed global state - the plugin did not ' + 'load, its tick loop never ran, or the bringup grace never elapsed, so no ' + 'assertion below could mean anything', + ) + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=30.0), + 'GET /faults never answered 200 - the gateway never reached the ' + 'fault_manager in this launch, so a missing fault would prove nothing', + ) + self.assertTrue( + _poll_apps_present(PORT, NOT_MANAGED_NODE, timeout=30.0), + f'{NOT_MANAGED_NODE} never appeared on GET /apps - the fixture never came up, ' + 'so there is no node here to crash-loop', + ) + + # Keep killing until the fault has appeared AND the node has genuinely looped: + # accepting the first sighting would let a single restart stand in for a loop, and + # would make the cycle count depend on how fast this machine is. Written this way + # the loop also proves the fault SURVIVES the restarts that follow it, not merely + # that it appeared once. + deadline = time.monotonic() + RESTART_LOOP_WINDOW_SEC + cycles = 0 + fault = None + while time.monotonic() < deadline: + time.sleep(RESTART_LOOP_UPTIME_SEC) + fault = poll_faults(PORT, FAULT_CODE_NOT_MANAGED, timeout=0.1, interval=0.1) + if fault is not None and cycles >= RESTART_LOOP_MIN_CYCLES: + break + cycles += 1 + self._kill_and_wait_for_the_return(target_node, cycles) + + self.assertGreaterEqual( + cycles, RESTART_LOOP_MIN_CYCLES, + f'the node was only restarted {cycles} time(s) - too few for this to be a crash ' + 'LOOP rather than a single departure, so the result below does not discriminate', + ) + self.assertIsNotNone( + fault, + f'{FAULT_CODE_NOT_MANAGED} never raised for a required node that crash-looped ' + f'for {RESTART_LOOP_WINDOW_SEC}s across {cycles} restarts - every absence ' + 'discarded the evidence it had accumulated while present, so the node this ' + 'detector most exists to catch stays silent forever', + ) + self.assertIn(NOT_MANAGED_NODE, fault.get('description', '')) + self.assertEqual( + fault.get('severity_label'), 'WARN', + f'{FAULT_CODE_NOT_MANAGED} must carry SEVERITY_WARN on the stored record, got ' + f'{fault.get("severity_label")!r}', + ) + + +class TestLifecycleExpectationCapPressure(unittest.TestCase): + """A FULL tracked-node cap must never turn into unreported health. + + The tracker keeps state for at most `tracked_node_cap` nodes. This launch sets it to + ONE against TWO required nodes, so one of them is refused on every single tick - + reachable at this tier only because the cap is a config key; against the compile-time + 512 it would need 513 real lifecycle nodes. + + Three separate claims, in the order the run reaches them: + + 1. **The refusal is visible** (test_01). A required node is going unchecked, which is + exactly the state a detector for silent faults must not be silent about. The + gateway log is not readable from here; the detector's own status block on + ``GET /x-medkit-watchdog`` is. + 2. **A refusal WITHHOLDS the GRAPH_NODE_INACTIVE clear** (test_02). The tracked node's + lifecycle services are dropped, so its unmeasured clock matures, ownership passes to + GRAPH_NODE_NOT_MANAGED and GRAPH_NODE_INACTIVE's own content goes empty - with + nothing pending, which is precisely when the level-triggered emitter would clear. It + must not: the detector declined to check the other required node, so it cannot assert + that every required node is healthy. The instrument is ``last_passed``, which the + fault_manager sets on the first PASSED a fault ever receives and never unsets, so a + single transient clear is caught even though the fault later re-raises. + 3. **A DEPARTED entry never crowds out a PRESENT one** (test_03). The tracked node is + killed. Its entry can never become idle again - becoming idle needs a real + measurement of a node that is gone - so under a cap full of the dead the present, + genuinely broken node would be refused forever and the detector would report health + it had refused to check. The departed entry is collapsed into a count instead (which + keeps its own code's content non-empty, so its departure still heals nothing) and the + present node is admitted and reported. + """ + + @classmethod + def setUpClass(cls): + rclpy.init() + cls._client_node = Node('lifecycle_expectation_cap_e2e_client') + + @classmethod + def tearDownClass(cls): + cls._client_node.destroy_node() + rclpy.shutdown() + + def test_01_the_refused_node_is_reported_as_saturation(self): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed global state - the plugin did not ' + 'load, its tick loop never ran, or the bringup grace never elapsed, so no ' + 'assertion below could mean anything', + ) + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=30.0), + 'GET /faults never answered 200 - the gateway never reached the fault_manager ' + 'in this launch', + ) + # Both required nodes must actually be present and matched, or "one of them was + # refused" would just be "one of them never came up". + for node_id in (CAP_TRACKED_NODE, CAP_REFUSED_NODE): + self.assertTrue( + _poll_apps_present(PORT, node_id, timeout=30.0), + f'{node_id} never appeared on GET /apps - with only one required node ' + 'present the cap of one is not full and nothing is refused', + ) + self.assertIsNotNone( + _poll_watchdog_entity(PORT, CAP_TRACKED_NODE, 'unconfigured', timeout=30.0), + f'{CAP_TRACKED_NODE} never read "unconfigured" - the node that must win the ' + 'single slot was never measured as a violation at all', + ) + + fault = poll_faults(PORT, FAULT_CODE, timeout=60.0) + self.assertIsNotNone( + fault, f'{FAULT_CODE} never raised for {CAP_TRACKED_NODE}, which holds the one slot') + self.assertIn(CAP_TRACKED_NODE, fault.get('description', '')) + + # The refusal itself, on the operator-visible surface. + self.assertTrue( + poll_detector_status(PORT, DETECTOR_ID, 'tracking_saturated', True, timeout=30.0), + 'GET /x-medkit-watchdog never reported the lifecycle_expectation tracked-node ' + f'cap as saturated while {CAP_REFUSED_NODE} was present, required and refused - ' + 'a required node is going unchecked and nothing an operator can read says so', + ) + block = watchdog_detector_status(PORT, DETECTOR_ID) + self.assertIsNotNone(block, 'the detector status block vanished between two reads') + self.assertEqual( + block.get('tracked_node_cap'), CAP_NODE_CAP, + 'the status route reports a different tracked_node_cap than the one this launch ' + f'configured ({CAP_NODE_CAP}) - the key did not reach the detector: {block}') + + def test_02_a_refused_node_withholds_the_clear(self): + before = _fault_record(PORT, FAULT_CODE, timeout=30.0) + self.assertIsNotNone( + before, f'{FAULT_CODE} is not in the store, so there is nothing a clear could heal') + self.assertIsNone( + before.get('last_passed'), + f'{FAULT_CODE} already had a PASSED on record before this test ' + f'(last_passed={before.get("last_passed")!r}), so nothing below could be ' + 'attributed to the withhold', + ) + + # Drop the tracked node's lifecycle services. It stays present and stays non-idle + # (an unmeasured clock is evidence), so it keeps the single slot - but once that + # clock matures, ownership passes to GRAPH_NODE_NOT_MANAGED and GRAPH_NODE_INACTIVE + # has no content and nothing pending. That is the tick the clear would flow on. + self.assertTrue( + _set_bool_parameter( + type(self)._client_node, f'/{CAP_TRACKED_NODE}/set_parameters', + 'drop_services', True, timeout=30.0), + f'setting drop_services:=true on /{CAP_TRACKED_NODE}/set_parameters never ' + 'succeeded, so the state this test needs was never reached', + ) + self.assertIsNotNone( + poll_faults(PORT, FAULT_CODE_NOT_MANAGED, timeout=CAP_NOT_MANAGED_RAISE_TIMEOUT_SEC), + f'{FAULT_CODE_NOT_MANAGED} never raised for {CAP_TRACKED_NODE} after its ' + 'lifecycle services were dropped - its unmeasured clock never matured, so ' + f'{FAULT_CODE} still has content and the withhold below would prove nothing', + ) + time.sleep(CAP_WITHHOLD_WINDOW_SEC) + after = _fault_record(PORT, FAULT_CODE, timeout=30.0) + self.assertIsNotNone(after, f'{FAULT_CODE} vanished from the store entirely') + self.assertIsNone( + after.get('last_passed'), + f'{FAULT_CODE} was reported PASSED while {CAP_REFUSED_NODE} was present, ' + f'required and REFUSED by the tracked-node cap (last_passed=' + f'{after.get("last_passed")!r}) - the detector asserted that every required ' + 'node is healthy after declining to check one of them', + ) + self.assertNotEqual( + after.get('status'), 'healed', + f'{FAULT_CODE} healed while a required node was never checked') + + # And the window really was about a REFUSAL: the tracked node is not idle (an + # unmeasured clock is evidence), so it kept the single slot the whole way through and + # the other required node was never admitted behind our back. + self.assertTrue( + poll_detector_status(PORT, DETECTOR_ID, 'tracking_saturated', True, timeout=15.0), + 'the cap stopped reporting itself saturated once the tracked node went ' + 'not-managed - the refused node was admitted, so the window above was not about ' + 'a withheld clear at all', + ) + + def test_03_a_departed_entry_is_collapsed_so_the_present_node_is_checked(self, tracked_node): + old_pid = tracked_node.process_details['pid'] + os.kill(old_pid, signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, CAP_TRACKED_NODE, timeout=30.0), + f'{CAP_TRACKED_NODE} (pid {old_pid}) is still listed on GET /apps after SIGTERM ' + '- it never left the graph, so its entry was never a DEPARTED one', + ) + self.assertTrue( + _poll_apps_present(PORT, CAP_REFUSED_NODE, timeout=30.0), + f'{CAP_REFUSED_NODE} is not on GET /apps - the node that must now be admitted ' + 'is not even present', + ) + + # The whole hypothesis: the slot goes to the PRESENT node. + self.assertTrue( + _poll_fault_description_contains( + PORT, FAULT_CODE, CAP_REFUSED_NODE, timeout=CAP_ADMISSION_TIMEOUT_SEC), + f'{FAULT_CODE} never named {CAP_REFUSED_NODE} once {CAP_TRACKED_NODE} left the ' + 'graph - a present, genuinely broken required node is being refused by an entry ' + 'for a node that is gone and can never become idle again, so the detector ' + 'reports health it has refused to check', + ) + # And the refusal is over, so the latch that reports it has re-armed for a later one. + self.assertTrue( + poll_detector_status(PORT, DETECTOR_ID, 'tracking_saturated', False, timeout=30.0), + 'the cap still reports itself saturated after the departed entry was collapsed ' + 'and the present node admitted - a later, real saturation would be indistinguishable', + ) + # The departed node's own fault is not healed by its departure: its evidence lives on + # as a count, which is what keeps that code's content non-empty. + record = _fault_record(PORT, FAULT_CODE_NOT_MANAGED, timeout=30.0) + self.assertIsNotNone( + record, f'{FAULT_CODE_NOT_MANAGED} vanished from the store after the departure') + self.assertIsNone( + record.get('last_passed'), + f'{FAULT_CODE_NOT_MANAGED} was reported PASSED once {CAP_TRACKED_NODE} left ' + f'(last_passed={record.get("last_passed")!r}) - collapsing its entry to free a ' + 'slot threw its evidence away instead of keeping it as a count', + ) + # Nothing anywhere in this run ever cleared GRAPH_NODE_INACTIVE. + final = _fault_record(PORT, FAULT_CODE, timeout=30.0) + self.assertIsNone( + final.get('last_passed'), + f'{FAULT_CODE} was reported PASSED at some point in this run ' + f'(last_passed={final.get("last_passed")!r}) - a required node was refused or ' + 'stuck for the whole of it', + ) + + +class TestLifecycleExpectationUnsettledDeparture(unittest.TestCase): + """One bad sweep must not turn a healthy departure into a permanent fault. + + A present, healthy, MANAGED node reads "not a managed lifecycle node" for a tick or two + whenever its ``get_state`` path is missing from one sweep - ``LifecycleWatcher::update()`` + drops a tracked id whose path is absent from the current sweep, and ``discover_apps()`` + can yield an app with no services when a sweep races service enumeration. Absence + continues whatever the node was last observed as, so if that tick happens to be the last + one before a clean shutdown, a healthy departure matures into a permanent + GRAPH_NODE_NOT_MANAGED about a node that left in good health. + + Two legs of the SAME fixture, differing only in how long the dropped state is held, so + the pair is discriminating rather than a bare silence proof: + + - ``blink_departer`` drops its lifecycle services and is killed immediately, well inside + the settling budget. Nothing may ever be reported about it. + - ``settled_departer`` drops its lifecycle services and holds that state long past the + settling budget before being killed. It IS genuinely not managed when it leaves, and + must still be reported - the positive control that proves the fixture, the discovery + path and the detector all work, so the blinker's silence is a measurement rather than + a stack that never came up. + + The blink leg's own budget is MEASURED, not assumed: the elapsed time from the drop to + the node being gone from ``GET /apps`` - the same ThreadSafeEntityCache the detector's + per-tick snapshot reads - must be under the settling budget, or the leg is silently + exercising the settled case instead and proves the opposite of what it claims. + """ + + @classmethod + def setUpClass(cls): + rclpy.init() + cls._client_node = Node('lifecycle_expectation_unsettled_e2e_client') + + @classmethod + def tearDownClass(cls): + cls._client_node.destroy_node() + rclpy.shutdown() + + def _drop_services(self, node_id): + self.assertTrue( + _set_bool_parameter( + type(self)._client_node, f'/{node_id}/set_parameters', + 'drop_services', True, timeout=30.0), + f'setting drop_services:=true on /{node_id}/set_parameters never succeeded - ' + 'the node never stopped looking like a managed lifecycle node', + ) + + def test_01_both_required_nodes_are_present_and_healthy(self): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed global state, so no assertion below ' + 'could mean anything', + ) + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=30.0), + 'GET /faults never answered 200 - the gateway never reached the fault_manager ' + 'in this launch, so an absent fault would prove nothing', + ) + for node_id in (UNSETTLED_BLINK_NODE, UNSETTLED_SETTLED_NODE): + self.assertIsNotNone( + _poll_watchdog_entity(PORT, node_id, 'active', timeout=30.0), + f'{node_id} never read lifecycle "active" - it was not discovered as a ' + 'managed node, so it cannot be a HEALTHY managed node whose services then ' + 'disappear', + ) + + def test_02_a_one_sweep_blink_before_a_clean_exit_is_measured_as_such(self, blink_node): + self._drop_services(UNSETTLED_BLINK_NODE) + dropped_at = time.monotonic() + # Prove the detector's own view actually went not-managed for at least one tick: + # the watchdog route reports a null lifecycle exactly when the watcher no longer + # tracks the id, which is what the tracker classifies as kNotManaged. + self.assertIsNotNone( + _poll_watchdog_entity(PORT, UNSETTLED_BLINK_NODE, None, timeout=15.0), + f'{UNSETTLED_BLINK_NODE} never lost its tracked lifecycle state after its ' + 'services were dropped - this leg never produced the unmeasured observation it ' + 'is about', + ) + os.kill(blink_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, UNSETTLED_BLINK_NODE, timeout=30.0, interval=0.05), + f'{UNSETTLED_BLINK_NODE} never left GET /apps after SIGTERM') + unmanaged_for = time.monotonic() - dropped_at + settle_budget_sec = (UNSETTLED_SETTLE_TICKS * UNSETTLED_TICK_INTERVAL_MS) / 1000.0 + self.assertLess( + unmanaged_for, settle_budget_sec, + f'{UNSETTLED_BLINK_NODE} was observable as not-managed for {unmanaged_for:.2f}s, ' + f'past the {settle_budget_sec:.1f}s settling budget - this leg exercised a ' + 'SETTLED not-managed departure, which must be reported, so its silence below ' + 'would be a failure rather than the property under test', + ) + + def test_03_a_settled_not_managed_departure_is_still_reported(self, settled_node): + self._drop_services(UNSETTLED_SETTLED_NODE) + self.assertIsNotNone( + _poll_watchdog_entity(PORT, UNSETTLED_SETTLED_NODE, None, timeout=15.0), + f'{UNSETTLED_SETTLED_NODE} never lost its tracked lifecycle state after its ' + 'services were dropped', + ) + # Held well past the settling budget, so the observation is corroborated before the + # node leaves - the difference, and the only difference, from the blink leg above. + time.sleep((UNSETTLED_SETTLED_HOLD_TICKS * UNSETTLED_TICK_INTERVAL_MS) / 1000.0) + os.kill(settled_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, UNSETTLED_SETTLED_NODE, timeout=30.0), + f'{UNSETTLED_SETTLED_NODE} never left GET /apps after SIGTERM') + + fault = poll_faults(PORT, FAULT_CODE_NOT_MANAGED, timeout=UNSETTLED_RAISE_TIMEOUT_SEC) + self.assertIsNotNone( + fault, + f'{FAULT_CODE_NOT_MANAGED} never raised for {UNSETTLED_SETTLED_NODE}, which was ' + 'genuinely not a managed lifecycle node for many consecutive ticks before it ' + 'left the graph - the departure carve-out swallowed a real finding', + ) + self.assertIn(UNSETTLED_SETTLED_NODE, fault.get('description', '')) + + def test_04_the_blinking_node_is_never_reported_at_all(self): + # By now the settled leg has been reported, which means every horizon that could + # have matured the blinker's own one-sweep reading has been passed on this same + # stack: it dropped its services first and left the graph first. + for code in (FAULT_CODE_NOT_MANAGED, FAULT_CODE_UNREADABLE, FAULT_CODE): + record = _fault_record(PORT, code, timeout=5.0) + description = (record or {}).get('description', '') + self.assertNotIn( + UNSETTLED_BLINK_NODE, description, + f'{code} names {UNSETTLED_BLINK_NODE}, a healthy managed node whose ' + 'lifecycle services were missing from one sweep before it shut down ' + f'cleanly: {description!r}', + ) + + +class TestLifecycleExpectationWideGrace(unittest.TestCase): + """A `grace` past the accepted maximum must not silence the detector for days. + + ``grace`` used to be accepted all the way to ``INT_MAX - 1``. A node under such a value + is never CONFIRMED - its streak advances one per tick and would need billions of them - + and never cleared either: a streak above zero puts the node in the tracker's pending set, + the withheld-clear guard returns early on every tick, and GRAPH_NODE_INACTIVE can neither + raise nor heal for ANY node for the life of the process. That is not a wide tolerance, it + is an off switch with no warning attached. + + Accepting a value that large is the defect, so the fix is a sane accepted maximum: this + launch configures the old maximum and the detector must refuse it and keep the documented + default, which the run then measures the ordinary way - by the fault actually appearing. + """ + + @classmethod + def setUpClass(cls): + rclpy.init() + + @classmethod + def tearDownClass(cls): + rclpy.shutdown() + + def test_01_an_out_of_range_grace_is_refused_and_the_default_applies(self): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed global state, so a missing fault would ' + 'prove nothing', + ) + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=30.0), + 'GET /faults never answered 200 - the gateway never reached the fault_manager, ' + 'so a missing fault would prove nothing', + ) + self.assertIsNotNone( + _poll_watchdog_entity(PORT, TARGET_NODE, 'unconfigured', timeout=30.0), + f'{TARGET_NODE} never read "unconfigured" - the violation this test needs was ' + 'never present', + ) + + fault = poll_faults(PORT, FAULT_CODE, timeout=WIDE_GRACE_RAISE_TIMEOUT_SEC) + self.assertIsNotNone( + fault, + f'{FAULT_CODE} never raised for a node stuck inactive under grace=' + f'{WIDE_GRACE_VALUE}. The value was accepted, so the streak has to climb to it ' + 'before anything is reported and the node sits in the tracker pending set ' + 'meanwhile, which also withholds the clear - GRAPH_NODE_INACTIVE can neither ' + 'raise nor heal for any node for days', + ) + self.assertIn(TARGET_NODE, fault.get('description', '')) + + def test_02_the_fault_survives_the_node_leaving(self, target_node): + os.kill(target_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=30.0), + f'{TARGET_NODE} never left GET /apps after SIGTERM') + self.assertTrue( + _poll_fault_description_contains( + PORT, FAULT_CODE, 'has since left the graph', timeout=30.0), + f'{FAULT_CODE} still describes {TARGET_NODE} as a present node after it left ' + 'the graph', + ) + record = _fault_record(PORT, FAULT_CODE, timeout=10.0) + self.assertIsNone( + record.get('last_passed'), + f'{FAULT_CODE} was reported PASSED once {TARGET_NODE} left the graph ' + f'(last_passed={record.get("last_passed")!r}) - within one gateway lifetime a ' + 'departure never heals a fault', + ) + + +class TestLifecycleExpectationRestartDeparted(unittest.TestCase): + """Where "a departure never heals a fault" actually ends: a gateway restart. + + This test does not fix anything - it RECORDS a boundary, so it is a decision on file + rather than a surprise. Inside one gateway process a node that leaves while violating + keeps its fault raised for as long as the process lives. Across a restart it does not, + and nothing else in this suite says so. + + Why it cannot: after a restart the tracker is empty and the departed node is not in the + graph, so its ``require_active`` entry matches nothing. An entry that matches nothing is + indistinguishable from a misspelt one - the only component that knows the difference is + the fault store, which this detector does not read at startup - and the hold for a + never-matched entry is deliberately bounded, because a typo must not block healing + forever. Once it expires there is no content and nothing pending, so the level-triggered + clear flows and the record heals without any measurement having been taken. + + Re-seeding the tracker from the fault store at startup would change this. That is a real + feature and deliberately not what this test asks for; the test asks only that the + boundary stop being invisible. + """ + + @classmethod + def setUpClass(cls): + rclpy.init() + + @classmethod + def tearDownClass(cls): + rclpy.shutdown() + + def test_01_the_fault_raises_and_survives_the_node_leaving(self, target_node): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=60.0), + 'graph_watchdog never reported an armed global state') + self.assertIsNotNone( + _poll_watchdog_entity(PORT, TARGET_NODE, 'unconfigured', timeout=30.0), + f'{TARGET_NODE} never read "unconfigured"') + self.assertIsNotNone( + poll_faults(PORT, FAULT_CODE, timeout=60.0), + f'{FAULT_CODE} never raised for the stuck {TARGET_NODE}') + + os.kill(target_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=30.0), + f'{TARGET_NODE} never left GET /apps after SIGTERM') + time.sleep(max(4.0, (GRACE + 8) * RESTART_TICK_INTERVAL_MS / 1000.0)) + record = _fault_record(PORT, FAULT_CODE, timeout=30.0) + self.assertIsNotNone(record, f'{FAULT_CODE} vanished from the store') + self.assertIsNone( + record.get('last_passed'), + f'{FAULT_CODE} was reported PASSED while the gateway that measured ' + f'{TARGET_NODE} was still running (last_passed={record.get("last_passed")!r}) - ' + 'within one gateway lifetime a departure never heals a fault', + ) + + def test_02_a_gateway_restart_re_baselines_and_the_record_heals(self, gateway_node): + old_pid = gateway_node.process_details['pid'] + os.kill(old_pid, signal.SIGTERM) + self.assertTrue( + _wait_until_port_is_down(PORT, timeout=60.0), + f'the gateway (pid {old_pid}) kept answering after SIGTERM - nothing restarted') + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=90.0), + 'the gateway never came back armed after the restart') + self.assertNotEqual( + gateway_node.process_details['pid'], old_pid, + 'the gateway process id did not change, so this test never restarted anything') + + # The recorded boundary: the restarted detector has no measurement of the departed + # node and cannot tell its entry from a typo, so the bounded never-matched hold + # expires and the clear flows. This is the documented scope of "a departure never + # heals a fault" - within a gateway lifetime. + self.assertTrue( + poll_cleared(PORT, FAULT_CODE, timeout=RESTART_HEAL_TIMEOUT_SEC), + f'{FAULT_CODE} is still an active fault long past the never-matched hold after ' + 'a gateway restart. That is not what the code does today, so either the hold ' + 'stopped being bounded or something now re-seeds the tracker at startup - ' + 'either way the documented scope of the promise needs rewriting, not this test', + ) + record = _fault_record(PORT, FAULT_CODE, timeout=30.0) + self.assertIsNotNone(record, f'{FAULT_CODE} vanished from the store entirely') + self.assertIsNotNone( + record.get('last_passed'), + f'{FAULT_CODE} left the active list without ever being reported PASSED - it was ' + 'not the detector that healed it, so this test is no longer measuring the ' + 'boundary it is named for', + ) + + +# Each CTest target launches this file with one scenario, so only that scenario's case +# may run. Removing the others from the module (rather than skipping them) means each +# run reports exactly one case, and a missing result is a real failure rather than an +# expected line of output - see test_config_plumbing_e2e.test.py's identical rationale. +_SCENARIO_CASES = { + 'main': 'TestLifecycleExpectationMain', + 'default_config': 'TestLifecycleExpectationDefaultConfig', + 'negative_control': 'TestLifecycleExpectationNegativeControl', + 'unreadable': 'TestLifecycleExpectationUnreadable', + 'healing_threshold': 'TestLifecycleExpectationHealingThreshold', + 'departure_keeps': 'TestLifecycleExpectationDepartureKeeps', + 'not_managed': 'TestLifecycleExpectationNotManaged', + 'restart_loop': 'TestLifecycleExpectationRestartLoop', + 'cap_pressure': 'TestLifecycleExpectationCapPressure', + 'unsettled_departure': 'TestLifecycleExpectationUnsettledDeparture', + 'wide_grace': 'TestLifecycleExpectationWideGrace', + 'restart_departed': 'TestLifecycleExpectationRestartDeparted', +} +if SCENARIO not in _SCENARIO_CASES: + raise RuntimeError( + f'WATCHDOG_E2E_SCENARIO={SCENARIO!r} is not one of {sorted(_SCENARIO_CASES)}; ' + 'the CTest target and this file disagree about which scenarios exist') +for _scenario, _case_name in _SCENARIO_CASES.items(): + if _scenario != SCENARIO: + del globals()[_case_name] + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + """Verify the gateway/fault_manager/demo stack exits cleanly.""" + + def test_exit_codes(self, proc_info): + for info in proc_info: + self.assertIn( + info.returncode, + ALLOWED_EXIT_CODES, + f'Process {info.process_name} exited with {info.returncode}', + ) diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_drift_e2e.test.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_drift_e2e.test.py index cd527d003..31e91ca1b 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_drift_e2e.test.py +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_param_drift_e2e.test.py @@ -49,9 +49,11 @@ # I100 as well as E402: `harness` is only importable because of the sys.path line above, so this # import cannot be moved up to where the alphabetical order would put it. from harness import ( # noqa: E402, I100 + assert_fault_absent_throughout, create_watchdog_test_launch, poll_cleared, poll_faults, + wait_until_faults_endpoint_live, wait_until_watchdog_armed, ) @@ -149,13 +151,23 @@ def test_01_runtime_change_raises_and_names_the_parameter(self): ) # Give the detector time to capture the baseline BEFORE changing anything. Capturing - # must be silent, so a fault appearing here at all would mean the detector reports its - # own first read as drift. - self.assertIsNone( - poll_faults(PORT, FAULT_CODE, timeout=SILENT_CAPTURE_SEC), - 'GRAPH_PARAM_DRIFT was raised before any parameter changed, so baseline capture is ' - 'not silent', + # must be silent, so a fault appearing here at all would mean the detector reports + # its own first read as drift. assert_fault_absent_throughout, not + # assertIsNone(poll_faults(...)): the latter swallows every non-200 response and + # transport error into the same None a genuine absence produces, so a /faults that + # died partway through this window would still pass - assert_fault_absent_throughout + # instead fails naming which poll could not even ask. + # First contact with GET /faults happens HERE, and the window below is strict: one + # transport error fails it. On the distro running default FastDDS that first call also + # pays service discovery to the fault_manager, which can outlast the window's own 5 s + # per-poll timeout - so prove the channel is up first, with a budget that tolerates + # discovery, and let the window police only what it is for. + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=60.0), + 'GET /faults never answered 200 - the fault surface is not up, so a silence ' + 'assertion below could not tell "no such fault" from "could not ask"', ) + assert_fault_absent_throughout(self, PORT, FAULT_CODE, SILENT_CAPTURE_SEC) self._node.set_parameters( [rclpy.parameter.Parameter(PARAM_NAME, rclpy.Parameter.Type.DOUBLE, DRIFTED_VALUE)]) diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_graph_watchdog_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_graph_watchdog_plugin.cpp index 8436a1ae8..702322dd8 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_graph_watchdog_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_graph_watchdog_plugin.cpp @@ -168,39 +168,44 @@ class FakeContext : public ros2_medkit_gateway::RosPluginContext { rclcpp::Node * node() const override { return node_; } - std::optional get_entity(const std::string &) const override { + std::optional get_entity(const std::string & /*id*/) const override { return std::nullopt; } - std::vector get_child_apps(const std::string &) const override { + std::vector + get_child_apps(const std::string & /*component_id*/) const override { return {}; } - nlohmann::json list_entity_faults(const std::string &) const override { + nlohmann::json list_entity_faults(const std::string & /*entity_id*/) const override { return nlohmann::json::array(); } std::optional - validate_entity_for_route(const ros2_medkit_gateway::PluginRequest &, ros2_medkit_gateway::PluginResponse &, - const std::string &) const override { + validate_entity_for_route(const ros2_medkit_gateway::PluginRequest & /*req*/, + ros2_medkit_gateway::PluginResponse & /*res*/, + const std::string & /*entity_id*/) const override { return std::nullopt; } - void register_capability(ros2_medkit_gateway::SovdEntityType, const std::string &) override { + void register_capability(ros2_medkit_gateway::SovdEntityType /*entity_type*/, + const std::string & /*capability_name*/) override { } - void register_entity_capability(const std::string &, const std::string &) override { + void register_entity_capability(const std::string & /*entity_id*/, const std::string & /*capability_name*/) override { } - std::vector get_type_capabilities(ros2_medkit_gateway::SovdEntityType) const override { + std::vector get_type_capabilities(ros2_medkit_gateway::SovdEntityType /*entity_type*/) const override { return {}; } - std::vector get_entity_capabilities(const std::string &) const override { + std::vector get_entity_capabilities(const std::string & /*entity_id*/) const override { return {}; } - ros2_medkit_gateway::LockAccessResult check_lock(const std::string &, const std::string &, - const std::string &) const override { + ros2_medkit_gateway::LockAccessResult check_lock(const std::string & /*entity_id*/, const std::string & /*client_id*/, + const std::string & /*collection*/) const override { return {}; } tl::expected - acquire_lock(const std::string &, const std::string &, const std::vector &, int) override { + acquire_lock(const std::string & /*entity_id*/, const std::string & /*client_id*/, + const std::vector & /*scopes*/, int /*expiration_seconds*/) override { return tl::make_unexpected(ros2_medkit_gateway::LockError{}); } - tl::expected release_lock(const std::string &, const std::string &) override { + tl::expected release_lock(const std::string & /*entity_id*/, + const std::string & /*client_id*/) override { return {}; } ros2_medkit_gateway::ResourceChangeNotifier * get_resource_change_notifier() override { @@ -524,8 +529,8 @@ TEST_F(GraphWatchdogPluginTest, FaultClientDeliversRequestsAndDrainsItsOwnRespon int received = 0; auto srv = sink->create_service( "/fault_manager/report_fault", - [&received, &received_mutex](const std::shared_ptr, - std::shared_ptr resp) { + [&received, &received_mutex](const std::shared_ptr & /*req*/, + const std::shared_ptr & resp) { { std::lock_guard lk(received_mutex); ++received; diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_expectation_integration.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_expectation_integration.cpp new file mode 100644 index 000000000..a4e06a7dc --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_expectation_integration.cpp @@ -0,0 +1,3935 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Drives the REAL mechanism: the detector's own LifecycleExpectationTracker state, a +// snapshot-driven presence diff (via set_apps()), and a REAL ReliabilityGate to arm the +// global bringup grace and to feed the detector its lifecycle labels via +// ReliabilityGate::lifecycle_state_of() - raising/clearing through a real ReportFault +// service round-trip to a fake fault_manager. NOT a full-gateway e2e; this proves the +// detector end to end within its own scope (the e2e tier proves the real +// lifecycle-transition pipeline). +// +// Two ordering rules are load-bearing here, or the test silently proves nothing: +// 1. Arm the global gate FIRST. The aggregated raise goes out under +// source_id="graph_watchdog", unknown to the WarmupTracker, so allows_raise() takes +// the global-bringup path and stays suppressed until a non-empty gate.update() has +// run and the global grace has elapsed: gate.update(snapshot, warmup) then +// gate.update(snapshot, warmup + 3). +// 2. Inject the lifecycle label AFTER the last gate.update(), and never call +// gate.update() again. LifecycleWatcher::update() rebuilds its tracked set from +// snapshot.apps SERVICES and ERASES any id it cannot find a get-state path for; the +// fixture's apps carry no services, so any gate.update() after +// set_lifecycle_state_for_test() wipes the injected label (state_of() then returns +// nullopt = no expectation = no raise). +// +// Alongside the fixture, plain configure()-level TESTs pin the config contract (unknown +// keys, validation warnings, the prune clamp): this binary links the detector .cpp, so +// REGISTER_DETECTOR is in-image and the registry hands out instances without any ROS +// scaffolding. +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "ros2_medkit_gateway/core/providers/introspection_provider.hpp" +#include "ros2_medkit_graph_watchdog/aggregated_fault.hpp" +#include "ros2_medkit_graph_watchdog/detector.hpp" +#include "ros2_medkit_graph_watchdog/detector_registry.hpp" +#include "ros2_medkit_graph_watchdog/graph_fault_codes.hpp" +#include "ros2_medkit_graph_watchdog/lifecycle_expectation_tracker.hpp" // kDefaultAbsenceGrace +#include "ros2_medkit_graph_watchdog/reliability_gate.hpp" + +using ros2_medkit_gateway::App; +using ros2_medkit_gateway::IntrospectionInput; +using ros2_medkit_graph_watchdog::DetectorContext; +using ros2_medkit_graph_watchdog::DetectorMode; +using ros2_medkit_graph_watchdog::kDefaultAbsenceGrace; +using ros2_medkit_graph_watchdog::ReliabilityGate; +using ros2_medkit_graph_watchdog::graph_fault_codes::kNodeInactive; +using ros2_medkit_graph_watchdog::graph_fault_codes::kNodeNotManaged; +using ros2_medkit_graph_watchdog::graph_fault_codes::kNodeUnreadable; +using Fault = ros2_medkit_msgs::msg::Fault; +using ReportFault = ros2_medkit_msgs::srv::ReportFault; +using namespace std::chrono_literals; + +namespace { +// The detector class is file-local in lifecycle_expectation_detector.cpp but +// self-registers via REGISTER_DETECTOR, which runs when that .cpp is linked into this +// test. Pull an instance from the registry (no production factory needed). +std::unique_ptr make_lifecycle_expectation() { + for (auto & d : ros2_medkit_graph_watchdog::DetectorRegistry::instance().create_all()) { + if (d->id() == "lifecycle_expectation") { + return std::move(d); + } + } + return nullptr; +} +// Aggregated fault source: no Component in the test snapshot, so graph_source_id() +// falls back to this literal (see aggregated_fault.hpp). +constexpr const char * kGraphSource = "graph_watchdog"; +constexpr int kWarmupCycles = 3; +// Mirrors the detector's kUnmeasuredHoldTicks (file-local there by design): consecutive +// matched-but-never-read ticks a required node holds back GRAPH_NODE_INACTIVE's clear +// before that hold releases - either into silence (not-managed) or into its own +// GRAPH_NODE_UNREADABLE report (unreadable), with no further bound past this point. +constexpr int kHoldTicks = 60; + +/// Captures rcutils log output for as long as it is alive and restores the console handler on every +/// exit path. Leaving the process-global handler installed would swallow the log output of every +/// later case in this binary, so the restore cannot be left to the success path. +/// +/// One implementation for the whole file. The rcutils handler CONTRACT hands over a caller-supplied +/// format string and a va_list - it is not a choice the handler makes - so `vsnprintf` here is the +/// one place in this file that cannot use a literal format. +/// +/// The handler is a plain C function pointer with no user-data slot, so the live capture is reached +/// through a file-static. It is written by the test thread and read by whichever thread logs, +/// hence the atomic. +class LogCapture { + public: + LogCapture() { + active().store(this); + rcutils_logging_set_output_handler(&LogCapture::handler); + } + ~LogCapture() { + rcutils_logging_set_output_handler(rcutils_logging_console_output_handler); + active().store(nullptr); + } + LogCapture(const LogCapture &) = delete; + LogCapture & operator=(const LogCapture &) = delete; + LogCapture(LogCapture &&) = delete; + LogCapture & operator=(LogCapture &&) = delete; + + /// How many captured lines contain `needle`. + int count(const std::string & needle) const { + std::lock_guard lk(mutex_); + return static_cast(std::count_if(lines_.begin(), lines_.end(), [&needle](const std::string & line) { + return line.find(needle) != std::string::npos; + })); + } + + private: + static std::atomic & active() { + static std::atomic current{nullptr}; + return current; + } + + static void handler(const rcutils_log_location_t * /*location*/, int /*severity*/, const char * /*name*/, + rcutils_time_point_value_t /*timestamp*/, const char * format, va_list * args) { + char buf[1024]; + va_list copy; + va_copy(copy, *args); + // The format string arrives from the logging call site through the handler signature, so there + // is no literal to write here and no other way to reach the arguments than the va_list. The + // build runs -Werror=format=2; GCC exempts va_list-taking formatters from -Wformat-nonliteral, + // clang does not, so the CI clang-tidy job fails on this line alone. A pragma rather than a + // lint-suppression comment, because the diagnostic is raised as an ERROR and clang-tidy does + // not honour suppression comments for those. Scoped to the single call. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-nonliteral" + vsnprintf(buf, sizeof(buf), format, copy); +#pragma GCC diagnostic pop + va_end(copy); + LogCapture * capture = active().load(); + if (capture == nullptr) { + return; + } + std::lock_guard lk(capture->mutex_); + capture->lines_.emplace_back(buf); + } + + mutable std::mutex mutex_; + std::vector lines_; +}; + +/// Adapts a body that wants the request and the response by reference to the shape rclcpp requires. +/// +/// create_service deduces the callback by comparing its argument tuple against +/// `std::function, std::shared_ptr)>` EXACTLY, so a handler +/// cannot declare those parameters by reference however little it needs a copy of them. Stating +/// that shape once here keeps it out of the fake service below (and keeps the by-value shared_ptr +/// copies out of clang-tidy's sight - see test_param_drift_integration.cpp's identical adapter). +template +std::function, std::shared_ptr)> +service_callback(BodyT body) { + return [body = std::move(body)](std::shared_ptr req, + std::shared_ptr resp) { + body(*req, *resp); + }; +} + +/// Snapshot builder for the configure()-level TESTs below (the fixture has its own). +IntrospectionInput snapshot_of(const std::vector & ids) { + IntrospectionInput snapshot; + for (const auto & id : ids) { + App a; + a.id = id; + a.bound_fqn = "/" + id; + snapshot.apps.push_back(a); + } + return snapshot; +} + +/// One app under an EXPLICIT (id, fqn), for the identity-churn sweep below: every +/// respawn arrives as a brand-new fqn under a namespace, matched by the same bare entry +/// through its leaf. +IntrospectionInput namespaced_snapshot_of(const std::string & ns, const std::string & leaf) { + IntrospectionInput snapshot; + App a; + a.id = ns + "_" + leaf; + a.bound_fqn = "/" + ns + "/" + leaf; + snapshot.apps.push_back(a); + return snapshot; +} + +/// How many CONFIRMED-inactive "filler" nodes, all named `0000`, `0001`, ... +/// (fixed 4-digit zero-padded suffix, so every one is the SAME length), are needed to exceed +/// AggregatedFault::kMaxDescriptionChars on their own - i.e. the smallest batch that would +/// already fill the description cap without any help from a later-crossing node. Derived from +/// the REAL detail-building code via a throwaway tracker instance (not a hand-rebuilt formula): +/// this is what R11's ordering fix has to survive against, since a batch this size sorts before +/// any node whose id starts with a later letter and, before this slice, would have silently +/// evicted it forever. Fills `ids_out` with those ids and returns the count. +std::size_t fill_count_past_cap(const std::string & prefix, std::vector & ids_out) { + ros2_medkit_graph_watchdog::LifecycleExpectationTracker probe({prefix + "0000"}, /*grace=*/0); + const std::string sample_fqn = "/" + prefix + "0000"; + auto probe_report = + probe.update({ros2_medkit_graph_watchdog::LifecycleMatch{prefix + "0000", sample_fqn, std::string("inactive")}}); + const std::size_t entry_len = probe_report.affected.at(sample_fqn).size(); + constexpr std::size_t kJoinSep = 2; // "; " - AggregatedFault::describe_ordered's join separator + const std::size_t cap = ros2_medkit_graph_watchdog::AggregatedFault::kMaxDescriptionChars; + std::size_t count = 1; + while (count * entry_len + (count - 1) * kJoinSep <= cap) { + ++count; + } + ids_out.clear(); + for (std::size_t i = 0; i < count; ++i) { + std::string suffix = std::to_string(i); + suffix.insert(0, 4 - std::min(4, suffix.size()), '0'); // 4-digit zero pad + ids_out.push_back(prefix + suffix); + } + return count; +} +} // namespace + +class LifecycleExpectationIntegrationTest : public ::testing::Test { + protected: + // Must run before the FIRST test's fixture is constructed - see the identical + // rationale in test_param_drift_integration.cpp. + static void SetUpTestSuite() { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + } + + void SetUp() override { + gateway_ = std::make_shared("le_it_gateway"); + sink_ = std::make_shared("le_it_sink"); + srv_ = sink_->create_service( + "/fault_manager/report_fault", + service_callback([this](ReportFault::Request & req, ReportFault::Response & resp) { + { + std::lock_guard lk(mtx_); + received_.push_back(req); + } + resp.accepted = true; // ReportFault.srv response field is `bool accepted` + })); + client_ = gateway_->create_client("/fault_manager/report_fault"); + exec_.add_node(gateway_); + exec_.add_node(sink_); + spin_ = std::thread([this]() { + exec_.spin(); + }); + ASSERT_TRUE(client_->wait_for_service(5s)); + } + + void TearDown() override { + exec_.cancel(); + if (spin_.joinable()) { + spin_.join(); + } + exec_.remove_node(gateway_); + exec_.remove_node(sink_); + client_.reset(); + srv_.reset(); + sink_.reset(); + gateway_.reset(); + } + + DetectorContext make_ctx(DetectorMode mode, ReliabilityGate * gate) { + DetectorContext ctx; + ctx.gateway_node = gateway_.get(); + ctx.node_mutex = &node_mutex_; + ctx.mode = mode; + ctx.gate = gate; // the detector reads ctx.gate->lifecycle_state_of() - must be real, not null + ctx.fault_client = client_; + ctx.snapshot = &snapshot_; // tick() early-returns without a snapshot; each test seeds set_apps() + return ctx; + } + + // (Re)populate the owned entity snapshot the detector reads each tick - the + // fixture's presence-diff seam. components stays empty, so the aggregated fault's + // source_id falls back to the literal "graph_watchdog". + void set_apps(const std::vector & ids) { + snapshot_.apps.clear(); + for (const auto & id : ids) { + App a; + a.id = id; + a.bound_fqn = "/" + id; + snapshot_.apps.push_back(a); + } + } + + // Seed apps with EXPLICIT (id, fqn) pairs - to reproduce App::id instability, where the + // gateway gives a bare-name-colliding node a namespaced id while its fqn stays stable. + void set_apps_raw(const std::vector> & id_fqns) { + snapshot_.apps.clear(); + for (const auto & [id, fqn] : id_fqns) { + App a; + a.id = id; + a.bound_fqn = fqn; + snapshot_.apps.push_back(a); + } + } + + // Seed the snapshot with ONE managed lifecycle app under an EXPLICIT (id, fqn): the + // GetState + ChangeState services are what make the gate's internal lifecycle tracking + // keep the id across gate.update() calls (the service-less apps above are erased by + // every update - ordering rule 2). No live node answers at `fqn`, so seeds fail into + // the unknown (empty) label. + void set_managed_app(const std::string & id, const std::string & fqn) { + snapshot_.apps.clear(); + ros2_medkit_gateway::ServiceInfo get_state; + get_state.full_path = fqn + "/get_state"; + get_state.type = "lifecycle_msgs/srv/GetState"; + ros2_medkit_gateway::ServiceInfo change_state; + change_state.full_path = fqn + "/change_state"; + change_state.type = "lifecycle_msgs/srv/ChangeState"; + App a; + a.id = id; + a.bound_fqn = fqn; + a.services = {get_state, change_state}; + snapshot_.apps.push_back(a); + } + + // Arm the global bringup grace the aggregated "graph_watchdog" source needs (see the + // file-level ordering-rule doc comment, rule 1). Must be the LAST gate.update() call + // in a test before any set_lifecycle_state_for_test() injection (rule 2). + void arm_global_grace(ReliabilityGate & gate) { + gate.update(snapshot_, 5); + gate.update(snapshot_, 5 + kWarmupCycles); + } + + // How many matching faults are currently in the log (for absence checks + snapshots). + // `code` defaults to kNodeInactive since most of this fixture's tests are about that + // fault; a test exercising the sibling GRAPH_NODE_UNREADABLE record passes it explicitly. + std::size_t count_faults(const std::string & source_id, uint8_t event_type, + const std::string & code = kNodeInactive) { + std::lock_guard lk(mtx_); + std::size_t n = 0; + for (const auto & r : received_) { + if (r.source_id == source_id && r.fault_code == code && r.event_type == event_type) { + ++n; + } + } + return n; + } + + // EVERY request the fake fault_manager has received, of ANY source/code/event kind. + // The zero-config claim is "the fault_manager hears nothing at all", so the instrument + // has to count everything, not just the matching FAILED events. + std::size_t count_all_requests() { + std::lock_guard lk(mtx_); + return received_.size(); + } + + // Description of the LAST matching FAILED request (empty if none arrived). + std::string last_failed_description(const std::string & source_id, const std::string & code = kNodeInactive) { + std::lock_guard lk(mtx_); + std::string desc; + for (const auto & r : received_) { + if (r.source_id == source_id && r.fault_code == code && r.event_type == ReportFault::Request::EVENT_FAILED) { + desc = r.description; + } + } + return desc; + } + + // True if any recorded FAILED for `source_id` carries a description mentioning `needle`. + bool any_failed_desc_contains(const std::string & source_id, const std::string & needle, + const std::string & code = kNodeInactive) { + std::lock_guard lk(mtx_); + for (const auto & r : received_) { + if (r.source_id != source_id || r.fault_code != code || r.event_type != ReportFault::Request::EVENT_FAILED) { + continue; + } + if (r.description.find(needle) != std::string::npos) { + return true; + } + } + return false; + } + + // Severity of the LAST matching FAILED request. nullopt if none arrived - a missing + // raise must fail the assertion that reads this, not silently compare against 0. + std::optional last_failed_severity(const std::string & source_id, const std::string & code = kNodeInactive) { + std::lock_guard lk(mtx_); + std::optional severity; + for (const auto & r : received_) { + if (r.source_id == source_id && r.fault_code == code && r.event_type == ReportFault::Request::EVENT_FAILED) { + severity = r.severity; + } + } + return severity; + } + + // Tick until a NEW matching fault appears beyond `baseline_count` (arrived AFTER the + // trigger), or timeout. NEVER calls gate.update() - only the detector is ticked, per + // ordering rule 2 (a further gate.update() would erase the injected lifecycle label). + bool poll_for_new(const std::string & source_id, uint8_t event_type, std::size_t baseline_count, + ros2_medkit_graph_watchdog::Detector & det, DetectorContext & ctx, + const std::string & code = kNodeInactive) { + for (int i = 0; i < 130; ++i) { + det.tick(ctx); + std::this_thread::sleep_for(50ms); + if (count_faults(source_id, event_type, code) > baseline_count) { + return true; + } + } + return false; + } + + // Wait (WITHOUT ticking) until at least `target` matching requests have arrived - for + // the exactly-N assertions, where further ticks would legitimately emit more. + bool wait_for_count(const std::string & source_id, uint8_t event_type, std::size_t target, + const std::string & code = kNodeInactive) { + for (int i = 0; i < 150; ++i) { + if (count_faults(source_id, event_type, code) >= target) { + return true; + } + std::this_thread::sleep_for(20ms); + } + return count_faults(source_id, event_type, code) >= target; + } + + rclcpp::Node::SharedPtr gateway_, sink_; + rclcpp::Service::SharedPtr srv_; + rclcpp::Client::SharedPtr client_; + rclcpp::executors::MultiThreadedExecutor exec_; + std::thread spin_; + std::mutex mtx_, node_mutex_; + std::vector received_; + IntrospectionInput snapshot_; // owned entity snapshot the detector reads each tick +}; + +// The whole point of splitting the unreadable case into its own fault code: a confirmed +// violation heals while an UNREADABLE sibling is present and STAYS present (never read, +// for the whole test). GRAPH_NODE_INACTIVE must clear on "a" healing regardless of what +// "b" is doing - and GRAPH_NODE_UNREADABLE, which is entirely about "b", must not care +// that "a" healed either. "b" is run well past its own hold first, so under the OLD +// shared-record design it would already be poisoning the one record both codes used to +// share; under this design it is its own fault and never touches GRAPH_NODE_INACTIVE at +// all once past the hold. +TEST_F(LifecycleExpectationIntegrationTest, ConfirmedNodeHealsWhileUnreadableSiblingStaysPresentClearsInactiveOnly) { + set_apps({"a", "b"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "b"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "inactive"); // confirmed violation + gate.set_lifecycle_state_for_test("b", ""); // unreadable, never read for the rest of the test + ASSERT_TRUE(gate.lifecycle_state_of("b").has_value() && gate.lifecycle_state_of("b")->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeInactive)) + << "\"a\" never raised, so nothing below can be attributed to it healing"; + + // Run "b" well past its own hold, so it is already reported as its own + // GRAPH_NODE_UNREADABLE fault before "a" heals. + for (int i = 0; i < kHoldTicks + 5; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + ASSERT_TRUE(any_failed_desc_contains(kGraphSource, "/b", kNodeUnreadable)) + << "\"b\"'s unreadable hold never converted into its own record, so nothing below is tested"; + // Baseline captured AFTER "b" has already converted to content: from this point on, + // GRAPH_NODE_UNREADABLE is raising every tick (never clearing) because "b" stays + // present and unread - the level-triggered PASSED stream that ran during the hold, + // before "b" had anything to report, is over. + const auto unreadable_passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeUnreadable); + + // "a" heals. GRAPH_NODE_INACTIVE must clear PROMPTLY - not after "b" is eventually read, + // and not after any content-window delay - because "b" no longer influences it at all. + gate.set_lifecycle_state_for_test("a", "active"); + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeInactive); + bool cleared = false; + for (int i = 0; i < 20 && !cleared; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + cleared = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeInactive) > passed_before; + } + EXPECT_TRUE(cleared) << "GRAPH_NODE_INACTIVE did not clear once its only confirmed violation healed, even " + "though the only other required node was merely unreadable, never confirmed"; + + // GRAPH_NODE_UNREADABLE, meanwhile, is entirely unaffected by "a" healing: "b" is still + // unread, so its own fault must still be raised and must not have cleared even once. + EXPECT_GT(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable), 0u) + << "GRAPH_NODE_UNREADABLE never raised at all"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "/b", kNodeUnreadable)) + << "GRAPH_NODE_UNREADABLE stopped naming \"b\" once the unrelated \"a\" healed - the two faults must " + "be independent"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeUnreadable), unreadable_passed_before) + << "GRAPH_NODE_UNREADABLE cleared merely because a DIFFERENT required node (\"a\") healed"; +} + +// The OTHER clear direction, left untested by the test above: it proves an INACTIVE +// clear leaves UNREADABLE alone, not that an UNREADABLE clear leaves a concurrently +// raised INACTIVE alone. Wiring GRAPH_NODE_UNREADABLE's clear to also clear +// GRAPH_NODE_INACTIVE would pass every other test in this file today - nothing else +// reads a PASSED for kNodeInactive across a sibling's clear - since "a" here is never +// healed at all. +TEST_F(LifecycleExpectationIntegrationTest, UnreadableSiblingClearingDoesNotTouchAConcurrentlyRaisedInactive) { + set_apps({"a", "b"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "b"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "inactive"); // confirmed violation, NEVER healed in this test + gate.set_lifecycle_state_for_test("b", ""); // unreadable, will be read below + ASSERT_TRUE(gate.lifecycle_state_of("b").has_value() && gate.lifecycle_state_of("b")->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeInactive)) + << "\"a\" never raised, so nothing below can be attributed to it being left alone"; + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeUnreadable)) + << "\"b\"'s unreadable hold never converted into GRAPH_NODE_UNREADABLE, so nothing below " + "isolates the clear direction under test"; + + const auto inactive_passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeInactive); + + // "b" is genuinely read: GRAPH_NODE_UNREADABLE clears. + gate.set_lifecycle_state_for_test("b", "active"); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, 0, *det, ctx, kNodeUnreadable)) + << "\"b\" being read never cleared GRAPH_NODE_UNREADABLE, so nothing below proves anything"; + + // GRAPH_NODE_INACTIVE, meanwhile, must be entirely undisturbed: "a" is still confirmed + // inactive and was never touched. + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeInactive), inactive_passed_before) + << "GRAPH_NODE_UNREADABLE clearing also cleared GRAPH_NODE_INACTIVE for an unrelated node - " + "the two faults are not independent in this direction"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "/a", kNodeInactive)) + << "GRAPH_NODE_INACTIVE stopped naming \"a\" once the unrelated \"b\" cleared"; +} + +// Retitled honestly rather than left where its position in this file implied more than it +// checks: this pattern (a real "inactive" read alternating with unread ticks) passes +// unchanged against the design that preceded the two-clock model, because a real "inactive" +// read always resets the unmeasured clock and the violation streak alone accumulates across +// the unread gaps. So it pins the STREAK-ACCUMULATION rule - that only a genuine "active" +// read resets the streak - and nothing about the cause-blind unmeasured clock. The pairs +// that do discriminate the current model live in +// AlternatingUnreadableAndNotManagedRaisesOneOfTheTwoUnmeasuredCodes and +// InactiveAlternatingWithNotManagedAcrossAbsenceGapsRaisesInactive below. +TEST_F(LifecycleExpectationIntegrationTest, MeasuredInactiveSeparatedByUnreadTicksRaisesInactiveViaTheStreak) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + constexpr int kGrace = 5; + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", kGrace}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + // Alternate: one genuine "inactive" read, then a run of unread ticks LONGER than + // kDefaultAbsenceGrace (3, fixed in the tracker) - otherwise a single unread run alone + // could never have spent the OLD absence-borrowing budget in one go, and this test + // would not distinguish the fix from the bug. Each unread run still stays far under + // GRAPH_NODE_UNREADABLE's own 60-tick hold (kHoldTicks), so this pattern can only ever + // be reported through GRAPH_NODE_INACTIVE's streak accumulating across the gaps, never + // through its sibling maturing. Neither det->tick() nor this loop ever calls + // gate.update() (ordering rule 2), so each set_lifecycle_state_for_test() injection + // below sticks until the next one overwrites it. + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeInactive); + bool raised = false; + for (int cycle = 0; cycle < 20 && !raised; ++cycle) { + gate.set_lifecycle_state_for_test("a", "inactive"); + det->tick(ctx); + std::this_thread::sleep_for(5ms); + gate.set_lifecycle_state_for_test("a", ""); + for (int i = 0; i < kDefaultAbsenceGrace + 2; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + raised = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeInactive) > failed_before; + } + EXPECT_TRUE(raised) << "a node alternating between measured-inactive and unreadable never raised " + "GRAPH_NODE_INACTIVE - the streak kept being erased by the intervening unread ticks, " + "which is exactly what left it invisible to both this code and GRAPH_NODE_UNREADABLE"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable), 0u) + << "GRAPH_NODE_UNREADABLE also fired for this pattern - the alternation was not short enough " + "to isolate GRAPH_NODE_INACTIVE's streak-accumulation fix from its sibling's own hold"; +} + +// The alternation this slice's redesign closes, driven through the REAL gate rather than +// the injection seam (which can only ever SET a value, never remove tracking - the seam +// cannot produce a genuine nullopt for a previously-tracked fqn). Toggling whether "a" +// carries GetState/ChangeState services drives the two unmeasured causes directly: +// present -> LifecycleWatcher tracks it and seeds "" (UNREADABLE, no live responder); +// absent -> the tracked entry is dropped (NOT-MANAGED, lifecycle_state_of() -> nullopt). +// Before this redesign, this exact 80-tick alternation left BOTH GRAPH_NODE_UNREADABLE and +// (before this slice added it) whatever code NOT-MANAGED might have had permanently silent - +// watched directly against the pre-redesign detector, in the same shape as this test, before +// this slice began. Now the two causes share one cause-blind clock, so the alternation +// matures it regardless of which cause is live on any given tick. +TEST_F(LifecycleExpectationIntegrationTest, AlternatingUnreadableAndNotManagedRaisesOneOfTheTwoUnmeasuredCodes) { + set_apps({}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + std::uint64_t tick_counter = 5000; + bool raised = false; + for (int cycle = 0; cycle < 40 && !raised; ++cycle) { + // UNREADABLE leg: services present, no live GetState responder -> seeds "". + set_managed_app("a", "/a"); + gate.update(snapshot_, ++tick_counter); + det->tick(ctx); + std::this_thread::sleep_for(5ms); + + // NOT-MANAGED leg: services gone -> the tracked entry is dropped -> nullopt. + set_apps_raw({{"a", "/a"}}); + gate.update(snapshot_, ++tick_counter); + det->tick(ctx); + std::this_thread::sleep_for(5ms); + + raised = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable) > 0 || + count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeNotManaged) > 0; + } + EXPECT_TRUE(raised) << "a node alternating between unreadable and not-managed never raised EITHER " + "unmeasured code, even past 80 total ticks - the shared clock this redesign " + "adds did not close the alternation it exists to close"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "GRAPH_NODE_INACTIVE raised for a node that was never confirmed non-active, only unmeasured"; +} + +// The full story: "a" present, armed via a REAL wired gate (arming the global bringup +// grace the aggregated "graph_watchdog" source needs), then injected as +// lifecycle-inactive (AFTER the last gate.update(), per ordering rule 2) - ticking the +// detector past grace raises GRAPH_NODE_INACTIVE naming "a"; driving "a" to "active" +// clears it. +TEST_F(LifecycleExpectationIntegrationTest, RequiredNodeStuckInactivePastGraceRaisesThenActiveClears) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + // Inject the label AFTER the last gate.update() (rule 2) - never call gate.update() + // again from here on. + gate.set_lifecycle_state_for_test("a", "inactive"); + + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "a")); + + // "a" reaches active -> the aggregate clears. + gate.set_lifecycle_state_for_test("a", "active"); + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)); +} + +// Positive control: a required node that is active from the moment it is observed must +// never raise, however many ticks pass. +TEST_F(LifecycleExpectationIntegrationTest, RequiredNodeActiveFromArmingNeverRaises) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "active"); + + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "a required node reported active from the start was flagged GRAPH_NODE_INACTIVE"; +} + +// Absence control: once "a" vanishes from the snapshot entirely, GRAPH_NODE_INACTIVE must +// STAY raised. The operator declared the node must be active, the detector measured that it +// was not, and the node then leaving the graph answers nothing - so its clear must not flow. +// This is the deliberate behaviour change: the design this replaces cleared here, which is +// also what let a node that vanishes periodically shed the evidence against it. Absence is +// driven via set_apps({}), NEVER gate.update({}) (which would reset the gate's own +// graph_seen_ global-bringup marker, see the file-level ordering-rule doc comment). +TEST_F(LifecycleExpectationIntegrationTest, AbsenceAfterRaiseKeepsTheFaultAndSaysTheNodeIsGone) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "inactive"); + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // "a" vanishes from the graph entirely (a crash, not a lifecycle transition). + set_apps({}); + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < kDefaultAbsenceGrace + 20; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(10ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "GRAPH_NODE_INACTIVE cleared once the required node vanished - the node was measured " + "not-active and nothing has said otherwise since, so leaving the graph must not heal it"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "has since left the graph")) + << "the fault kept describing a node as merely inactive after it left the graph, which sends " + "the operator looking for a node that is not there"; +} + +// This slice's fix, at the tier that proves the detector's real OUTPUT rather than the +// tracker's internal report: a node the detector has already reported must not emit a +// spurious PASSED on its very first absent tick. Companion to +// AbsenceAfterRaiseClearsNotNodeDeathsDomain above (which proves the OPPOSITE and must +// stay true: sustained absence past the grace DOES clear) - here the node returns well +// before the absence grace elapses, so no PASSED may reach the fake service at any +// point during the blink. +TEST_F(LifecycleExpectationIntegrationTest, NoPassedReachesTheServiceWhileTheNodeIsInsideTheAbsenceGrace) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "inactive"); + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // "a" blinks out of the snapshot for a few ticks - well inside kDefaultAbsenceGrace - + // and stays absent for the whole check below. Not one PASSED may reach the fake + // service across the blink. + set_apps({}); + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < kDefaultAbsenceGrace; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "a node the detector already reported was cleared on a blink well inside the " + "absence grace - the withheld-clear guard's pending leg did not hold it"; +} + +// R10 pin, at the tier that proves the SERVICE saw no churn. The re-raise on return is +// CORRECT and must be preserved: before this slice's fix the node cleared on its very +// first absent tick and then re-raised on return - a raise/clear/raise churn. With the +// fix there is no clear in between, so across the whole blink-and-return sequence the +// fake service must never see a PASSED (the emitter is level-triggered and re-sends +// FAILED on every tick the condition holds regardless of a blink, so a recurring FAILED +// count is expected and not itself evidence of churn - only an interleaved PASSED is). +TEST_F(LifecycleExpectationIntegrationTest, ReRaiseOnReturnProducesNoClearChurnThroughTheRealGate) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "inactive"); + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + + // A brief blink, well inside the absence grace, then back - still inactive throughout. + set_apps({}); + for (int i = 0; i < 2; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + set_apps({"a"}); + for (int i = 0; i < 5; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "the blink must never have produced a clear - the streak survived it with no fresh " + "grace to re-earn, so there is nothing here for a re-raise to follow"; +} + +// Change: config changes while a node's absence-grace hold counters are live. A +// reconfigure rebuilds the tracker (the old streak is gone), but the withheld-clear +// guard's OTHER leg - the fresh tracker has not matched the entry yet - must pick the +// hold back up without a gap, so no PASSED may leak across the reconfigure boundary +// either. +TEST_F(LifecycleExpectationIntegrationTest, ReconfigureDuringAnAbsenceHoldNeverLeaksAPassed) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + const nlohmann::json config{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}; + det->configure(config); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "inactive"); + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)); + + // "a" blinks out of the snapshot, and the detector is reconfigured mid-blink. + set_apps({}); + det->tick(ctx); + std::this_thread::sleep_for(20ms); + det->configure(config); // rebuilds the tracker: the old streak is gone + + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "a reconfigure mid-blink must not spuriously heal a fault that was never re-confirmed healthy"; +} + +// Regression: App::id is recomputed each sweep and gets namespaced on a same-bare-name +// collision, so a bare-name require_active entry must still match a live node via its stable +// fqn leaf - otherwise the check silently stops on a multi-robot graph. Here the app carries a +// namespaced id ("robot1_a") but a stable fqn "/robot1/a"; require_active: ["a"] (bare) must +// match it by leaf and still enforce the expectation. +TEST_F(LifecycleExpectationIntegrationTest, BareNameRequireActiveMatchesNamespacedAppByLeaf) { + set_apps_raw({{"robot1_a", "/robot1/a"}}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + // Lifecycle state is keyed by the node's current App::id ("robot1_a"); the require_active + // entry "a" matches it via the fqn leaf. + gate.set_lifecycle_state_for_test("robot1_a", "inactive"); + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "a bare-name require_active entry stopped matching a node whose App::id was namespaced by a " + "collision - the check must match via the stable fqn leaf instead of silently stopping"; +} + +// A FULL-FQN entry must match through the fqn arm (id != fqn), not by luck of the App::id. +// The app carries an id that equals neither the entry nor its leaf, so only the fqn +// comparison can bind them. +TEST_F(LifecycleExpectationIntegrationTest, FullFqnRequireActiveMatchesByFqnWhenIdDiffers) { + set_apps_raw({{"nsb", "/ns/b"}}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"/ns/b"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("nsb", "inactive"); + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "a full-FQN require_active entry did not match an app whose id differs from its fqn"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "/ns/b")); +} + +// The shipped default: an empty detector config checks nothing and CONTACTS nothing. The +// instrument counts every request of any kind, so a stray clear-spam (or any other +// chatter) fails this, not just a wrong raise. The inactive label makes the claim sharp: +// even with a would-be-offending node in view, an unconfigured detector stays silent. +TEST_F(LifecycleExpectationIntegrationTest, EmptyConfigNeverContactsTheFaultManager) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json::object()); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + gate.set_lifecycle_state_for_test("a", "inactive"); + + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + std::this_thread::sleep_for(200ms); // let any in-flight request land before counting + EXPECT_EQ(count_all_requests(), 0u) + << "an unconfigured lifecycle_expectation sent the fault_manager a request; the zero-config " + "default must emit neither raises nor clears"; +} + +// Same claim for the explicit `require_active: []` spelling of "nothing configured". +TEST_F(LifecycleExpectationIntegrationTest, ExplicitEmptyRequireActiveNeverContactsTheFaultManager) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array()}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + gate.set_lifecycle_state_for_test("a", "inactive"); + + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + std::this_thread::sleep_for(200ms); + EXPECT_EQ(count_all_requests(), 0u) << "require_active: [] must behave exactly like no config at all"; +} + +// An entry that matches a present node with NO tracked lifecycle state is probably a typo +// (or a plain node). The warning must reach the log after kUnmanagedWarnTicks consecutive +// such ticks, exactly once per entry per configuration - and a reconfigure that removes +// and re-adds the entry must warn AGAIN, because that is exactly when the operator wants +// to hear the entry still names nothing managed. +TEST_F(LifecycleExpectationIntegrationTest, UnmanagedEntryWarnsOncePerConfiguration) { + const LogCapture log; + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + const nlohmann::json config{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}; + det->configure(config); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + // Deliberately NO set_lifecycle_state_for_test: "a" is matched but has no tracked + // lifecycle state, which is the unmanaged case under test. + + const std::string needle = "require_active entry 'a' has matched a present node"; + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count(needle), 1) << "expected the unmanaged-entry warning exactly once over ten ticks " + "(0 = never surfaced, >1 = the once-per-entry guard is gone)"; + + // Remove the entry, then re-add it: the warn latch is scoped to the CURRENT config, so + // the re-added entry must warn again. + det->configure(nlohmann::json{{"require_active", nlohmann::json::array()}}); + det->configure(config); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count(needle), 2) << "an entry removed and re-added by reconfigure() never warned again - " + "the latch must reset with the config"; +} + +// Withheld-clear guard, the never-measured half (see the detector's withheld-clear design +// note). A gateway restart re-instantiates the detector with everything unread; the fault +// it raised BEFORE the restart is still in the store. Emitting the level-triggered clear +// before a single label has been read would spuriously HEAL a still-real inactive fault - +// so through the bounded hold window the detector must emit NOTHING, and the clear must +// flow only after a label is actually read. +TEST_F(LifecycleExpectationIntegrationTest, RestartStandInWithholdsTheClearUntilALabelIsRead) { + set_apps({"a"}); + + // Phase 1 (pre-restart): a configured + labeled run raises the prior fault. + ReliabilityGate gate1(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate1); + auto det1 = make_lifecycle_expectation(); + ASSERT_TRUE(det1); + det1->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx1 = make_ctx(DetectorMode::Raise, &gate1); + gate1.set_lifecycle_state_for_test("a", "inactive"); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det1, ctx1)) + << "phase 1 never raised, so phase 2 would prove nothing"; + std::this_thread::sleep_for(200ms); // let phase-1 in-flight requests land before baselining + + // Phase 2 (restart stand-in): fresh gate, fresh detector, NO labels read yet. + ReliabilityGate gate2(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate2); + auto det2 = make_lifecycle_expectation(); + ASSERT_TRUE(det2); + det2->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx2 = make_ctx(DetectorMode::Raise, &gate2); + + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < kHoldTicks; ++i) { + det2->tick(ctx2); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(300ms); // any withheld-but-actually-sent clear must have landed by now + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "a freshly restarted detector cleared GRAPH_NODE_INACTIVE before reading a single lifecycle " + "label - the restart spuriously heals a still-real fault"; + + // The label is finally read (still-inactive would re-raise; active releases the clear). + gate2.set_lifecycle_state_for_test("a", "active"); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before, *det2, ctx2)) + << "once the label IS read and healthy, the clear must flow"; +} + +// Withheld-clear guard, the typo unblock: an entry that matches a node whose label is +// NEVER read - an unmanaged node, or a typo that happens to match - may hold the clear +// back only for the bounded hold window. Past it the clear must flow, or one such entry +// blocks healing forever. +TEST_F(LifecycleExpectationIntegrationTest, NeverReadNodePastTheHoldBoundStopsBlockingTheClear) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + // NO label injection, ever: "a" stays unread for the whole test. + + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the clear stayed withheld past the bounded hold - an unmanaged or typo'd entry must not " + "block healing forever"; +} + +// Withheld-clear guard, the window that actually matters: a restart whose labels seed +// FAST. The plugin seeds lifecycle labels BEFORE it ticks the detectors, so on a +// responsive stack the first tick of a fresh detector already reads the real label - +// "unread" is the rare case, not the common one. What IS fresh after a restart is the +// violation streak, and while it climbs back through grace the tracker reports nothing +// affected. Emitting the level-triggered clear there asserts "nothing is stuck" about a +// node this detector's own last read found stuck, and heals the fault the restart was +// supposed to preserve. +TEST_F(LifecycleExpectationIntegrationTest, RestartStandInWithholdsTheClearWhileTheNodeReadsNotActiveBelowGrace) { + set_apps({"a"}); + + // Phase 1 (pre-restart): a configured + labeled run raises the prior fault. + ReliabilityGate gate1(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate1); + auto det1 = make_lifecycle_expectation(); + ASSERT_TRUE(det1); + det1->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx1 = make_ctx(DetectorMode::Raise, &gate1); + gate1.set_lifecycle_state_for_test("a", "inactive"); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det1, ctx1)) + << "phase 1 never raised, so phase 2 would prove nothing"; + std::this_thread::sleep_for(200ms); // let phase-1 in-flight requests land before baselining + + // Phase 2 (restart stand-in): fresh gate, fresh detector - but the label is seeded + // BEFORE the first tick, and the node is still stuck exactly where it was. + ReliabilityGate gate2(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate2); + auto det2 = make_lifecycle_expectation(); + ASSERT_TRUE(det2); + constexpr int kGrace = 4; + det2->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", kGrace}}); + auto ctx2 = make_ctx(DetectorMode::Raise, &gate2); + gate2.set_lifecycle_state_for_test("a", "inactive"); + + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + for (int i = 0; i < kGrace; ++i) { // the whole below-grace window, every tick measured stuck + det2->tick(ctx2); + std::this_thread::sleep_for(20ms); + } + std::this_thread::sleep_for(300ms); // any emitted request must have landed by now + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "the restarted detector cleared GRAPH_NODE_INACTIVE while its own reads said the node was " + "still not active - a young streak is not health"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), failed_before) + << "the raise must still wait for grace; only the clear is withheld"; + + // Past grace the raise resumes, so the hold is not a silence that never ends. + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det2, ctx2)) + << "the streak never reached grace + 1, so the still-real fault was never re-raised"; +} + +// The grace contract, through the REAL config path: "grace" is the number of consecutive +// not-active ticks a required NODE tolerates. An operator combining the two documented +// require_active forms (bare name = fleet-wide, full FQN = pin one robot) has one node +// named by two entries, and the detector pushes one match per (entry, node) - so counting +// per match raises after ceil((grace+1)/2) ticks instead of grace+1, a transient false +// positive on a node that is simply still coming up. +TEST_F(LifecycleExpectationIntegrationTest, TwoEntriesMatchingOneNodeDoNotHalveTheConfiguredGrace) { + set_apps({"a"}); // App::id "a", fqn "/a" - matched by BOTH entries below + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + constexpr int kGrace = 4; + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "/a"})}, {"grace", kGrace}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + gate.set_lifecycle_state_for_test("a", "inactive"); + + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + for (int i = 0; i < kGrace; ++i) { // exactly the tolerated window + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), failed_before) + << "the node was reported inactive INSIDE its grace because two require_active entries named " + "it - the streak must advance once per node per tick, not once per match"; + + // And the expectation is still enforced once the streak really does pass grace. + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "the raise never came at all - de-duplicating the matches must not disable the check"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "/a")); +} + +// The churn story the absence grace exists for, driven through the detector: a stuck node +// blinks out of the snapshot and comes back with the label LifecycleWatcher seeds when its +// GetState has not answered yet (""). Treating that re-seed tick as a healthy read restarts +// the violation count, so a node that blinks once every few ticks never accumulates +// grace + 1 consecutive stuck ticks and is never reported at all. +TEST_F(LifecycleExpectationIntegrationTest, BlinkAndUnreadReseedDoNotRestartTheViolationCount) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 3}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + // Tick-exact on purpose. "It raises eventually" is true whether or not the streak + // survives - a reset only delays the raise - so an unbounded poll would pass against a + // detector that restarts the count on every blink. The claim is that the streak reaches + // grace + 1 on THIS tick, so the run is counted out and nothing ticks afterwards. + gate.set_lifecycle_state_for_test("a", "inactive"); + det->tick(ctx); // stuck 1 + det->tick(ctx); // stuck 2 + + set_apps({}); // blink: out of the snapshot for one tick + det->tick(ctx); // absent 1 - within the absence grace, streak preserved + + set_apps({"a"}); // back, but the re-seed has not answered yet + gate.set_lifecycle_state_for_test("a", ""); // exactly what a missed GetState seed leaves + det->tick(ctx); // unread: absence-like, streak preserved + gate.set_lifecycle_state_for_test("a", "inactive"); + det->tick(ctx); // stuck 3 == grace, not past it yet + + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + det->tick(ctx); // stuck 4 > grace - the raise is due on this tick and no later + EXPECT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before + 1)) + << "a node stuck through a blink and an unread re-seed was not reported on the tick its streak " + "passed grace - the blink or the re-seed restarted its violation count, which is how a stuck " + "node on a churning graph stays silent forever"; +} + +// Guard bookkeeping is per NODE, not per (entry, node) pair. An operator combining the +// documented bare-name and pinned-FQN forms has one node named by two entries; burning +// its not-managed hold twice a tick halves the documented bound and lets the clear out +// at half the horizon the design promises. +// +// C2: named "Unread" before this correction, but "a" carries no lifecycle services +// (set_apps, not set_managed_app) and nothing ever injects a label for it, so +// lifecycle_state_of("a") is nullopt for the whole test - this pins the NOT-MANAGED cause +// of the shared unmeasured clock, not the UNREADABLE one. See +// TwoEntriesMatchingOneNodeDoNotHalveTheUnreadableHold below for the genuine +// optional("") case. +TEST_F(LifecycleExpectationIntegrationTest, TwoEntriesMatchingOneNodeDoNotHalveTheNotManagedHold) { + set_apps({"a"}); // App::id "a", fqn "/a" - matched by BOTH entries below + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "/a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + // NO label injection, ever: "/a" stays nullopt (NOT-MANAGED), so only the hold can + // release the clear. + ASSERT_FALSE(gate.lifecycle_state_of("a").has_value()) + << "\"a\" was unexpectedly tracked - this test would not be exercising NOT-MANAGED"; + + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < kHoldTicks / 2 + 5; ++i) { // comfortably past a halved hold, short of the real one + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "the not-managed hold released at half its documented length because two entries named " + "the same node - the guard must count per node, not per match"; + + // The bound itself still applies: past it, the clear flows. + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the hold never released at all - it must stay bounded"; +} + +// The genuine optional("") twin of the test above: "a" IS tracked (injected once, via +// the seam, which writes straight into the gate's internal tracked map regardless of +// services - see set_managed_app's doc comment for the alternative, real-seeding path), +// but its label is deliberately left "" for the whole test, so BOTH entries matching it +// share ONE unmeasured clock, not two. +TEST_F(LifecycleExpectationIntegrationTest, TwoEntriesMatchingOneNodeDoNotHalveTheUnreadableHold) { + set_apps({"a"}); // App::id "a", fqn "/a" - matched by BOTH entries below + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "/a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + gate.set_lifecycle_state_for_test("a", ""); + ASSERT_TRUE(gate.lifecycle_state_of("a").has_value() && gate.lifecycle_state_of("a")->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable); + for (int i = 0; i < kHoldTicks / 2 + 5; ++i) { // comfortably past a halved hold, short of the real one + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable), failed_before) + << "the unreadable hold converted to content at half its documented length because two " + "entries named the same node - the guard must count per node, not per match"; + + // The bound itself still applies: past it, the hold converts into a report. + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx, kNodeUnreadable)) + << "the hold never converted at all - it must stay bounded"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "/a", kNodeUnreadable)); +} + +// Mixed set, direction 1: node "a" was raised and has now healed, while sibling "b" has +// never been measured. Within the hold the clear waits - a run in which "b" was never +// read is not a run in which "b" was found healthy. +// +// C2: named "Unread" before this correction, but "b" carries no lifecycle services and +// is never injected, so lifecycle_state_of("b") is nullopt for the whole test - this +// pins the NOT-MANAGED leg, not the unreadable one. See +// MixedSetClearWaitsForTheUnreadableSiblingWithinTheHold below for the genuine +// optional("") case. +TEST_F(LifecycleExpectationIntegrationTest, MixedSetClearWaitsForTheNotManagedSiblingWithinTheHold) { + set_apps({"a", "b"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "b"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "inactive"); // "b" is never read in this test + ASSERT_FALSE(gate.lifecycle_state_of("b").has_value()) + << "\"b\" was unexpectedly tracked - this test would not be exercising NOT-MANAGED"; + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx)) + << "the raise never happened, so the heal leg would prove nothing"; + + gate.set_lifecycle_state_for_test("a", "active"); // the raised node heals + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < 20; ++i) { // well inside "b"'s hold + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "the clear flowed while a required sibling had never been measured once"; + + gate.set_lifecycle_state_for_test("b", "active"); // now everything really is measured healthy + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "once every required node has been read healthy the clear must flow"; +} + +// The genuine optional("") twin of the test above: "b" IS tracked (via the injection +// seam) but its label is deliberately left "" for the whole test, so it exercises the +// unmeasured clock's UNREADABLE cause, not its NOT-MANAGED one. +TEST_F(LifecycleExpectationIntegrationTest, MixedSetClearWaitsForTheUnreadableSiblingWithinTheHold) { + set_apps({"a", "b"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "b"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "inactive"); + gate.set_lifecycle_state_for_test("b", ""); // unreadable, never read in this test + ASSERT_TRUE(gate.lifecycle_state_of("b").has_value() && gate.lifecycle_state_of("b")->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx)) + << "the raise never happened, so the heal leg would prove nothing"; + + gate.set_lifecycle_state_for_test("a", "active"); // the raised node heals + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < 20; ++i) { // well inside "b"'s hold + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "the clear flowed while a required sibling had never been measured once"; + + gate.set_lifecycle_state_for_test("b", "active"); // now everything really is measured healthy + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "once every required node has been read healthy the clear must flow"; +} + +// Mixed set, direction 2: the hold is a bound on the WAIT, not a promise that every node +// was measured. "b"'s budget burns on every matched tick, including the ticks a raise +// about "a" was flowing, so a raise that outlives the hold leaves the clear free to flow +// the moment "a" heals - deliberately, or one unmanaged sibling would block healing for +// as long as the raise lasted. +// +// C2: named "Unread" before this correction, but "b" is genuinely nullopt (NOT-MANAGED) +// here, same as the pair above - see MixedSetClearWaitsForTheUnreadableSiblingWithinTheHold +// above for the corresponding UNREADABLE direction. GRAPH_NODE_INACTIVE's own clear behaves +// identically either way: once "b"'s unmeasured clock matures - whichever cause it matured +// under - ownership passes to "b"'s own fault code and GRAPH_NODE_INACTIVE has nothing left +// to hold for it, so "a" healing is free to clear GRAPH_NODE_INACTIVE regardless. What DOES +// differ is "b"'s own content: it converts into GRAPH_NODE_NOT_MANAGED here (asserted +// below), the sibling code to GRAPH_NODE_UNREADABLE - both stay reported with no further +// bound once matured (see UnreadableNodeStaysReportedWithNoExpiryAsLongAsItStaysUnreadable). +TEST_F(LifecycleExpectationIntegrationTest, MixedSetClearFlowsOnceTheNotManagedSiblingsHoldIsSpent) { + set_apps({"a", "b"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "b"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "inactive"); // "b" is never read in this test + ASSERT_FALSE(gate.lifecycle_state_of("b").has_value()) + << "\"b\" was unexpectedly tracked - this test would not be exercising NOT-MANAGED"; + for (int i = 0; i < kHoldTicks + 5; ++i) { // the raise outlives "b"'s hold + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + ASSERT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, 1)) + << "the raise never happened, so the heal leg would prove nothing"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "/b", kNodeNotManaged)) + << "\"b\"'s not-managed clock never converted into its own record, so the clear below is not " + "proving what this test claims - the withhold releasing via ownership transfer, not silence"; + + gate.set_lifecycle_state_for_test("a", "active"); + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the clear never came: past its hold a not-managed sibling must stop blocking, or one " + "unmanaged entry would keep a healed fault raised for as long as the raise lasted"; +} + +// R7's underlying concern, restated under the current (post-redesign) model, which has no +// "a label was read" latch of its own to go stale: a process that dies and respawns under +// the same name must not have its fresh, unread incarnation's unmeasured clock exempted by +// anything the DEAD incarnation did. Here there is nothing to exempt it WITH - a healthy +// read resets the unmeasured clock to zero (LifecycleExpectationTracker's own rule), so a +// respawn that comes back unreadable starts a clock at zero exactly like a first-contact +// unreadable node would, and only a REAL read - never the dead incarnation's - resets it +// again. This is what makes the sticky-`""` failure mode R7 originally closed structurally +// impossible now, rather than something a second bit has to keep correctly synchronized. +TEST_F(LifecycleExpectationIntegrationTest, RespawnComingBackUnreadableStartsAFreshUnmeasuredClockNotADeadLatch) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 4}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "active"); // resets both clocks + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "the healthy baseline never cleared, so nothing below can be attributed to the respawn"; + + set_apps({}); // the process dies + for (int i = 0; i < 3; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + // Respawned under the same fqn with an unseeded label ("" is what LifecycleWatcher + // seeds for a tracked node whose GetState has not answered yet). + set_apps({"a"}); + gate.set_lifecycle_state_for_test("a", ""); + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(200ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "a respawn that came back unreadable was cleared on the strength of the DEAD " + "incarnation's earlier read - the fresh incarnation's own unmeasured clock must " + "start from zero and be honored on its own, or the sticky-\"\" failure mode R7 " + "exists to close would just reappear on every respawn"; + + // And it is not stuck in silence forever either: once the fresh incarnation is + // actually read, the clear flows normally, same as any first-contact node. + gate.set_lifecycle_state_for_test("a", "active"); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the clear never flowed once the respawned incarnation was genuinely read active"; +} + +// The other half of the same story: the surviving latch does NOT leave a respawned node +// unguarded, because the streak the respawn starts is itself a hold. A node that comes +// back stuck must not have its still-real fault healed while its streak climbs to grace. +TEST_F(LifecycleExpectationIntegrationTest, RespawnedNodeStuckBelowGraceStillWithholdsTheClear) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + constexpr int kGrace = 5; + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", kGrace}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "active"); // read once: the latch is set + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, 0, *det, ctx)); + + set_apps({}); // the process dies, past the absence grace + for (int i = 0; i < 5; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(300ms); + + // It comes back - stuck. The latch is still set from the dead incarnation, so only the + // streak can hold the clear back. + set_apps({"a"}); + gate.set_lifecycle_state_for_test("a", "inactive"); + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < kGrace; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "a node that respawned still stuck had its fault cleared while the detector's own reads " + "said it was not active"; +} + +// R25, second finding: a re-contact earns the same protection as a first contact. "a" +// stays present in the GRAPH the whole time (this is not the presence/absence case, +// already covered by AbsenceAfterRaiseClearsNotNodeDeathsDomain), but a REAL gate.update() +// call drops its LifecycleWatcher-tracked entry once its GetState/ChangeState services +// are gone from the snapshot passed to it - exactly what happens when the underlying +// process restarts and its services have not been rediscovered yet (see the class doc on +// lifecycle_watcher.cpp's drop loop). lifecycle_state_of("a") then reads nullopt, and +// GRAPH_NODE_INACTIVE's already-CONFIRMED violation must not be cleared on the strength +// of that alone - it might be coming back stuck. +TEST_F(LifecycleExpectationIntegrationTest, RestartReadingNullOptWhileServicesAreRediscoveredDoesNotClearALiveStreak) { + set_managed_app("a", "/a"); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + // Inject AFTER the last gate.update() (ordering rule 2): confirmed violation. + gate.set_lifecycle_state_for_test("a", "inactive"); + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "\"a\" never raised, so nothing below proves a live streak survived the restart"; + + // Restart stand-in: "a" stays present in ctx.snapshot->apps (never removed via + // set_apps({})) but loses its GetState/ChangeState services for this one gate.update() + // call - the new incarnation has not advertised them yet. That is what makes + // LifecycleWatcher drop the tracked entry, so lifecycle_state_of("a") reads nullopt + // from here on, exactly as a live rediscovery window would. + set_apps_raw({{"a", "/a"}}); // present, no services + gate.update(snapshot_, /*tick=*/999); // drops the tracked entry + ASSERT_FALSE(gate.lifecycle_state_of("a").has_value()) + << "the restart stand-in did not actually produce nullopt - this test would not be exercising " + "the rediscovery window"; + + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < kDefaultAbsenceGrace; ++i) { // well inside the rediscovery window + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + std::this_thread::sleep_for(200ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "a live GRAPH_NODE_INACTIVE cleared while its node was restarting and reading nullopt with " + "its services not yet rediscovered - it might be coming back stuck"; + + // Not held open-ended either: once the fresh incarnation is genuinely read active, the + // clear flows normally, same as any first-contact node. + gate.set_lifecycle_state_for_test("a", "active"); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the clear never flowed once the respawned incarnation was genuinely read active"; +} + +// R25, first finding: a node must earn a fresh unmeasured-clock hold across a REAL +// absence-crossing restart. "a" leaves ctx.snapshot->apps entirely and stays away past +// kDefaultAbsenceGrace - the tracker's own absence loop resets both clocks to zero on +// that crossing - then returns reading nullopt (a REAL gate.update() drops the cached +// "active" label, same restart stand-in as the test above). The observable signal: +// AggregatedFault's emit_ordered has no dedup of its own (see its class doc), so EVERY +// tick left unheld re-sends a clear - a clock that resumed from wherever it was before +// the restart, instead of starting fresh at zero, would let the level-triggered clear +// keep firing on every tick after the return. +TEST_F(LifecycleExpectationIntegrationTest, ReturningNodeAfterARealRestartEarnsAFreshHoldInsteadOfResumingTheOldClock) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "active"); // read once: both clocks reset + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "the healthy baseline never cleared, so nothing below can be attributed to the restart"; + + // A REAL restart: "a" leaves the graph entirely, past the absence grace - not a blink. + set_apps({}); + for (int i = 0; i < kDefaultAbsenceGrace + 3; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + + // It returns, but its GetState/ChangeState services have not been rediscovered - same + // restart stand-in as the test above: a REAL gate.update() drops the dead incarnation's + // cached "active" label, so lifecycle_state_of("a") now reads nullopt. + set_apps_raw({{"a", "/a"}}); + gate.update(snapshot_, /*tick=*/999); + ASSERT_FALSE(gate.lifecycle_state_of("a").has_value()) + << "the restart stand-in did not actually produce nullopt - this test would not be exercising " + "the return path"; + + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < kHoldTicks - 5; ++i) { // comfortably below the fresh hold's own bound + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(200ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "a node returning after a real restart resumed the dead incarnation's unmeasured clock " + "instead of earning a fresh hold - every tick since should have been withheld, not " + "re-affirmed"; +} + +// The window BEFORE the first match, which neither the unmeasured clock nor the streak can see. +// A restarted gateway does not have the graph yet: for the first ticks after the plugin +// comes up the entity snapshot has not caught up, so a require_active entry matches +// nothing at all. Both other halves of the guard are keyed by a MATCHED node, so both are +// empty, the tracker reports nothing affected, and the level-triggered clear flows - about +// a node the detector has never once looked at. The fault the restart was supposed to +// preserve heals in that gap. +// +// Distinct from a node that VANISHES: an entry that has matched before and stops matching +// is a presence problem (pinned by AbsenceAfterRaiseClearsNotNodeDeathsDomain, which must +// stay green). This is an entry that has never matched anything since configure(). +TEST_F(LifecycleExpectationIntegrationTest, ClearIsWithheldUntilARequiredEntryHasMatchedAtLeastOnce) { + set_apps({}); // the required node has not reached the snapshot yet + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < 20; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "the detector asserted GRAPH_NODE_INACTIVE was healthy before it had ever matched the node " + "it was told to check - after a restart that heals a fault that is still real"; + + // Once the node is actually there and reads healthy, the clear flows. + set_apps({"a"}); + gate.set_lifecycle_state_for_test("a", "active"); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "the clear never flowed once the required node appeared and read active"; +} + +// The bound on that hold, same shape as the unread one: an entry that matches nothing +// because it is misspelt would otherwise block healing for the process lifetime. +TEST_F(LifecycleExpectationIntegrationTest, NeverMatchedEntryPastTheHoldBoundStopsBlockingTheClear) { + set_apps({"something_else"}); // "a" matches nothing, and never will + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "a misspelt entry that can never match blocked healing forever - the never-matched hold " + "must be bounded exactly like the unread one"; +} + +// A withhold is silence, and silence is indistinguishable from a detector that is working +// and finding nothing. The two reasons release on different conditions, so the log has to +// say which one is in effect - here the measured-but-below-grace one, whose hold ends when +// the node reads active or its streak passes grace, NOT after a fixed number of ticks. +TEST_F(LifecycleExpectationIntegrationTest, WithholdingForAYoungStreakSaysSoInTheLog) { + const LogCapture log; + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + // Grace wide enough that the whole run stays inside the pending window. + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 200}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + gate.set_lifecycle_state_for_test("a", "inactive"); // read every tick: only the streak can hold + + for (int i = 0; i < 30; ++i) { // well past the 10-tick reporting horizon + det->tick(ctx); + } + EXPECT_EQ(log.count("withheld from clearing"), 1) + << "a hold that outlives the reporting horizon must be explained exactly once per episode"; + EXPECT_EQ(log.count("measured not-active but still within grace"), 1) + << "the log named the wrong reason: nothing here is unread, the streak is simply young"; + EXPECT_EQ(log.count("no lifecycle label read this run"), 0) + << "the unread reason was reported for a node whose label was read on every tick"; +} + +// Withheld-clear guard boundary: a raise is NEVER withheld. A violation read from the +// nodes that did answer is real regardless of how many other required nodes are unread. +TEST_F(LifecycleExpectationIntegrationTest, RaiseIsNeverWithheldByAnUnreadNode) { + set_apps({"a", "b"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "b"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "inactive"); // "b" stays unread + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "an unread sibling node held back a raise about a node that WAS read inactive"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "/a")); +} + +// Scale: more required nodes stuck inactive than the 480-char cap can name, PLUS one MORE +// crossing fresh on a LATER tick, after the cap is already full. This replaces the old +// TwentyFiveStuckNodesRaiseOneCappedFault, which pinned that the LEXICOGRAPHICALLY FIRST +// affected node opens the description ("node /a00 expected active..." at offset 0) with 25 +// nodes crossing together in one tick. That assertion is now WRONG, on purpose: R11 makes the +// order "entered `affected` on THIS tick, first", not lexicographic, so a fresh violation must +// not be hidden behind alphabetically-earlier ones that have been stale for ticks - which is +// exactly the failure mode the old lexicographic emit() had (a later-arriving robot in a +// `require_active: [...]` fleet could be silently unnamed forever once earlier ones filled the +// cap). This test proves the fix directly: `fill_count_past_cap` derives - from the real +// detail-building code, not a hard-coded 25 - the smallest filler batch that alone exceeds the +// cap, then one MORE node crosses on a later tick with an id chosen to sort LAST among all of +// them; the old code would have dropped it from the description entirely. +TEST_F(LifecycleExpectationIntegrationTest, ANodeCrossingAfterTheCapIsFullIsNamedOverOlderEntries) { + std::vector ids; + const std::size_t fill_count = fill_count_past_cap("e", ids); + ASSERT_GT(fill_count, 0u); + const std::string late_id = "zlate"; // sorts after every "e..." filler, lexicographically last + ids.push_back(late_id); + + nlohmann::json require = nlohmann::json::array(); + for (const auto & id : ids) { + require.push_back(id); + } + set_apps(ids); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", require}, {"grace", 0}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + for (const auto & id : ids) { + if (id != late_id) { + gate.set_lifecycle_state_for_test(id, "inactive"); + } + } + gate.set_lifecycle_state_for_test(late_id, "active"); // healthy for now + + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + det->tick(ctx); // ONE tick: the filler batch crosses grace=0 together; "zlate" stays healthy + ASSERT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before + 1)); + std::this_thread::sleep_for(200ms); // a second request from the same tick would land by now + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), failed_before + 1) + << "the filler batch crossing together in one tick must aggregate into exactly one FAILED"; + { + const std::string desc = last_failed_description(kGraphSource); + const std::string marker = "..."; + EXPECT_LE(desc.size(), ros2_medkit_graph_watchdog::AggregatedFault::kMaxDescriptionChars) + << "the aggregated description exceeded the cap"; + ASSERT_GE(desc.size(), marker.size()); + EXPECT_EQ(desc.compare(desc.size() - marker.size(), marker.size(), marker), 0) + << "the filler batch alone was sized (by fill_count_past_cap) to exceed the cap, so this " + "description must be truncated, got tail: " + << desc.substr(desc.size() - std::min(desc.size(), 20)); + EXPECT_EQ(desc.find("/zlate"), std::string::npos) << "\"zlate\" has not gone inactive yet and must " + "not be named"; + } + + // "zlate" goes inactive - the fresh crossing, on a LATER tick, after the cap is already + // full of filler entries that all sort before it. + gate.set_lifecycle_state_for_test(late_id, "inactive"); + const auto failed_before_fresh = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + det->tick(ctx); // ONE tick: "zlate" crosses grace=0 fresh; every filler is already old news + ASSERT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before_fresh + 1)); + std::this_thread::sleep_for(200ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), failed_before_fresh + 1) + << "the fresh crossing must also aggregate into exactly one FAILED"; + + const std::string desc = last_failed_description(kGraphSource); + EXPECT_LE(desc.size(), ros2_medkit_graph_watchdog::AggregatedFault::kMaxDescriptionChars) + << "the aggregated description exceeded the cap"; + EXPECT_EQ(desc.find("node /zlate expected active"), 0u) + << "the freshly-crossed node must open the description, even though its fqn sorts LAST " + "lexicographically among every entry - the old lexicographic emit() would have dropped " + "it entirely, since the filler batch alone already exceeds the cap. Full description: " + << desc; +} + +// The other half of the same ordering rule, and the one the R11 fix above does not reach: a +// DEPARTED node and a PRESENT one crossing `grace` on the SAME tick. Absence-driven crossings +// enter `newly_affected` alongside present ones, so a single lexicographic list lets a node +// that LEFT the graph spend the description budget and truncate away the node that just +// broke - and the operator is then told about the one that is gone instead of the one that +// needs attention. +// +// Paced so both crossings land on one tick: the departed batch is measured not-active once +// and then vanishes, its streaks resuming past the 3-tick absence grace, while the present +// node starts its own streak exactly late enough that the two reach `grace + 1` together. The +// batch is sized by fill_count_past_cap from the REAL detail builder, so its details alone +// exceed the 480-char cap - which is what makes "which one is named" a real choice rather +// than a cosmetic ordering, and the present node's id sorts LAST among them all. +TEST_F(LifecycleExpectationIntegrationTest, PresentNodeCrossingWithDepartedOnesIsNamedAheadOfThem) { + constexpr int kGrace = 5; + std::vector departed_ids; + const std::size_t departed_count = fill_count_past_cap("g", departed_ids); + ASSERT_GT(departed_count, 1u); + const std::string present_id = "zpresent"; // sorts after every "g..." departed node + + std::vector all = departed_ids; + all.push_back(present_id); + nlohmann::json require = nlohmann::json::array(); + for (const auto & id : all) { + require.push_back(id); + } + set_apps(all); // every id known to the gate before the labels are injected + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", require}, {"grace", kGrace}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + for (const auto & id : all) { + gate.set_lifecycle_state_for_test(id, "inactive"); + } + + // Tick 1: the soon-to-depart batch is measured not-active (streak 1). The present node is + // not in the graph yet, so it is not tracked at all. + set_apps(departed_ids); + det->tick(ctx); + // Ticks 2-3: everything absent. Inside the 3-tick absence grace, so the batch's streaks are + // simply held. + set_apps({}); + det->tick(ctx); + det->tick(ctx); + + // Ticks 4-9: only the present node is in the graph, reading not-active. Its streak runs + // 1..6 across these six ticks; the batch's absence passes the grace on tick 5 and its + // streaks resume 2..6 across ticks 5-9. Both reach grace + 1 on tick 9 - the same tick. + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + set_apps({present_id}); + for (int i = 0; i < kGrace + 1; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + ASSERT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before + 1)) + << "nothing crossed at all, so this test never reached the tick it is about"; + std::this_thread::sleep_for(300ms); + + const std::string desc = last_failed_description(kGraphSource); + EXPECT_LE(desc.size(), ros2_medkit_graph_watchdog::AggregatedFault::kMaxDescriptionChars); + ASSERT_NE(desc.find(present_id), std::string::npos) + << "the PRESENT node that crossed on this very tick was truncated out of the description by " + "nodes that had already left the graph - the operator is told about the departures and not " + "about the node that just broke. Full description: " + << desc; + EXPECT_LT(desc.find(present_id), desc.find(departed_ids.front())) + << "the present node is named, but behind a departed one - a departure is never more urgent " + "than a node that is still there and has just gone bad. Full description: " + << desc; +} + +// Change over time: a required node that APPEARS mid-run (graph growth) and then sticks +// inactive must be raised - the entry was configured before the node ever existed, so +// this pins enforcement of late arrivals, not just nodes present from arming. +TEST_F(LifecycleExpectationIntegrationTest, NodeAppearingMidRunThenStuckInactiveRaises) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "b"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + gate.set_lifecycle_state_for_test("a", "active"); + gate.set_lifecycle_state_for_test("b", "inactive"); // label parked before "b" exists - harmless + + for (int i = 0; i < 5; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "raised about a node that is not in the graph yet"; + + // "b" joins the graph and sits inactive. + set_apps({"a", "b"}); + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "a required node appearing mid-run and sticking inactive was never raised"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "/b")); +} + +// A re-bind under the SAME App::id must not keep enforcing the OLD node's label: "k" is +// bound to one managed node, read inactive there, then the id moves to a DIFFERENT node +// whose state nothing has read yet. Whatever label survives that move is what the +// detector enforces - and an unread new binding is unknown (benign), so no raise may +// flow. Uses managed-service snapshots (not the service-less fixture apps) because the +// re-bind is expressed through gate.update() itself. +TEST_F(LifecycleExpectationIntegrationTest, RebindUnderSameAppIdDoesNotEnforceTheOldNodesLabel) { + set_managed_app("k", "/rb_old"); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"k"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + // The OLD binding's label, injected while "k" is still bound to /rb_old. + gate.set_lifecycle_state_for_test("k", "inactive"); + + // The re-bind: same App::id, different node. This gate.update() AFTER the injection is + // deliberate (the one exception to ordering rule 2): with lifecycle services in the + // snapshot the update keeps tracking "k", the injected label stands in for the OLD + // binding's last read, and the NEW binding's own GetState fails into unknown - so + // whatever label survives this update is exactly what the detector will enforce. + set_managed_app("k", "/rb_new"); + gate.update(snapshot_, 9); + + for (int i = 0; i < 15; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + std::this_thread::sleep_for(300ms); // let any in-flight raise land before counting + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "the OLD binding's inactive label was enforced against the re-bound node - a re-bind must " + "re-seed the id, not keep the departed node's label"; +} + +// The withheld-clear guard hands a node over to the presence class the moment it is +// absent past the absence grace. Without that hand-off a never-answered node that +// crashes keeps blocking the clear until its guard entry is pruned - a whole minute at +// the shipped horizons instead of three ticks, so a GRAPH_NODE_INACTIVE about a +// DIFFERENT node takes that long to heal after the blocking node dies. +// +// C2: this pins ONLY the NOT-MANAGED cause of the shared unmeasured clock - "b" carries +// no lifecycle services and is never injected, so lifecycle_state_of("b") is nullopt for +// the whole test. The claim used to be written as if it covered "both causes" on the +// strength of sharing one absence counter; that is not the same as exercising both. See +// UnreadableNodeThatVanishesStopsBlockingTheClearAtTheAbsenceGrace below for the genuine +// optional("") case, proven separately rather than assumed. +TEST_F(LifecycleExpectationIntegrationTest, NotManagedNodeThatVanishesReleasesTheClearBySettlingNotByBeingDropped) { + set_apps({"a", "b"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "b"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "active"); // "b" is matched but NEVER tracked + ASSERT_FALSE(gate.lifecycle_state_of("b").has_value()) + << "\"b\" was unexpectedly tracked - this test would not be exercising NOT-MANAGED"; + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < 5; ++i) { // far inside "b"'s 60-tick not-managed hold + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(300ms); + ASSERT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "the clear was never withheld in the first place, so the release below would prove nothing"; + + // "b" crashes out of the graph. Its clock keeps climbing while it is gone, so the hold + // does NOT end at the absence grace - it ends when the clock MATURES and "b" becomes + // GRAPH_NODE_NOT_MANAGED's own content. The hold is released by SETTLING the node's + // status, never by giving up on it. + set_apps({"a"}); + for (int i = 0; i < kDefaultAbsenceGrace + 3; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(200ms); + ASSERT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "the hold was released at the absence grace, discarding the evidence \"b\" had earned " + "instead of continuing to count it"; + + for (int i = 0; i < kHoldTicks + 5; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + EXPECT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before + 1)) + << "GRAPH_NODE_INACTIVE never cleared even after \"b\"'s clock matured and ownership passed to " + "its own code - the withhold has no release at all"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "/b", kNodeNotManaged)) + << "\"b\" left the graph while unmeasured and was never reported under its own code - the hold " + "released into silence rather than into content"; +} + +// The genuine optional("") twin: "b" IS tracked (via the injection seam) but its label +// is left "" the whole time, exercising the UNREADABLE cause rather than NOT-MANAGED - +// proving the same release-by-settling applies to the unreadable cause too, rather than +// assuming it from the two causes sharing one clock. +TEST_F(LifecycleExpectationIntegrationTest, UnreadableNodeThatVanishesReleasesTheClearBySettlingNotByBeingDropped) { + set_apps({"a", "b"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "b"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "active"); + gate.set_lifecycle_state_for_test("b", ""); // "b" is matched but never answers + ASSERT_TRUE(gate.lifecycle_state_of("b").has_value() && gate.lifecycle_state_of("b")->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + for (int i = 0; i < 5; ++i) { // far inside "b"'s 60-tick unreadable hold + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(300ms); + ASSERT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "the clear was never withheld in the first place, so the release below would prove nothing"; + + // "b" crashes out of the graph, still well inside its unreadable hold. Same rule as the + // not-managed sibling: the clock keeps climbing while it is gone. + set_apps({"a"}); + for (int i = 0; i < kDefaultAbsenceGrace + 3; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(200ms); + ASSERT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), passed_before) + << "the hold was released at the absence grace, discarding the evidence \"b\" had earned"; + + for (int i = 0; i < kHoldTicks + 5; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + EXPECT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before + 1)) + << "GRAPH_NODE_INACTIVE never cleared even after \"b\"'s clock matured under the unreadable cause"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "/b", kNodeUnreadable)) + << "\"b\" left the graph unread and was never reported under GRAPH_NODE_UNREADABLE - the hold " + "released into silence rather than into content"; +} + +// The witness for GRAPH_NODE_UNREADABLE's OWN record across a departure, at the tier that +// sees the wire: "a" is ticked past its unreadable hold FIRST, so the record is genuine +// content (a real FAILED, not merely a withheld GRAPH_NODE_INACTIVE clear) before it +// vanishes. The record must survive the departure - the node's lifecycle promise was never +// verified, and it leaving the graph does not verify it - and the fault must say the node +// is gone rather than keep describing a graph it left. +TEST_F(LifecycleExpectationIntegrationTest, UnreadableNodeAlreadyReportedThatVanishesKeepsItsOwnRecord) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", ""); + ASSERT_TRUE(gate.lifecycle_state_of("a").has_value() && gate.lifecycle_state_of("a")->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeUnreadable)) + << "the unreadable hold never converted into a report, so this test would not exercise an " + "ALREADY-REPORTED node vanishing"; + + // "a" vanishes, having never once been read. + set_apps({}); + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeUnreadable); + for (int i = 0; i < kDefaultAbsenceGrace + 20; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(10ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeUnreadable), passed_before) + << "a node already reported under GRAPH_NODE_UNREADABLE cleared once it vanished - its " + "lifecycle promise is no more verified now than it was while the node was present"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "has since left the graph", kNodeUnreadable)) + << "the record kept describing a present-but-unread node after that node left the graph"; +} + +// The return leg: a node whose unmeasured clock had already matured, that vanishes and then +// comes BACK still unreadable, must stay exactly where it was - one continuous fault, no +// clear on the way out and no fresh raise on the way back. A clock that absence reset would +// produce raise/clear/raise churn on the fault surface for a node whose situation ("still +// cannot be measured") never changed. `poll_for_new`'s budget covers the whole sequence. +TEST_F(LifecycleExpectationIntegrationTest, ReturningUnreadableNodeStaysReportedWithoutClearChurn) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", ""); + ASSERT_TRUE(gate.lifecycle_state_of("a").has_value() && gate.lifecycle_state_of("a")->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeUnreadable)) + << "the unreadable hold never converted into a report, so nothing below is tested"; + + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeUnreadable); + + // "a" vanishes well past the absence grace, then returns, still unreadable - the injected + // label was never changed, so it must still read optional(""). + set_apps({}); + for (int i = 0; i < kDefaultAbsenceGrace + 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + set_apps({"a"}); + ASSERT_TRUE(gate.lifecycle_state_of("a").has_value() && gate.lifecycle_state_of("a")->empty()) + << "\"a\"'s injected state did not survive the absence - this test would not be exercising " + "the return path"; + // Let everything the ticks above put in flight land BEFORE the baseline is taken, or the + // exact-count assertion below would be racing arrivals from the departure leg. + std::this_thread::sleep_for(300ms); + const auto failed_before_return = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable); + + constexpr int kReturnTicks = 10; + for (int i = 0; i < kReturnTicks; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(300ms); + + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeUnreadable), passed_before) + << "the departure-and-return produced a clear, so the operator saw GRAPH_NODE_UNREADABLE heal " + "and re-raise for a node that was never once read"; + // The aggregate is level-triggered, so every one of those ticks emits exactly one FAILED + // while its content is non-empty - no more. An EXTRA one on the return is the churn this + // test is named for, and counting is the only way to see it: "a fault is present" is + // satisfied by the ORIGINAL raise and would pass whatever the return did. + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable), + failed_before_return + kReturnTicks) + << "the return emitted a different number of reports than one per tick - a node coming back " + "still unreadable must continue the one report it already had, not add a fresh raise on top"; + EXPECT_NE(last_failed_description(kGraphSource, kNodeUnreadable).find("/a"), std::string::npos) + << "the returning node stopped being named - checked on the LAST report rather than on any, " + "since 'some report named it' is satisfied by the original raise before the departure"; +} + +// A withhold is silence, so it is explained once per episode - and the episode bookkeeping +// is what makes "once" survive a hold that lives for minutes AND lets a LATER hold speak up +// again. Nothing measured the log line itself: deleting the whole reporting body, or just +// its latch, left every hold test green. +TEST_F(LifecycleExpectationIntegrationTest, WithheldClearIsExplainedOncePerEpisodeNotOncePerProcess) { + const LogCapture log; + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + // Grace wide enough that the second episode below never leaves the pending window. + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 200}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + const std::string needle = "withheld from clearing"; + // Episode 1: "a" is matched and never read. The horizon is 10 ticks, so 12 crosses it. + for (int i = 0; i < 12; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count(needle), 1) << "a hold that outlived the 10-tick reporting horizon was never explained"; + EXPECT_EQ(log.count("no lifecycle label read this run"), 1) << "the log named the wrong reason for an unread node"; + + for (int i = 0; i < 20; ++i) { // the SAME episode, twice as long + det->tick(ctx); + } + EXPECT_EQ(log.count(needle), 1) << "the reason was repeated within one episode - it is once per episode, or " + "a hold that lasts minutes fills the log with the same line"; + + // The episode ends: the label reads active, everything is measured healthy, the clear flows. + gate.set_lifecycle_state_for_test("a", "active"); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "the hold never released, so no second episode can start"; + + // Episode 2, a different reason: the node reads not-active again and its fresh streak is + // young, which withholds on its own. + gate.set_lifecycle_state_for_test("a", "inactive"); + for (int i = 0; i < 12; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count(needle), 2) << "the second withhold episode was never explained - the latch is scoped to " + "the process, so an operator only ever hears about the first one"; + EXPECT_EQ(log.count("measured not-active but still within grace"), 1) + << "the second episode named the first episode's reason"; +} + +// The detector's own no-match warning, at the tier that can see it: an entry that matches +// nothing at all is silent in every other path (it never reaches the violation branch, and +// the presence class never tracks a node that was never present), so this log line is the +// only signal a misspelt entry produces. +TEST_F(LifecycleExpectationIntegrationTest, EntryThatNeverMatchesAnythingIsWarnedAboutInTheLog) { + const LogCapture log; + set_apps({"something_else"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"typoed_name"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + const std::string needle = "has matched no node at all since startup"; + for (int i = 0; i < 12; ++i) { // the default no-match horizon is 10 ticks + det->tick(ctx); + } + EXPECT_EQ(log.count(needle), 1) + << "a require_active entry that can never match produced no fault and no log line either, which is " + "the whole failure mode this warning exists for (0 = never fired at detector level; >1 = the " + "once-per-entry latch is gone)"; +} + +// The other side of that warning: it must not be said about an entry whose node WAS there +// and left. The tracker counts CONSECUTIVE no-match ticks, so a departed node surfaces the +// same way a misspelt entry does - but for it both halves of the sentence are false, and +// the presence class (GRAPH_NODE_DISAPPEARED) owns it, exactly as the never-matched hold +// distinguishes the two cases. +TEST_F(LifecycleExpectationIntegrationTest, EntryWhoseNodeVanishedIsNotAccusedOfNeverComingUp) { + const LogCapture log; + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "active"); + det->tick(ctx); // "a" matches here, and never again + + set_apps({}); + for (int i = 0; i < 15; ++i) { // well past the 10-tick no-match horizon + det->tick(ctx); + } + EXPECT_EQ(log.count("has matched no node at all since startup"), 0) + << "a node that was present and then departed was reported as one that never came up - and as one " + "the presence class cannot see, which is exactly backwards: a departed node is what " + "GRAPH_NODE_DISAPPEARED tracks"; +} + +// `grace` arrives from operator YAML as a 64-bit ROS integer parameter, so a value past the +// int range reaches configure() intact and is only truncated by the narrowing read - 2^32 +// becomes 0, passes a >= 0 check, and silently installs a hair-trigger that reports a node +// on its very first not-active tick. The documented contract is that anything invalid warns +// and keeps the default, so the DEFAULT (5) must be what is actually in force here. +TEST_F(LifecycleExpectationIntegrationTest, WideGraceIsRejectedInsteadOfTruncatedIntoAHairTrigger) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 4294967296}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + gate.set_lifecycle_state_for_test("a", "inactive"); + + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + for (int i = 0; i < 3; ++i) { // a grace truncated to 0 raises on the FIRST of these + det->tick(ctx); + std::this_thread::sleep_for(20ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), failed_before) + << "a grace past the int range was truncated to 0 and reported a node that had been not-active for " + "three ticks - the default grace of 5 is what an invalid value must leave in force"; + + // And the default really is in force, so the expectation is still enforced past it. + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "rejecting the invalid grace disabled the check instead of falling back to the default"; +} + +// ---- the unreadable/not-managed split (matched-but-unread label collapsed two facts +// into one before this slice) ---- + +// Test-plan row 1 + the "instrument measures the claim" shape requirement: a REAL +// managed lifecycle node (GetState/ChangeState services present, via set_managed_app) +// whose seed never answers because no live node listens at its fqn - the gate's REAL +// seeding path, not the set_lifecycle_state_for_test() injection seam every other test +// here uses. The fixture's service-less set_apps() apps are never tracked at all +// (nullopt, the NOT-MANAGED case): a test that believed THAT path exercised "unreadable" +// would prove nothing, which is why this one asserts optional("") was actually measured +// before relying on it. +// A managed node whose GetState genuinely never answers is never a GRAPH_NODE_INACTIVE +// concern at all (it was never CONFIRMED non-active) - it is reported under its own +// GRAPH_NODE_UNREADABLE record instead, once the hold expires. Since "a" is the ONLY +// required node here and it is never confirmed anything, GRAPH_NODE_INACTIVE has nothing +// to say about it either way: no FAILED, ever, though its own level-triggered clear may +// eventually flow once the withhold guard's hold on it lapses - what must NEVER happen is +// GRAPH_NODE_UNREADABLE itself reporting the node healthy. +TEST_F(LifecycleExpectationIntegrationTest, ManagedNodeWhoseGetStateNeverAnswersIsReportedUnreadableNotInactive) { + set_managed_app("a", "/a"); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); // the two gate.update() calls also run the (failing) GetState seed + + const auto state = gate.lifecycle_state_of("a"); + ASSERT_TRUE(state.has_value()) << "\"a\" was never tracked at all - set_managed_app's services did not " + "make the gate treat it as a managed lifecycle node, so this test " + "would be exercising NOT-MANAGED, not UNREADABLE"; + EXPECT_TRUE(state->empty()) << "the seed unexpectedly succeeded with a real label: " << *state; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeUnreadable)) + << "the unreadable hold never converted into GRAPH_NODE_UNREADABLE past its bound"; + // Baseline captured AFTER the first raise: from here on GRAPH_NODE_UNREADABLE must never + // clear, however many more ticks pass, since "a" never gets read. + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeUnreadable); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeUnreadable), passed_before) + << "a managed node whose GetState never answers was reported healthy under GRAPH_NODE_UNREADABLE - " + "an unreadable node must never be cleared, only reported, for as long as it stays unreadable"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "GRAPH_NODE_INACTIVE raised about a node that was never CONFIRMED non-active, only unread"; + + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "could not be read", kNodeUnreadable)); + ASSERT_TRUE(last_failed_severity(kGraphSource, kNodeUnreadable).has_value()); + EXPECT_EQ(*last_failed_severity(kGraphSource, kNodeUnreadable), Fault::SEVERITY_WARN) + << "an unreadable node is an unverified promise, not a confirmed violation, and must not " + "carry the same severity as a confirmed GRAPH_NODE_INACTIVE"; +} + +// ---- test-plan row 3: an unreadable node is reported for as long as it stays +// unreadable, with no window and no expiry ---- + +// The sibling-healing story restated with the OTHER instrument: "a" is a confirmed +// violation that heals almost immediately; "b" is unreadable and never read again for +// the rest of the test. GRAPH_NODE_INACTIVE reflects that "a" - the only OTHER required +// node - has been healthy ever since it healed (proven elsewhere by +// ConfirmedNodeHealsWhileUnreadableSiblingStaysPresentClearsInactiveOnly above); this +// test instead proves GRAPH_NODE_UNREADABLE's own content survives far longer than the +// old, now-deleted content window ever tolerated - many ticks past where it used to +// silently drop out - with no PASSED reaching the service in between. +TEST_F(LifecycleExpectationIntegrationTest, UnreadableNodeStaysReportedWithNoExpiryAsLongAsItStaysUnreadable) { + set_apps({"a", "b"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "b"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "inactive"); // confirmed violation, heals below + gate.set_lifecycle_state_for_test("b", ""); // unreadable, NEVER read again in this test + ASSERT_TRUE(gate.lifecycle_state_of("b").has_value() && gate.lifecycle_state_of("b")->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx)) + << "\"a\" never raised, so nothing below can be attributed to it healing"; + gate.set_lifecycle_state_for_test("a", "active"); + + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeUnreadable)) + << "\"b\"'s unreadable hold never converted into GRAPH_NODE_UNREADABLE"; + + // Tick FAR past where the old, now-deleted content window (another kHoldTicks past the + // hold) would have silently dropped "b" out of the record. "b" is still never read. + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeUnreadable); + for (int i = 0; i < kHoldTicks * 3; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "/b", kNodeUnreadable)) + << "an unreadable node dropped out of its own record even though it was never read - there is " + "no expiry on this fault, only on the initial hold"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeUnreadable), passed_before) + << "GRAPH_NODE_UNREADABLE cleared while \"b\" was still, and had always been, unreadable"; +} + +// A genuine read while the node is being reported resets the counter entirely, and a +// LATER unreadable spell must earn a full fresh hold before it is reported again - none +// of the earlier progress may leak across the read. +TEST_F(LifecycleExpectationIntegrationTest, ReadWhileReportedUnreadableResetsAndRequiresAFreshHold) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + gate.set_lifecycle_state_for_test("a", ""); + ASSERT_TRUE(gate.lifecycle_state_of("a").has_value() && gate.lifecycle_state_of("a")->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeUnreadable)) + << "the unreadable hold never converted into a report, so nothing below is tested"; + + // Tick a while longer, then genuinely read the node. + for (int i = 0; i < 20; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + gate.set_lifecycle_state_for_test("a", "active"); + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeUnreadable); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx, kNodeUnreadable)) + << "the read never cleared GRAPH_NODE_UNREADABLE"; + + // Go unreadable again. If the earlier progress had survived the read, this would + // convert to content again almost immediately; it must instead need a full fresh hold. + gate.set_lifecycle_state_for_test("a", ""); + ASSERT_TRUE(gate.lifecycle_state_of("a").has_value() && gate.lifecycle_state_of("a")->empty()) + << "the re-injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable); + for (int i = 0; i < kHoldTicks - 5; ++i) { // comfortably below a FRESH hold on its own + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(200ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable), failed_before) + << "the later unreadable spell converted to a report before a FRESH hold elapsed - the " + "earlier spell's progress leaked across the intervening read"; + + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx, kNodeUnreadable)) + << "the fresh spell after the read never converted to a report at all"; +} + +// Test-plan row 2 (and the config-sweep hold boundary at exactly kUnmeasuredHoldTicks and +// kUnmeasuredHoldTicks + 1): the hold releases INTO a GRAPH_NODE_UNREADABLE report on the +// EXACT tick, never into a GRAPH_NODE_INACTIVE clear one tick early. Driven through the +// injection seam (not set_managed_app's real, variable-latency seeding) so the boundary +// tick is pinned exactly - the same tradeoff BlinkAndUnreadReseedDoNotRestartTheViolationCount +// makes for the tracker's own tick-exact claims. +TEST_F(LifecycleExpectationIntegrationTest, UnreadableHoldReleasesIntoAReportOnTheExactTickNotIntoAClear) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + // Inject AFTER the last gate.update() (ordering rule 2); never call gate.update() + // again, so the label stays exactly "" for the rest of the test. + gate.set_lifecycle_state_for_test("a", ""); + ASSERT_TRUE(gate.lifecycle_state_of("a").has_value()); + ASSERT_TRUE(gate.lifecycle_state_of("a")->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + + for (int i = 0; i < kHoldTicks; ++i) { // exactly the tolerated hold + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable), 0u) + << "the unreadable hold converted into a report before its bound - the boundary must be " + "exactly kUnmeasuredHoldTicks, not one tick early"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "GRAPH_NODE_INACTIVE raised about a node that was never CONFIRMED non-active, only unread"; + + det->tick(ctx); // the (kHoldTicks + 1)th tick: the exact boundary + EXPECT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, 1, kNodeUnreadable)) + << "the hold outlived its documented bound - it must release into a GRAPH_NODE_UNREADABLE " + "report exactly one tick past kUnmeasuredHoldTicks"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "could not be read", kNodeUnreadable)); + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "/a", kNodeUnreadable)); +} + +// Test-plan row 4 + row 5's "never before" half: the NOT-MANAGED leg shares the unmeasured +// clock with the UNREADABLE leg (that sharing is the whole point of this slice's redesign +// - a cause-blind clock closes the alternation both used to be individually blind to), so +// past the hold it now converts into its OWN fault code (GRAPH_NODE_NOT_MANAGED) exactly +// as UNREADABLE converts into GRAPH_NODE_UNREADABLE - it no longer releases into silence. +// GRAPH_NODE_INACTIVE's clear still flows (a not-managed node was never a confirmed +// violation, whichever code ends up naming it), and GRAPH_NODE_UNREADABLE must never fire +// for it - the cause is NOT-MANAGED, never UNREADABLE. +TEST_F(LifecycleExpectationIntegrationTest, + NotManagedEntryPastTheHoldRaisesTheNotManagedCodeNeverInactiveOrUnreadable) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + // NO label injection, ever: "a" has no lifecycle services (set_apps, not + // set_managed_app), so lifecycle_state_of("a") stays nullopt - genuinely NOT-MANAGED. + ASSERT_FALSE(gate.lifecycle_state_of("a").has_value()) + << "\"a\" was unexpectedly tracked - this test would not be exercising NOT-MANAGED"; + + // Below the hold: GRAPH_NODE_NOT_MANAGED must never raise before its clock matures. + for (int i = 0; i < kHoldTicks; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(200ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeNotManaged), 0u) + << "GRAPH_NODE_NOT_MANAGED raised before its clock crossed the hold"; + + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeNotManaged)) + << "GRAPH_NODE_NOT_MANAGED never raised once the not-managed clock matured past its hold"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "/a", kNodeNotManaged)); + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "not a managed lifecycle node", kNodeNotManaged)); + ASSERT_TRUE(last_failed_severity(kGraphSource, kNodeNotManaged).has_value()); + EXPECT_EQ(*last_failed_severity(kGraphSource, kNodeNotManaged), Fault::SEVERITY_WARN) + << "a not-managed node is an unverified promise, not a confirmed violation"; + + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "a not-managed entry produced a GRAPH_NODE_INACTIVE FAILED - it was never a confirmed violation"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable), 0u) + << "a not-managed entry (nullopt) was reported under GRAPH_NODE_UNREADABLE, which is only for " + "a genuinely tracked lifecycle node whose label is unread (optional(\"\"))"; +} + +// Shape "Change": a node read once (which resets the unmeasured clock to zero, same as +// any real measurement) that LATER goes unreadable must still eventually surface. There +// is no special-casing needed for this - the unmeasured clock simply climbs from zero +// like a first-contact node's would: a respawned incarnation whose fresh GetState never +// answers is exactly R7's "development is the common case" scenario, and nothing about +// an earlier successful read on a dead incarnation can exempt it. +TEST_F(LifecycleExpectationIntegrationTest, NodeThatWasReadThenGoesUnreadableIsEventuallyReported) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + // Read once: resets both clocks to zero. + gate.set_lifecycle_state_for_test("a", "active"); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "the healthy baseline never cleared, so nothing below proves anything"; + + // Now it goes - and stays - unreadable, as if respawned into an incarnation whose + // GetState never answers. + gate.set_lifecycle_state_for_test("a", ""); + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable); + for (int i = 0; i < kHoldTicks; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable), failed_before) + << "reported before the hold expired"; + + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx, kNodeUnreadable)) + << "a node that was read once and later went unreadable never surfaced - an earlier " + "successful read must not silence a later, genuinely sustained unreadable spell"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "could not be read", kNodeUnreadable)); +} + +// Test-plan row 5: a read releases everything - the unreadable fault clears, AND the node +// returns to ordinary handling (the tracker's own grace-based tracking), not to some +// third, permanently-exempt state. A node already reported unreadable that is FINALLY read +// must stop being reported under GRAPH_NODE_UNREADABLE and clear normally; from there, if +// it goes non-active again, GRAPH_NODE_INACTIVE must enforce it exactly like any other +// node's ordinary streak - proving the node is back in the tracker's normal care, not +// stuck in some leftover unreadable-adjacent state. +TEST_F(LifecycleExpectationIntegrationTest, UnreadableContentClearsOnceTheNodeIsFinallyRead) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", ""); + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx, kNodeUnreadable)) + << "the unreadable hold never converted into a report, so the clear below would prove nothing"; + + gate.set_lifecycle_state_for_test("a", "active"); + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeUnreadable); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx, kNodeUnreadable)) + << "an unreadable node that was FINALLY read active never cleared under GRAPH_NODE_UNREADABLE - " + "content must not be a one-way latch"; + + // Ordinary handling resumes: a real read (not another unread spell) that goes non-active + // is enforced by the tracker's usual grace-based streak, ending in GRAPH_NODE_INACTIVE - + // never GRAPH_NODE_UNREADABLE, which this node has left behind for good this run. + gate.set_lifecycle_state_for_test("a", "inactive"); + const auto failed_before_inactive = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + const auto unreadable_failed_before_inactive = + count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before_inactive, *det, ctx)) + << "the node never returned to ordinary GRAPH_NODE_INACTIVE handling after its unreadable " + "content cleared"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable), + unreadable_failed_before_inactive) + << "a node that read active then inactive was reported under GRAPH_NODE_UNREADABLE again, which " + "it never re-entered"; +} + +// Shape "Change": reconfiguration while the unreadable clock is live. configure() +// rebuilds the tracker from scratch, so progress toward the hold must not survive a +// reconfigure - reapplying the SAME config must not accidentally convert a below-hold +// clock into a report using ticks counted under the OLD tracker instance. +TEST_F(LifecycleExpectationIntegrationTest, ReconfigureMidUnreadableHoldRestartsTheCountInsteadOfCarryingOverProgress) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + const nlohmann::json config{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}; + det->configure(config); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + gate.set_lifecycle_state_for_test("a", ""); + + constexpr int kHalfway = kHoldTicks - 20; // comfortably below the hold on its own + for (int i = 0; i < kHalfway; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + det->configure(config); // rebuilds the tracker: progress so far must not survive this + + // If the reconfigure did NOT reset the count, kHalfway + kHalfway = 2*(kHoldTicks-20) + // > kHoldTicks would already have converted partway through this second phase. It must + // not have: the fresh tracker's clock restarts at 0. + for (int i = 0; i < kHalfway; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable), 0u) + << "the unreadable count survived the reconfigure - two below-hold phases summed past the " + "bound instead of each restarting at 0"; + + // And the counter is alive, not broken by the reconfigure: it still converts once IT + // alone crosses the hold. + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeUnreadable)) + << "the unreadable counter never converted at all after the reconfigure"; +} + +// C6: every reconfigure test above calls configure() twice with IDENTICAL json. This one +// actually CHANGES `grace` while a streak is live: "a" accumulates 3 present-and-inactive +// ticks under grace=5 (comfortably below it), then the config shrinks grace to 1. +// configure() rebuilds the tracker from scratch (LifecycleExpectationTracker's own +// misses_ map is gone with the old instance), so the correct behaviour is that the +// 3-tick streak does NOT carry over and does NOT retroactively cross the new, smaller +// grace - a reconfigure is a fresh start, not a re-evaluation of history against a +// changed threshold. The fresh streak under the NEW grace then crosses in 2 ticks, not 5. +TEST_F(LifecycleExpectationIntegrationTest, ReconfigureShrinkingGraceMidStreakRestartsTheStreakUnderTheNewGrace) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 5}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + gate.set_lifecycle_state_for_test("a", "inactive"); + + for (int i = 0; i < 3; ++i) { // comfortably below grace=5 + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(200ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "raised before crossing the ORIGINAL grace - the setup itself is broken"; + + // Shrink grace while the 3-tick streak is live. + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "the reconfigure retroactively counted the old streak against the new, smaller grace - " + "a reconfigure must be a fresh start, not a re-evaluation of history"; + + // The fresh streak under grace=1 crosses within 2 ticks. + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx)) + << "the new (smaller) grace was never enforced after the reconfigure"; +} + +// C6's other half: a LIVE pending hold (a below-grace streak the withheld-clear guard is +// currently honouring) must not survive the entry that owns it being dropped from +// require_active entirely. "a" starts a below-grace streak (pending, not yet a fault); +// "b" is healthy. Before this reconfigure, GRAPH_NODE_INACTIVE has emitted nothing at +// all - the pending leg withholds even its clear, per the withheld-clear guard. +// GRAPH_NODE_UNREADABLE is unaffected by that guard (it is independent, see the class +// doc) and legitimately keeps sending its own routine clear every tick since nothing is +// unreadable here, so the instrument below is scoped to GRAPH_NODE_INACTIVE specifically, +// not every request of any kind. Dropping "a" from the config must free "b"'s +// already-healthy state to be reported, not leave the record stuck forever waiting on a +// node no longer checked. +TEST_F(LifecycleExpectationIntegrationTest, ReconfigureDroppingAnEntryWithALivePendingHoldStopsWithholdingForIt) { + set_apps({"a", "b"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "b"})}, {"grace", 5}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + gate.set_lifecycle_state_for_test("a", "inactive"); // starts a below-grace (pending) streak + gate.set_lifecycle_state_for_test("b", "active"); // healthy from the start + + for (int i = 0; i < 3; ++i) { // "a" pending (3 < grace 5), never crosses + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(200ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED) + + count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), + 0u) + << "\"a\"'s live pending hold must withhold even GRAPH_NODE_INACTIVE's clear before the " + "reconfigure below - if nothing was ever withheld, dropping the entry proves nothing"; + + // Drop "a" from require_active entirely while its pending hold is live. + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"b"})}, {"grace", 5}}); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "\"a\"'s pending hold survived being dropped from require_active - the record stayed " + "withheld waiting on a node no longer checked, instead of reflecting that \"b\" (the " + "only node still required) has been healthy the whole time"; +} + +// GRAPH_NODE_UNREADABLE's fixed severity, on its own: an unread label is an UNVERIFIED +// promise, not a CONFIRMED violation, and (being non-CRITICAL) must settle through the +// ordinary debounce path rather than bypassing it. +TEST_F(LifecycleExpectationIntegrationTest, OnlyUnreadableContentCarriesWarnSeverity) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + gate.set_lifecycle_state_for_test("a", ""); + + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeUnreadable)) + << "the unreadable hold never converted into a report"; + ASSERT_TRUE(last_failed_severity(kGraphSource, kNodeUnreadable).has_value()); + EXPECT_EQ(*last_failed_severity(kGraphSource, kNodeUnreadable), Fault::SEVERITY_WARN) + << "GRAPH_NODE_UNREADABLE must carry WARN, not the confirmed severity"; +} + +// Test-plan row 2: the two faults carry different severities, ERROR and WARN, and +// NEITHER changes with the other's content - each is a fixed member (like orphan_detector +// and param_drift_detector's AggregatedFault), not a per-tick choice on one shared record +// the way this used to work before the split. +TEST_F(LifecycleExpectationIntegrationTest, InactiveAndUnreadableSeveritiesAreFixedAndIndependentOfTheOthersContent) { + set_apps({"a", "b"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", "b"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "inactive"); // confirmed violation + gate.set_lifecycle_state_for_test("b", ""); // unreadable, sustained + + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx)) + << "\"a\" never raised, so nothing below can be attributed to it"; + ASSERT_TRUE(last_failed_severity(kGraphSource).has_value()); + EXPECT_EQ(*last_failed_severity(kGraphSource), Fault::SEVERITY_ERROR) + << "GRAPH_NODE_INACTIVE must carry ERROR from the moment it is raised"; + + // Let "b" cross its own hold too, so it starts its own, SEPARATE GRAPH_NODE_UNREADABLE + // record before the independence claim below is tested. + for (int i = 0; i < kHoldTicks + 5; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + ASSERT_TRUE(any_failed_desc_contains(kGraphSource, "/b", kNodeUnreadable)) + << "\"b\"'s unreadable hold never converted, so the independence claim below is untested"; + ASSERT_TRUE(last_failed_severity(kGraphSource, kNodeUnreadable).has_value()); + EXPECT_EQ(*last_failed_severity(kGraphSource, kNodeUnreadable), Fault::SEVERITY_WARN) + << "GRAPH_NODE_UNREADABLE must carry WARN"; + ASSERT_TRUE(last_failed_severity(kGraphSource).has_value()); + EXPECT_EQ(*last_failed_severity(kGraphSource), Fault::SEVERITY_ERROR) + << "GRAPH_NODE_INACTIVE's severity changed merely because GRAPH_NODE_UNREADABLE started " + "reporting a sibling node - the two faults must be independent"; + + // "a" heals: GRAPH_NODE_INACTIVE clears. "b" is still unreadable throughout, so + // GRAPH_NODE_UNREADABLE keeps raising WARN, unaffected by the unrelated clear. + gate.set_lifecycle_state_for_test("a", "active"); + const auto passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED); + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, passed_before, *det, ctx)) + << "GRAPH_NODE_INACTIVE never cleared once its only confirmed violation healed"; + ASSERT_TRUE(last_failed_severity(kGraphSource, kNodeUnreadable).has_value()); + EXPECT_EQ(*last_failed_severity(kGraphSource, kNodeUnreadable), Fault::SEVERITY_WARN) + << "GRAPH_NODE_UNREADABLE's severity changed merely because the unrelated GRAPH_NODE_INACTIVE " + "cleared"; +} + +// Scale: more unreadable nodes than the description cap can name - mirrors +// TwentyFiveStuckNodesRaiseOneCappedFault, but for content the guard ages in rather than +// content the tracker raises directly. +TEST_F(LifecycleExpectationIntegrationTest, ManyUnreadableNodesAggregateIntoOneCappedWarnFault) { + std::vector ids; + nlohmann::json require = nlohmann::json::array(); + for (int i = 0; i < 25; ++i) { + const std::string id = "a" + std::string(i < 10 ? "0" : "") + std::to_string(i); + ids.push_back(id); + require.push_back(id); + } + set_apps(ids); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", require}, {"grace", 0}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + for (const auto & id : ids) { + gate.set_lifecycle_state_for_test(id, ""); + } + + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable); + for (int i = 0; i < kHoldTicks + 5; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + ASSERT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before + 1, kNodeUnreadable)); + std::this_thread::sleep_for(200ms); + + const std::string desc = last_failed_description(kGraphSource, kNodeUnreadable); + const std::string marker = "..."; + EXPECT_LE(desc.size(), ros2_medkit_graph_watchdog::AggregatedFault::kMaxDescriptionChars) + << "the aggregated unreadable description exceeded the cap"; + ASSERT_GE(desc.size(), marker.size()); + EXPECT_EQ(desc.compare(desc.size() - marker.size(), marker.size(), marker), 0) + << "a description this far past the cap must end in the truncation marker"; + ASSERT_TRUE(last_failed_severity(kGraphSource, kNodeUnreadable).has_value()); + EXPECT_EQ(*last_failed_severity(kGraphSource, kNodeUnreadable), Fault::SEVERITY_WARN) + << "25 unreadable nodes must still carry WARN, not ERROR"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "GRAPH_NODE_INACTIVE raised about nodes that were never CONFIRMED non-active, only unread"; +} + +// C1/C3: the tracker's unreadable-detail builder (lifecycle_expectation_tracker.hpp, +// LifecycleExpectationTracker::update()'s matured-unreadable branch) re-applies the +// whole-detail backstop for the same reason the confirmed-violation detail does - the +// fqn and the "required by" list are graph- and config-controlled length - but nothing +// exercised it before this case. With only ONE affected node, AggregatedFault's own +// 480-char cap never engages (one entry at kMaxLifecycleDetailChars=150 is far under +// it), so a bounded description here proves the unreadable detail's OWN trim_to call +// did the work, not a coincidence of the outer cap. +TEST_F(LifecycleExpectationIntegrationTest, PathologicalFqnOnAnUnreadableNodeIsCappedByTheDetailBackstop) { + const std::string huge_fqn = "/" + std::string(2000, 'n'); + set_apps_raw({{"a", huge_fqn}}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + // The entry must equal the fqn (or its leaf) to match, so this also pins a + // pathologically long single "required by" entry, not just a long fqn. + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({huge_fqn})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + gate.set_lifecycle_state_for_test("a", ""); // unreadable, sustained + ASSERT_TRUE(gate.lifecycle_state_of("a").has_value() && gate.lifecycle_state_of("a")->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeUnreadable)) + << "the unreadable hold never converted into a report, so the backstop below is untested"; + const std::string desc = last_failed_description(kGraphSource, kNodeUnreadable); + EXPECT_LE(desc.size(), ros2_medkit_graph_watchdog::kMaxLifecycleDetailChars) + << "the unreadable detail's whole-detail backstop did not bound a pathological fqn/entry"; + // The fqn is 2001 chars and appears right after "node ", ahead of "expected active but + // its lifecycle state could not be read" in the fixed message - so with a fqn this much + // longer than kMaxLifecycleDetailChars, trim_to's head-keeping behaviour means the fixed + // tail text CANNOT survive; asserting for it would be asserting something structurally + // impossible, not proving the backstop works. What DOES prove it: the string starts with + // the fixed "node " prefix (trim_to keeps the head) and ends in the "..." marker (proving + // truncation actually fired, not that the detail happened to already fit). + const std::string marker = "..."; + EXPECT_EQ(desc.find("node "), 0u) << "the fixed prefix did not survive at the head of the trim"; + ASSERT_GE(desc.size(), marker.size()); + EXPECT_EQ(desc.compare(desc.size() - marker.size(), marker.size(), marker), 0) + << "a detail this far past the cap must end in the truncation marker, got tail: " + << desc.substr(desc.size() - std::min(desc.size(), 20)); +} + +// A node entering GRAPH_NODE_UNREADABLE's content because its hold just expired must +// order as NEW exactly like a tracker-confirmed violation orders for GRAPH_NODE_INACTIVE +// (ANodeCrossingAfterTheCapIsFullIsNamedOverOlderEntries above) - the operator's question +// is "what changed". Twenty-five "f..." fillers go unreadable together and fill +// GRAPH_NODE_UNREADABLE's own 480-char cap on their own (same shape and count as +// ManyUnreadableNodesAggregateIntoOneCappedWarnFault, proven there to overflow the cap for +// this exact detail string); "zunread" starts READ (active, so its own unmeasured clock +// never moves) and only goes unreadable once the filler batch has already converted - +// needing the REAL ReliabilityGate to produce a matched-but-unread label (optional("")), +// which is why this case is integration-only, unlike the label-trim cases in the tracker +// unit tests. +TEST_F(LifecycleExpectationIntegrationTest, AnUnreadableNodeAgingIntoContentOrdersAsNewLikeAConfirmedOne) { + std::vector ids; + ids.reserve(25); + for (int i = 0; i < 25; ++i) { + ids.push_back("f" + std::string(i < 10 ? "0" : "") + std::to_string(i)); + } + const std::string late_id = "zunread"; // sorts after every "f..." filler, lexicographically last + ids.push_back(late_id); + + nlohmann::json require = nlohmann::json::array(); + for (const auto & id : ids) { + require.push_back(id); + } + set_apps(ids); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", require}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + for (const auto & id : ids) { + if (id != late_id) { + gate.set_lifecycle_state_for_test(id, ""); // unreadable from tick 1 + } + } + gate.set_lifecycle_state_for_test(late_id, "active"); // read and healthy, for now + + // Run the filler batch through its hold so it converts into GRAPH_NODE_UNREADABLE's + // content together, well before "zunread" is ever touched. + for (int i = 0; i < kHoldTicks + 5; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + ASSERT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, 1, kNodeUnreadable)); + { + const std::string desc = last_failed_description(kGraphSource, kNodeUnreadable); + const std::string marker = "..."; + ASSERT_GE(desc.size(), marker.size()); + EXPECT_EQ(desc.compare(desc.size() - marker.size(), marker.size(), marker), 0) + << "the filler batch alone was sized (25 unreadable nodes, proven to overflow the cap by " + "ManyUnreadableNodesAggregateIntoOneCappedWarnFault) to exceed the cap, so this " + "description must be truncated"; + EXPECT_EQ(desc.find("/zunread"), std::string::npos) + << "\"zunread\" has not gone unreadable yet and must not be named"; + } + + // "zunread" goes unreadable now - the fresh crossing, on a LATER tick, after the cap is + // already full of filler entries that all sort before it. + gate.set_lifecycle_state_for_test(late_id, ""); + ASSERT_TRUE(gate.lifecycle_state_of(late_id).has_value() && gate.lifecycle_state_of(late_id)->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + + for (int i = 0; i < kHoldTicks; ++i) { // exactly the tolerated hold, one tick before conversion + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + EXPECT_FALSE(any_failed_desc_contains(kGraphSource, "/zunread", kNodeUnreadable)) + << "still inside \"zunread\"'s own hold - must not be content yet"; + + det->tick(ctx); // the exact crossing tick: converts "zunread" into content for the first time + std::this_thread::sleep_for(300ms); + const std::string desc = last_failed_description(kGraphSource, kNodeUnreadable); + EXPECT_LE(desc.size(), ros2_medkit_graph_watchdog::AggregatedFault::kMaxDescriptionChars) + << "the aggregated description exceeded the cap"; + EXPECT_EQ(desc.find("node /zunread expected active but its lifecycle state could not be read"), 0u) + << "the freshly-converted unreadable node must open the description, even though its fqn " + "sorts LAST lexicographically among every entry. Full description: " + << desc; +} + +// The wire string has no witness anywhere else in this file: every helper and assertion +// above compares against the constant kNodeUnreadable, so a typo inside +// graph_fault_codes.hpp would propagate through the whole suite unnoticed - every test +// would still be comparing the (equally wrong) constant against itself. +// GRAPH_NODE_INACTIVE does not share this exposure: the Python e2e hardcodes +// "GRAPH_NODE_INACTIVE" as its own literal (`FAULT_CODE` in +// test_lifecycle_expectation_e2e.test.py), independent of this header. This is +// GRAPH_NODE_UNREADABLE's equivalent witness: the literal below is hand-typed, never +// read from kNodeUnreadable on either side. +TEST_F(LifecycleExpectationIntegrationTest, UnreadableFaultCodeOnTheWireIsTheLiteralGraphNodeUnreadable) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", ""); + ASSERT_TRUE(gate.lifecycle_state_of("a").has_value() && gate.lifecycle_state_of("a")->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, "GRAPH_NODE_UNREADABLE")) + << "no FAILED request ever carried the literal wire string \"GRAPH_NODE_UNREADABLE\" - " + "either the raise never happened, or kNodeUnreadable no longer matches this literal"; +} + +// The same witness for the code this slice adds: the literal below is hand-typed, never +// read from kNodeNotManaged on either side, so a typo in graph_fault_codes.hpp would +// propagate through the whole suite unnoticed without this. +TEST_F(LifecycleExpectationIntegrationTest, NotManagedFaultCodeOnTheWireIsTheLiteralGraphNodeNotManaged) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + // No injection, ever: "a" has no lifecycle services, so lifecycle_state_of("a") stays + // nullopt - genuinely NOT-MANAGED. + ASSERT_FALSE(gate.lifecycle_state_of("a").has_value()) + << "\"a\" was unexpectedly tracked - this test would not be exercising NOT-MANAGED"; + + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, "GRAPH_NODE_NOT_MANAGED")) + << "no FAILED request ever carried the literal wire string \"GRAPH_NODE_NOT_MANAGED\" - " + "either the raise never happened, or kNodeNotManaged no longer matches this literal"; +} + +// R28, through the REAL gate: a node CONFIRMED inactive (a real measurement, past grace) +// that then goes unreadable long enough to mature must stop being reported under +// GRAPH_NODE_INACTIVE entirely - not merely "not in this tick's affected map", but +// actually CLEAR - because the violation streak is RELEASED the instant the unmeasured +// clock takes ownership. Two prior review passes found the old (pre-redesign) behaviour +// wrong here: the confirmed streak used to be merely held, never released, so +// GRAPH_NODE_INACTIVE's clear stayed withheld forever once a confirmed node went +// unreadable and never came back - this is the live pin that it does not happen anymore. +TEST_F(LifecycleExpectationIntegrationTest, ConfirmedInactiveNodeGoingUnreadableReleasesInactiveAndTransfersOwnership) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "inactive"); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeInactive)) + << "\"a\" never confirmed inactive, so nothing below proves a CONFIRMED node's ownership " + "was ever transferred"; + + // Now unreadable, sustained past the hold: the unmeasured clock matures and takes + // ownership, releasing the violation streak. + gate.set_lifecycle_state_for_test("a", ""); + ASSERT_TRUE(gate.lifecycle_state_of("a").has_value() && gate.lifecycle_state_of("a")->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + const auto inactive_passed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED, kNodeInactive); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeUnreadable)) + << "the unmeasured clock never matured, so ownership was never transferred"; + EXPECT_TRUE( + wait_for_count(kGraphSource, ReportFault::Request::EVENT_PASSED, inactive_passed_before + 1, kNodeInactive)) + << "GRAPH_NODE_INACTIVE did not clear once the unmeasured clock matured and took " + "ownership - the violation streak must be RELEASED, not merely held, once another " + "code owns the node"; + + // Returning to inactive must RE-EARN grace from zero: it must NOT re-cross immediately + // on the strength of the old, released streak. + gate.set_lifecycle_state_for_test("a", "inactive"); + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeInactive); + det->tick(ctx); // 1 <= grace(1): must NOT be enough on its own + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeInactive), failed_before) + << "the returning node re-raised GRAPH_NODE_INACTIVE on its very first inactive tick - " + "the released streak must re-earn grace from zero, not resume where it left off"; + EXPECT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx, kNodeInactive)) + << "the streak never re-crossed grace at all after genuinely re-earning it"; +} + +// Test-plan row 2, the not-managed twin of OnlyUnreadableContentCarriesWarnSeverity: an +// unmeasured cause is an UNVERIFIED promise, not a CONFIRMED violation, whichever of the +// two it is. +TEST_F(LifecycleExpectationIntegrationTest, OnlyNotManagedContentCarriesWarnSeverity) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + ASSERT_FALSE(gate.lifecycle_state_of("a").has_value()) + << "\"a\" was unexpectedly tracked - this test would not be exercising NOT-MANAGED"; + + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, 0, *det, ctx, kNodeNotManaged)) + << "the not-managed clock never converted into a report"; + ASSERT_TRUE(last_failed_severity(kGraphSource, kNodeNotManaged).has_value()); + EXPECT_EQ(*last_failed_severity(kGraphSource, kNodeNotManaged), Fault::SEVERITY_WARN) + << "GRAPH_NODE_NOT_MANAGED must carry WARN, not the confirmed severity"; +} + +// Scale + config sweep, the not-managed twin of ManyUnreadableNodesAggregateIntoOneCappedWarnFault: +// more not-managed nodes than the description cap can name aggregate into ONE capped fault. +TEST_F(LifecycleExpectationIntegrationTest, ManyNotManagedNodesAggregateIntoOneCappedWarnFault) { + std::vector ids; + nlohmann::json require = nlohmann::json::array(); + for (int i = 0; i < 25; ++i) { + const std::string id = "a" + std::string(i < 10 ? "0" : "") + std::to_string(i); + ids.push_back(id); + require.push_back(id); + } + set_apps(ids); // plain apps, no lifecycle services: every one is genuinely NOT-MANAGED + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", require}, {"grace", 0}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeNotManaged); + for (int i = 0; i < kHoldTicks + 5; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + ASSERT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before + 1, kNodeNotManaged)); + std::this_thread::sleep_for(200ms); + + const std::string desc = last_failed_description(kGraphSource, kNodeNotManaged); + const std::string marker = "..."; + EXPECT_LE(desc.size(), ros2_medkit_graph_watchdog::AggregatedFault::kMaxDescriptionChars) + << "the aggregated not-managed description exceeded the cap"; + ASSERT_GE(desc.size(), marker.size()); + EXPECT_EQ(desc.compare(desc.size() - marker.size(), marker.size(), marker), 0) + << "a description this far past the cap must end in the truncation marker"; + ASSERT_TRUE(last_failed_severity(kGraphSource, kNodeNotManaged).has_value()); + EXPECT_EQ(*last_failed_severity(kGraphSource, kNodeNotManaged), Fault::SEVERITY_WARN) + << "25 not-managed nodes must still carry WARN, not ERROR"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "GRAPH_NODE_INACTIVE raised about nodes that were never CONFIRMED non-active, only unmeasured"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable), 0u) + << "GRAPH_NODE_UNREADABLE raised for nodes that were NOT-MANAGED, never a genuinely tracked " + "lifecycle node with an unread label"; +} + +// ---- Absence CONTINUES whatever the node was already doing, through the real gate ---- +// +// The three tests below drive the shape a node in a RESTART LOOP produces on a real graph: +// present for one tick, then gone for a run of ticks, forever. That is what a crash-looping +// node looks like from the snapshot - start, crash, respawn delay, start - and it is the +// case this detector most exists to catch, so a clock that absence discards makes exactly +// that node permanently invisible. Each absence run is deliberately LONGER than +// kDefaultAbsenceGrace, i.e. past the blink tolerance, which is where the evidence used to +// be thrown away. The unit tier proves the pacing; these prove what leaves the DETECTOR. + +TEST_F(LifecycleExpectationIntegrationTest, UnreadableNodeInARestartLoopIsStillReported) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 5}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", ""); + ASSERT_TRUE(gate.lifecycle_state_of("a").has_value() && gate.lifecycle_state_of("a")->empty()) + << "the injection did not produce optional(\"\") - this test would not be exercising the " + "unreadable path"; + + bool raised = false; + for (int cycle = 0; cycle < kHoldTicks + 5 && !raised; ++cycle) { + set_apps({"a"}); + det->tick(ctx); + std::this_thread::sleep_for(2ms); + set_apps({}); // ABSENT: gone from the snapshot entirely, not merely unread + for (int i = 0; i < kDefaultAbsenceGrace + 2 && !raised; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(2ms); + raised = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable) > 0; + } + raised = raised || count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable) > 0; + } + EXPECT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, 1, kNodeUnreadable)) + << "a node that is unreadable whenever it is present and absent the rest of the time never " + "raised GRAPH_NODE_UNREADABLE - every absence run past the grace discarded the evidence, " + "so a restart loop evades this detector forever"; +} + +TEST_F(LifecycleExpectationIntegrationTest, NotManagedNodeInARestartLoopIsStillReported) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 5}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + ASSERT_FALSE(gate.lifecycle_state_of("a").has_value()) + << "\"a\" was unexpectedly tracked - this test would not be exercising NOT-MANAGED"; + + bool raised = false; + for (int cycle = 0; cycle < kHoldTicks + 5 && !raised; ++cycle) { + set_apps({"a"}); + det->tick(ctx); + std::this_thread::sleep_for(2ms); + set_apps({}); + for (int i = 0; i < kDefaultAbsenceGrace + 2 && !raised; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(2ms); + raised = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeNotManaged) > 0; + } + raised = raised || count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeNotManaged) > 0; + } + EXPECT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, 1, kNodeNotManaged)) + << "a node carrying no tracked lifecycle whenever it is present, absent the rest of the time, " + "never raised GRAPH_NODE_NOT_MANAGED"; +} + +// The pair no test at any tier covered: a MEASURED not-active read alternating with a +// NOT-MANAGED one, the two separated by absence runs longer than the absence grace. Driven +// through the REAL gate rather than the injection seam, which can only ever SET a label and +// never remove tracking: toggling whether "a" carries GetState/ChangeState services is what +// produces a genuine nullopt for a previously-tracked fqn. +TEST_F(LifecycleExpectationIntegrationTest, InactiveAlternatingWithNotManagedAcrossAbsenceGapsRaisesInactive) { + set_apps({}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 5}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + std::uint64_t tick_counter = 7000; + bool raised = false; + for (int cycle = 0; cycle < 20 && !raised; ++cycle) { + // MEASURED not-active leg: the node is tracked and reads a real label. + set_managed_app("a", "/a"); + gate.update(snapshot_, ++tick_counter); + gate.set_lifecycle_state_for_test("a", "inactive"); + det->tick(ctx); + std::this_thread::sleep_for(2ms); + + // A gap longer than the absence grace: the node is gone from the snapshot entirely. + set_apps({}); + for (int i = 0; i < kDefaultAbsenceGrace + 2 && !raised; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(2ms); + raised = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeInactive) > 0; + } + if (raised) { + break; + } + + // NOT-MANAGED leg: the node is back but its lifecycle services are gone, so the + // tracked entry is dropped and lifecycle_state_of() reads nullopt. + set_apps_raw({{"a", "/a"}}); + gate.update(snapshot_, ++tick_counter); + ASSERT_FALSE(gate.lifecycle_state_of("a").has_value()) + << "cycle " << cycle << ": the not-managed leg never produced a genuine nullopt"; + det->tick(ctx); + std::this_thread::sleep_for(2ms); + + set_apps({}); + for (int i = 0; i < kDefaultAbsenceGrace + 2 && !raised; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(2ms); + raised = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeInactive) > 0; + } + } + EXPECT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, 1, kNodeInactive)) + << "a node alternating between a measured not-active read and a not-managed one, with gaps " + "longer than the absence grace between them, was never reported under any code"; +} + +// The false positive this model must not have, at the tier that proves the DETECTOR's +// output: a node measured ACTIVE that then leaves the graph raises nothing at all, however +// long it stays gone. Absence continues what a node was already doing, and a healthy node +// was doing nothing. +TEST_F(LifecycleExpectationIntegrationTest, HealthyNodeThatVanishesRaisesNothingAtAll) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "active"); + for (int i = 0; i < 5; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + ASSERT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "the node was reported while it was present AND active - nothing below would be about absence"; + + set_apps({}); + for (int i = 0; i < kHoldTicks + 10; ++i) { // far past every clock in the tracker + det->tick(ctx); + std::this_thread::sleep_for(2ms); + } + std::this_thread::sleep_for(300ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "GRAPH_NODE_INACTIVE raised for a node that was measured ACTIVE and then shut down - " + "absence must continue a violation, never start one"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeUnreadable), 0u) + << "GRAPH_NODE_UNREADABLE raised for a node whose state WAS read, and read as active"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeNotManaged), 0u) + << "GRAPH_NODE_NOT_MANAGED raised for a node that was a tracked lifecycle node all along"; + EXPECT_EQ(det->tracked_count_for_test(), 0u) << "an idle entry for a departed healthy node was never reclaimed"; +} + +TEST_F(LifecycleExpectationIntegrationTest, InactiveNodeInARestartLoopIsStillReported) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 5}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "inactive"); + + bool raised = false; + for (int cycle = 0; cycle < kHoldTicks + 5 && !raised; ++cycle) { + set_apps({"a"}); + det->tick(ctx); + std::this_thread::sleep_for(2ms); + set_apps({}); + for (int i = 0; i < kDefaultAbsenceGrace + 2 && !raised; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(2ms); + raised = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeInactive) > 0; + } + raised = raised || count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED, kNodeInactive) > 0; + } + EXPECT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, 1, kNodeInactive)) + << "a node measured inactive whenever it is present and absent the rest of the time never " + "raised GRAPH_NODE_INACTIVE - the violation streak was discarded by every absence run"; +} + +// ---- configure()-level contract (no ROS scaffolding beyond a logger node) ---- + +// A key the detector does not read must be reported, or the README's "unknown keys are +// reported" promise is false for this detector. The misspelling chosen is the worst case: +// with `require_activ` the detector is ALSO unconfigured, so the warning must be logged +// before the zero-config early return, not after it. +TEST(LifecycleExpectationConfig, MisspeltKeyIsWarnedAboutOnce) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_misspelt"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_activ", nlohmann::json::array({"a"})}}); // misspelt on purpose + DetectorContext ctx; + ctx.gateway_node = node.get(); + for (int i = 0; i < 3; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count("unknown config key 'require_activ'"), 1) + << "expected the misspelt key logged exactly once across three ticks (0 = the sweep is missing " + "or logged after the zero-config early return; >1 = the once-guard is gone)"; +} + +// The known-key set must not drift from what configure() reads: a full valid config with +// every documented key plus the plugin-injected ones produces ZERO warnings. +TEST(LifecycleExpectationConfig, EveryKnownKeyIsAcceptedWithoutWarning) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_known"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{ + {"require_active", nlohmann::json::array({"a"})}, {"grace", 3}, {"prune_grace", 10}, {"tick_interval_ms", 100}}); + DetectorContext ctx; + ctx.gateway_node = node.get(); + for (int i = 0; i < 3; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count("graph_watchdog lifecycle_expectation"), 0) + << "a fully valid config (all known keys + the plugin-injected ones) produced a warning"; + + // The low end of the documented range: grace 0 (no tolerance at all) and prune_grace 0 + // (reclaim on the very next absent tick) are both valid, not merely "not yet past the + // range" - the same endpoint the prune_grace-specific tests exercise via tracked_count, + // pinned here on the log instead so a grace validator that quietly rejects 0 (an + // off-by-one on the >= 0 check) cannot hide behind those. + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 0}, {"prune_grace", 0}}); + for (int i = 0; i < 3; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count("graph_watchdog lifecycle_expectation"), 0) + << "grace: 0 and prune_grace: 0 are documented, valid low endpoints and must not warn"; +} + +TEST(LifecycleExpectationConfig, NegativeGraceWarnsAndKeepsTheDefault) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_neg_grace"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", -1}}); + DetectorContext ctx; + ctx.gateway_node = node.get(); + det->tick(ctx); + EXPECT_EQ(log.count("'grace' must be an integer in 0..300; keeping the default (5)"), 1) + << "grace: -1 must warn and name the default that stays in effect"; +} + +TEST(LifecycleExpectationConfig, NonIntegerGraceWarnsAndKeepsTheDefault) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_str_grace"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", "three"}}); + DetectorContext ctx; + ctx.gateway_node = node.get(); + det->tick(ctx); + EXPECT_EQ(log.count("'grace' must be an integer in 0..300"), 1) << "a non-integer grace must warn, not vanish"; +} + +// prune_grace is plugin-injected and exempt from the unknown-key sweep, so a bad value +// must still be validated here - and on the WIDE integer: 4294967296 truncated through +// get() would arrive as 0 and reclaim bookkeeping on the first absent tick. +TEST(LifecycleExpectationConfig, InvalidPruneGraceWarnsAndKeepsTheDefault) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_prune"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + DetectorContext ctx; + ctx.gateway_node = node.get(); + + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"prune_grace", -1}}); + det->tick(ctx); + EXPECT_EQ(log.count("'prune_grace' must be an integer in 0..3600; keeping 60"), 1); + + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"prune_grace", 4294967296}}); + det->tick(ctx); + EXPECT_EQ(log.count("'prune_grace' must be an integer in 0..3600; keeping 60"), 2) + << "an out-of-int-range prune_grace must be rejected on the wide value, not truncated into " + "a tiny prune horizon"; +} + +// `grace` is read off the same 64-bit JSON integer prune_grace is, so it needs the same +// wide range check: get() truncates FIRST, and 4294967296 arrives as 0 - a value that +// passes the >= 0 test and installs the most dangerous setting this key has. +TEST(LifecycleExpectationConfig, WideGraceWarnsAndKeepsTheDefault) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_wide_grace"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + DetectorContext ctx; + ctx.gateway_node = node.get(); + + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 4294967296}}); + det->tick(ctx); + EXPECT_EQ(log.count("'grace' must be an integer in 0..300; keeping the default (5)"), 1) + << "a grace past the int range was truncated to 0 and accepted in silence"; + + // The negative twin truncates to 0 as well, so the sign check alone cannot catch it. + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", -4294967296}}); + det->tick(ctx); + EXPECT_EQ(log.count("'grace' must be an integer in 0..300; keeping the default (5)"), 2) + << "a negative grace past the int range truncated to 0 and passed the >= 0 check"; +} + +// C4: kMaxGrace (lifecycle_expectation_detector.cpp) is checked at configure() time, but +// every existing grace test uses a value chosen to expose the truncation bug the WIDE +// check exists for (2^32, -2^32) - none of them sit at kMaxGrace's own boundary. Both +// documented endpoints and one value past each, since an off-by-one at either end goes +// unnoticed otherwise. +TEST(LifecycleExpectationConfig, GraceRangeEndpoints) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_grace_endpoints"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + DetectorContext ctx; + ctx.gateway_node = node.get(); + const std::string warning = "'grace' must be an integer in 0..300; keeping the default (5)"; + constexpr std::int64_t kMaxGrace = 300; + + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 0}}); + det->tick(ctx); + EXPECT_EQ(log.count("graph_watchdog lifecycle_expectation"), 0) + << "grace 0 is the documented lower endpoint and must be accepted without a word"; + + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", kMaxGrace}}); + det->tick(ctx); + EXPECT_EQ(log.count("graph_watchdog lifecycle_expectation"), 0) + << "kMaxGrace (" << kMaxGrace << ") is the documented upper endpoint and must be accepted without a word"; + + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", kMaxGrace + 1}}); + det->tick(ctx); + EXPECT_EQ(log.count(warning), 1) << "the first value past kMaxGrace (" << (kMaxGrace + 1) << ") was accepted"; + + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", -1}}); + det->tick(ctx); + EXPECT_EQ(log.count(warning), 2) << "the first value below the range was accepted"; + + // The value that USED to be the accepted maximum. A streak has to reach `grace` before a + // node is confirmed, and a node that left the graph while not-active sits in the tracker's + // pending set until it does - which withholds GRAPH_NODE_INACTIVE's clear for EVERY node + // meanwhile. At this value that is about 24 days at the shipped 1 s cadence, i.e. the + // detector is silent either way for longer than any deployment runs uninterrupted. + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, + {"grace", static_cast(std::numeric_limits::max()) - 1}}); + det->tick(ctx); + EXPECT_EQ(log.count(warning), 3) + << "a grace of INT_MAX - 1 was accepted - it neither raises nor heals GRAPH_NODE_INACTIVE for weeks"; +} + +// The documented prune_grace range endpoints. 3600 is inside the range and must be silent; +// 3601 is the first value outside it and must warn. Without both, an off-by-one at either +// end of the bound goes unnoticed. +TEST(LifecycleExpectationConfig, PruneGraceRangeEndpoints) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_prune_endpoints"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + DetectorContext ctx; + ctx.gateway_node = node.get(); + + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"prune_grace", 3600}}); + det->tick(ctx); + EXPECT_EQ(log.count("graph_watchdog lifecycle_expectation"), 0) + << "prune_grace 3600 is the documented upper endpoint and must be accepted without a word"; + + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"prune_grace", 3601}}); + det->tick(ctx); + EXPECT_EQ(log.count("'prune_grace' must be an integer in 0..3600; keeping 60"), 1) + << "the first value past the documented range was accepted"; +} + +// The documented tracked_node_cap range, at BOTH endpoints and one value past each. 0 is +// deliberately outside the range: a cap of nothing means the detector checks nothing, which +// is the silence it exists to prevent, so it must warn rather than be clamped up to 1. +TEST(LifecycleExpectationConfig, TrackedNodeCapRangeEndpoints) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_cap_endpoints"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + DetectorContext ctx; + ctx.gateway_node = node.get(); + const std::string warning = "'tracked_node_cap' must be an integer in 1..16384; keeping the default (512)"; + + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"tracked_node_cap", 1}}); + det->tick(ctx); + EXPECT_EQ(log.count("graph_watchdog lifecycle_expectation"), 0) + << "tracked_node_cap 1 is the documented lower endpoint and must be accepted without a word"; + + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"tracked_node_cap", 16384}}); + det->tick(ctx); + EXPECT_EQ(log.count("graph_watchdog lifecycle_expectation"), 0) + << "tracked_node_cap 16384 is the documented upper endpoint and must be accepted without a word"; + + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"tracked_node_cap", 0}}); + det->tick(ctx); + EXPECT_EQ(log.count(warning), 1) << "tracked_node_cap 0 - one below the range - was accepted in silence"; + + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"tracked_node_cap", 16385}}); + det->tick(ctx); + EXPECT_EQ(log.count(warning), 2) << "the first value past the documented range was accepted"; + + // The wide-integer check, for the same reason grace and prune_grace need one: get() + // truncates first, so 2^32 + 1 would arrive as 1 and silently install the tightest cap + // this key has - one tracked node for the whole fleet. + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"tracked_node_cap", 4294967297}}); + det->tick(ctx); + EXPECT_EQ(log.count(warning), 3) + << "a tracked_node_cap past the int range truncated to 1 and installed a cap of one node"; + + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"tracked_node_cap", "many"}}); + det->tick(ctx); + EXPECT_EQ(log.count(warning), 4) << "a non-integer tracked_node_cap must warn, not vanish"; +} + +// The lower endpoint IN FORCE, not merely accepted: at tracked_node_cap 1 exactly one node +// is tracked however many the entry matches. A key that parses and warns correctly but is +// never handed to the tracker would pass TrackedNodeCapRangeEndpoints above and change +// nothing at all. +TEST(LifecycleExpectationConfig, TrackedNodeCapOfOneTracksExactlyOneNode) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"tracked_node_cap", 1}}); + // No gate: every matched node reads nullopt, so each carries a live not-managed clock from + // its first tick and none is ever idle - the state in which the cap is the only bound. + auto snapshot = IntrospectionInput{}; + for (int i = 0; i < 5; ++i) { + App app; + app.id = "ns" + std::to_string(i) + "_a"; + app.bound_fqn = "/ns" + std::to_string(i) + "/a"; + snapshot.apps.push_back(app); + } + DetectorContext ctx; + ctx.snapshot = &snapshot; + for (int i = 0; i < 5; ++i) { + det->tick(ctx); + } + EXPECT_EQ(det->tracked_count_for_test(), 1u) + << "tracked_node_cap 1 was parsed but never reached the tracker - five matched nodes are being " + "kept where the operator asked for one"; +} + +// The key raises the bound past the shipped default, which is the whole reason it exists: a +// deployment whose require_active legitimately matches more than 512 PRESENT nodes had no +// lever at all before. Proven at 600 - past the 512 default, affordably short of the 16384 +// endpoint, and therefore impossible to pass with the default still in force. +TEST(LifecycleExpectationConfig, TrackedNodeCapAboveTheDefaultActuallyRaisesTheBound) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_cap_raised"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"tracked_node_cap", 600}}); + constexpr int kNodes = 600; + auto snapshot = IntrospectionInput{}; + for (int i = 0; i < kNodes; ++i) { + App app; + app.id = "ns" + std::to_string(i) + "_a"; + app.bound_fqn = "/ns" + std::to_string(i) + "/a"; + snapshot.apps.push_back(app); + } + DetectorContext ctx; + ctx.gateway_node = node.get(); + ctx.snapshot = &snapshot; + det->tick(ctx); + EXPECT_EQ(det->tracked_count_for_test(), static_cast(kNodes)) + << "the configured cap of 600 did not raise the bound - the shipped default of 512 is still in force"; + EXPECT_EQ(log.count("is NOT being checked"), 0) + << "600 nodes under a configured cap of 600 saturated, so the cap the operator wrote was not the " + "one being enforced"; +} + +// A require_active array of non-strings is what a ROS integer_array parameter delivers +// (`require_active: [1, 2]` in YAML). Every entry takes the warn-and-skip arm, and nothing +// may throw out of configure() - the plugin catches a throwing configure() and drops the +// detector entirely, so a typed-wrong list would silently disable the check. +TEST(LifecycleExpectationConfig, NonStringRequireActiveEntriesWarnAndAreSkipped) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_nonstring_entries"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + ASSERT_NO_THROW(det->configure(nlohmann::json{{"require_active", nlohmann::json::array({1, 2})}})); + DetectorContext ctx; + ctx.gateway_node = node.get(); + det->tick(ctx); + EXPECT_EQ(log.count("'require_active' entries must be non-empty node names; skipping one"), 2) + << "a non-string entry was dropped without a word, so the operator believes a node is covered"; +} + +// Bounded tracking, through the detector's own config and at SCALE past the shipped cap. +// Every churned identity here is matched with no gate wired, so each reads nullopt and is +// carrying a live not-managed clock from its first tick - which means the age horizon +// reclaims none of them, deliberately (that horizon is exactly what a restart loop evaded). +// The cap is the bound - and it is met by COLLAPSING the entries for the identities that are +// gone, never by refusing the live one: the newcomer here is the only node actually in the +// graph, and refusing it would mean the detector reporting health it declined to check. +TEST(LifecycleExpectationConfig, ChurningIdentitiesCarryingEvidenceAreBoundedByCollapsingTheDeparted) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_churn_cap"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 1}, {"prune_grace", 2}}); + IntrospectionInput snapshot; + DetectorContext ctx; + ctx.gateway_node = node.get(); + ctx.snapshot = &snapshot; // no gate, no client: the tracker bookkeeping is the observable + + const std::size_t cap = static_cast(ros2_medkit_graph_watchdog::kDefaultTrackedNodeCap); + for (std::size_t i = 0; i < cap + 50; ++i) { + snapshot = namespaced_snapshot_of("ns" + std::to_string(i), "a"); + det->tick(ctx); + ASSERT_LE(det->tracked_count_for_test(), cap) + << "iteration " << i + << ": one bookkeeping entry is retained for every fqn ever seen, so a graph that churns " + "node identities grows the detector's map without bound"; + } + EXPECT_GT(det->tracked_count_for_test(), 0u) << "nothing is tracked at all, so nothing was tested"; + EXPECT_EQ(log.count("is NOT being checked"), 0) + << "the PRESENT node was refused to make room for entries whose nodes are gone - the only node " + "actually in the graph is the one going unchecked"; +} + +// Saturation is a real capacity condition, not a permanent latch: it must be reported ONCE +// per episode, and reported AGAIN when a later one happens. A latch spent by the first +// episode makes every subsequent one silent, which is the worse failure - the operator has +// resolved one and has no way of learning about the next. +// +// A cap of one against two PRESENT nodes both carrying evidence is the only shape that +// saturates now that departed entries are collapsed: nothing is idle, nothing is departed, +// so there is genuinely no slot to free. The refused node then LEAVES (ending the episode) +// and comes back (starting a second one). +TEST(LifecycleExpectationConfig, SaturationIsReportedAgainWhenItEndsAndRecurs) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_saturation_rearm"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure( + nlohmann::json{{"require_active", nlohmann::json::array({"a", "b"})}, {"grace", 0}, {"tracked_node_cap", 1}}); + // No gate: both nodes read nullopt, so each carries a live not-managed clock from its + // first tick and neither is ever idle - the state in which a slot genuinely cannot be freed. + auto both = snapshot_of({"a", "b"}); + auto only_a = snapshot_of({"a"}); + DetectorContext ctx; + ctx.gateway_node = node.get(); + + ctx.snapshot = &both; + for (int i = 0; i < 3; ++i) { + det->tick(ctx); + } + ASSERT_EQ(det->tracked_count_for_test(), 1u) << "a cap of one tracked more than one node"; + EXPECT_EQ(log.count("is NOT being checked"), 1) << "the first saturation episode was not reported exactly once"; + + // "b" leaves: nothing is refused any more, so the episode ends and the latch re-arms. + ctx.snapshot = &only_a; + for (int i = 0; i < 3; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count("is NOT being checked"), 1) + << "the reason was repeated after the episode ended - it is once per episode, or a long " + "saturation fills the log with the same line"; + + // "b" comes back and is refused again: a SECOND episode, which must be reported. + ctx.snapshot = &both; + for (int i = 0; i < 3; ++i) { + det->tick(ctx); + } + EXPECT_EQ(log.count("is NOT being checked"), 2) + << "a second, real saturation was silent because the first one spent the only warning - the " + "operator resolves one capacity problem and never hears about the next"; +} + +TEST(LifecycleExpectationConfig, NonArrayRequireActiveWarnsAndIsIgnored) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_nonarray"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", "a"}}); // a plain string, not an array + DetectorContext ctx; + ctx.gateway_node = node.get(); + det->tick(ctx); + EXPECT_EQ(log.count("'require_active' must be a string array of node names; ignoring it"), 1) + << "a non-array require_active was dropped without a word"; +} + +TEST(LifecycleExpectationConfig, EmptyRequireActiveEntryWarnsAndIsSkipped) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_empty_entry"); + const LogCapture log; + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a", ""})}}); + DetectorContext ctx; + ctx.gateway_node = node.get(); + det->tick(ctx); + EXPECT_EQ(log.count("'require_active' entries must be non-empty node names; skipping one"), 1) + << "an empty require_active entry was dropped silently - the operator believes a node is covered"; +} + +// The age horizon reaches IDLE bookkeeping only, and it reaches it at exactly the +// configured prune_grace - no clamp, because there is no longer anything for one to +// protect: an entry carrying evidence is exempt by construction, not by arithmetic. The +// tightest endpoint first (prune_grace 0 with grace 0: reclaimed on the FIRST absent tick), +// against a node whose lifecycle read is "active" so it is genuinely idle. The gate is +// wired here precisely because a detector with no gate reads every node as not-managed, +// which is never idle. +TEST(LifecycleExpectationConfig, IdleEntryIsReclaimedAtExactlyPruneGraceZero) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_prune_zero"); + std::mutex node_mutex; + ReliabilityGate gate(/*warmup_cycles=*/1, node.get(), &node_mutex); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 0}, {"prune_grace", 0}}); + auto snapshot = snapshot_of({"a"}); + DetectorContext ctx; + ctx.snapshot = &snapshot; + ctx.gate = &gate; + gate.set_lifecycle_state_for_test("a", "active"); + det->tick(ctx); + ASSERT_EQ(det->tracked_count_for_test(), 1u); + snapshot.apps.clear(); + det->tick(ctx); // absent 1 > prune_ticks(0): reclaimed at once + EXPECT_EQ(det->tracked_count_for_test(), 0u) + << "prune_grace 0 must mean 0, not a clamped-up grace + 1 - the value the operator wrote is " + "the horizon an idle entry gets"; +} + +// The smallest POSITIVE prune_grace, which neither documented endpoint sweeps: 0 and 4 both +// pass an implementation that special-cases 1, rounds a positive value down to 0, or tests +// `<= prune_grace` instead of `>`. At 1 the idle entry must survive absent tick 1 exactly and +// be reclaimed on absent tick 2. +TEST(LifecycleExpectationConfig, IdleEntryIsReclaimedOnTheSecondAbsentTickAtPruneGraceOne) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_prune_one"); + std::mutex node_mutex; + ReliabilityGate gate(/*warmup_cycles=*/1, node.get(), &node_mutex); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 0}, {"prune_grace", 1}}); + auto snapshot = snapshot_of({"a"}); + DetectorContext ctx; + ctx.snapshot = &snapshot; + ctx.gate = &gate; + gate.set_lifecycle_state_for_test("a", "active"); // measured healthy, so the entry is genuinely IDLE + det->tick(ctx); + ASSERT_EQ(det->tracked_count_for_test(), 1u); + + snapshot.apps.clear(); + det->tick(ctx); // absent 1, NOT past prune_ticks(1) + EXPECT_EQ(det->tracked_count_for_test(), 1u) + << "prune_grace 1 reclaimed on the first absent tick - the operator asked for one tick of " + "tolerance and got none"; + det->tick(ctx); // absent 2 > prune_ticks(1): reclaimed + EXPECT_EQ(det->tracked_count_for_test(), 0u) + << "prune_grace 1 kept the idle entry past the tick it was written to expire on"; +} + +// The other endpoint of the documented range, and the shape an operator actually writes: a +// wider prune_grace keeps an idle entry for exactly that many absent ticks and drops it on +// the next one. +TEST(LifecycleExpectationConfig, IdleEntrySurvivesExactlyPruneGraceAbsentTicks) { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + auto node = std::make_shared("le_cfg_prune_four"); + std::mutex node_mutex; + ReliabilityGate gate(/*warmup_cycles=*/1, node.get(), &node_mutex); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 0}, {"prune_grace", 4}}); + auto snapshot = snapshot_of({"a"}); + DetectorContext ctx; + ctx.snapshot = &snapshot; + ctx.gate = &gate; + gate.set_lifecycle_state_for_test("a", "active"); + det->tick(ctx); + ASSERT_EQ(det->tracked_count_for_test(), 1u); + snapshot.apps.clear(); + for (int i = 0; i < 4; ++i) { + det->tick(ctx); // absent 1..4, none past prune_ticks(4) + } + EXPECT_EQ(det->tracked_count_for_test(), 1u) << "pruned before prune_grace absent ticks elapsed"; + det->tick(ctx); // absent 5 > prune_ticks: reclaimed + EXPECT_EQ(det->tracked_count_for_test(), 0u); +} + +// Config sweep at the tightest endpoint the documented space allows - `grace: 0` together +// with `prune_grace: 0`, both at the bottom of their ranges. A node carrying a LIVE clock +// (here the not-managed one: no gate is wired, so lifecycle_state_of() is never consulted +// and the node reads nullopt from its first tick) must survive that horizon: reclaiming +// bookkeeping by AGE erases the evidence the node earned, and at this endpoint the age is +// one tick, so the node's clock would restart from zero every time it blinked. +TEST(LifecycleExpectationConfig, LiveClockIsNotErasedAtTheTightestGraceAndPruneGraceEndpoint) { + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 0}, {"prune_grace", 0}}); + auto snapshot = snapshot_of({"a"}); + DetectorContext ctx; + ctx.snapshot = &snapshot; // no gate, no client: the tracker bookkeeping is the observable + det->tick(ctx); + ASSERT_EQ(det->tracked_count_for_test(), 1u) << "sanity: the node must be tracked before it vanishes"; + + snapshot.apps.clear(); + for (int i = 0; i < 10; ++i) { + det->tick(ctx); + EXPECT_EQ(det->tracked_count_for_test(), 1u) + << "absent tick " << (i + 1) + << ": a node carrying a live clock was erased by the prune horizon at grace: 0, " + "prune_grace: 0 - its evidence is gone, so a node that touches absence periodically " + "can never accumulate enough of it to be reported"; + } +} + +// The combination the old clamp existed for - a wide `grace` next to the tightest +// `prune_grace` - is now simply safe: a node still climbing toward that wide grace is +// carrying evidence, so the hair-trigger prune horizon never reaches it, and it goes on to +// be confirmed at exactly the tick it would have been confirmed at anyway. +// +// The instrument is the CONFIRMATION, not the map size. A map size of 1 is satisfied by an +// implementation that keeps the entry but freezes its clock through absence - which is the +// bug that makes a node vanishing while not-active permanently invisible, i.e. exactly the +// thing this test is named for. It is a fixture test rather than a bare configure()-level one +// for that reason: only the fake ReportFault sink can see the raise. +TEST_F(LifecycleExpectationIntegrationTest, WideGraceWithTheTightestPruneGraceStillConfirmsAVanishedNode) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 4}, {"prune_grace", 0}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + gate.set_lifecycle_state_for_test("a", "inactive"); + + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + det->tick(ctx); // streak 1, well below grace(4) + ASSERT_EQ(det->tracked_count_for_test(), 1u); + std::this_thread::sleep_for(200ms); + ASSERT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), failed_before) + << "the node was confirmed on its first not-active tick, so grace(4) was never in force and " + "the absence run below proves nothing about a clock that has to keep climbing"; + + // "a" vanishes with a streak of 1 out of 4. Absence must go on advancing it, so it is + // confirmed while GONE - and the tightest prune horizon must not reach it on the way. + set_apps({}); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) + << "a node that left the graph with a streak below a wide grace was never confirmed at all - " + "its clock was frozen or its entry pruned while absent, so a node that vanishes while " + "not-active is permanently invisible"; + EXPECT_EQ(det->tracked_count_for_test(), 1u) + << "the entry was pruned by the tightest prune_grace despite carrying evidence"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "has since left the graph")) + << "the confirmation did not say the node had left the graph, so an operator is sent looking " + "for a node that is no longer there"; +} diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_expectation_tracker.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_expectation_tracker.cpp new file mode 100644 index 000000000..c8fa77492 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_expectation_tracker.cpp @@ -0,0 +1,1320 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "ros2_medkit_graph_watchdog/lifecycle_expectation_tracker.hpp" + +#include +#include +#include +#include +#include +#include + +#include + +#include "ros2_medkit_graph_watchdog/aggregated_fault.hpp" + +namespace { +using ros2_medkit_graph_watchdog::AggregatedFault; +using ros2_medkit_graph_watchdog::kDefaultAbsenceGrace; +using ros2_medkit_graph_watchdog::kDefaultNoMatchWarnTicks; +using ros2_medkit_graph_watchdog::kDefaultObservationSettleTicks; +using ros2_medkit_graph_watchdog::kDefaultTrackedNodeCap; +using ros2_medkit_graph_watchdog::kDefaultUnmeasuredHoldTicks; +using ros2_medkit_graph_watchdog::kMaxLifecycleDetailChars; +using ros2_medkit_graph_watchdog::kMaxLifecycleLabelChars; +using ros2_medkit_graph_watchdog::kMaxNamedDepartedEntries; +using ros2_medkit_graph_watchdog::LifecycleExpectationTracker; +using ros2_medkit_graph_watchdog::LifecycleMatch; +using M = std::vector; + +// Violations are keyed by the NODE, so a match carries both the config entry that +// matched and the node's stable fqn. +LifecycleMatch match(const std::string & entry, const std::string & fqn, std::optional state) { + return LifecycleMatch{entry, fqn, std::move(state)}; +} + +// ---- Violation streak: unchanged behaviour from before this slice's redesign ---- + +TEST(LifecycleExpectation, RequiredNodeStuckInactivePastGraceRaises) { + LifecycleExpectationTracker t({"a"}, /*grace=*/2); + EXPECT_TRUE(t.update(M{match("a", "/a", "inactive")}).affected.empty()); // 1 <= grace + EXPECT_TRUE(t.update(M{match("a", "/a", "inactive")}).affected.empty()); // 2 == grace + auto report = t.update(M{match("a", "/a", "inactive")}); // 3 > grace + ASSERT_EQ(report.affected.size(), 1u); + EXPECT_TRUE(report.affected.count("/a")) << "the fault must be keyed by the node, not the config entry"; + EXPECT_NE(report.affected.at("/a").find("/a"), std::string::npos); + EXPECT_NE(report.affected.at("/a").find("required by 'a'"), std::string::npos) + << "the entry that demanded it is still worth naming, but as context"; +} + +TEST(LifecycleExpectation, RequiredNodeActiveNeverRaises) { + LifecycleExpectationTracker t({"a"}, 0); + for (int i = 0; i < 5; ++i) { + EXPECT_TRUE(t.update(M{match("a", "/a", "active")}).affected.empty()); + } +} + +TEST(LifecycleExpectation, ReachesActiveWithinGraceNeverRaises) { + LifecycleExpectationTracker t({"a"}, 2); + t.update(M{match("a", "/a", "unconfigured")}); // 1 + t.update(M{match("a", "/a", "inactive")}); // 2 + EXPECT_TRUE(t.update(M{match("a", "/a", "active")}).affected.empty()); // reset + EXPECT_TRUE(t.update(M{match("a", "/a", "active")}).affected.empty()); +} + +TEST(LifecycleExpectation, AbsentNodeIsNotOurJob) { + LifecycleExpectationTracker t({"a"}, 0); + for (int i = 0; i < 3; ++i) { + EXPECT_TRUE(t.update(M{}).affected.empty()); // absent -> GRAPH_NODE_DISAPPEARED owns it + } +} + +// The bare-name form is deliberately fleet-wide, so one entry covers N nodes. Keying the +// violation on the ENTRY meant the fault could not say which namesake broke, and with two +// offenders only one survived into the report at all. +TEST(LifecycleExpectation, BothNamesakesAreReportedSeparately) { + LifecycleExpectationTracker t({"controller_server"}, /*grace=*/0); + auto report = t.update(M{match("controller_server", "/left/controller_server", "inactive"), + match("controller_server", "/right/controller_server", "unconfigured")}); + ASSERT_EQ(report.affected.size(), 2u) << "one entry, two offenders, two reports"; + EXPECT_TRUE(report.affected.count("/left/controller_server")); + EXPECT_TRUE(report.affected.count("/right/controller_server")); +} + +TEST(LifecycleExpectation, HealthyNamesakeDoesNotMaskABrokenOne) { + LifecycleExpectationTracker t({"controller_server"}, /*grace=*/0); + auto report = t.update(M{match("controller_server", "/left/controller_server", "active"), + match("controller_server", "/right/controller_server", "inactive")}); + ASSERT_EQ(report.affected.size(), 1u); + EXPECT_TRUE(report.affected.count("/right/controller_server")); +} + +// A node that blinks out of one snapshot used to have its violation streak zeroed outright, +// so a genuinely stuck node on a churning graph could never accumulate grace+1 consecutive +// present-and-inactive ticks. +TEST(LifecycleExpectation, SingleSnapshotBlinkDoesNotResetTheViolationStreak) { + LifecycleExpectationTracker t({"a"}, /*grace=*/2, /*absence_grace=*/3); + t.update(M{match("a", "/a", "inactive")}); // 1 + t.update(M{}); // blink: absent for one tick + t.update(M{match("a", "/a", "inactive")}); // 2 + auto report = t.update(M{match("a", "/a", "inactive")}); // 3 > grace + EXPECT_FALSE(report.affected.empty()) << "one absent snapshot must not restart the count"; +} + +// The bare-name form is fleet-wide and the full-FQN form pins one robot, so an operator +// combining them - both documented, both recommended - has ONE node named by TWO entries. +// The caller pushes one match per (entry, node) pair, so that node arrives twice in the +// same sweep; counting per match instead of per node advances its streak twice a tick and +// halves the grace the operator configured. +TEST(LifecycleExpectation, TwoEntriesMatchingOneNodeDoNotHalveItsGrace) { + LifecycleExpectationTracker t({"a", "/a"}, /*grace=*/2); + const M both{match("a", "/a", "inactive"), match("/a", "/a", "inactive")}; + EXPECT_TRUE(t.update(both).affected.empty()) << "tick 1: one node, one streak - 1 <= grace"; + EXPECT_TRUE(t.update(both).affected.empty()) << "tick 2: 2 == grace, not past it yet"; + auto report = t.update(both); // tick 3: 3 > grace + ASSERT_EQ(report.affected.size(), 1u) << "two entries naming one node must yield one report"; + ASSERT_TRUE(report.affected.count("/a")); + EXPECT_NE(report.affected.at("/a").find("'a'"), std::string::npos) << "every matching entry is context"; + EXPECT_NE(report.affected.at("/a").find("'/a'"), std::string::npos) << "every matching entry is context"; +} + +// The same node, the same tick, one entry seeing it inactive: a violating read must never +// be tied-out by a benign one from the other entry. +TEST(LifecycleExpectation, ViolatingReadWinsOverABenignOneForTheSameNode) { + LifecycleExpectationTracker t({"a", "/a"}, /*grace=*/0); + auto report = t.update(M{match("a", "/a", "active"), match("/a", "/a", "inactive")}); + ASSERT_EQ(report.affected.size(), 1u) << "a measured violation must survive a disagreeing duplicate match"; + EXPECT_TRUE(report.affected.count("/a")); +} + +// A duplicate match disagreeing between the two UNMEASURED causes - or between an unmeasured +// read and a healthy one - must pick the more informative reading too, mirroring the +// violating-wins rule above. This is edge-case hardening: two entries naming one node are +// the same physical app.id read twice, so in practice they never actually disagree. +TEST(LifecycleExpectation, DuplicateMatchPicksUnreadableOverNotManagedAndOverActive) { + LifecycleExpectationTracker t({"a", "/a"}, /*grace=*/0); + // "" (unreadable) must win over nullopt (not-managed): run both past the hold and check + // which content map claims the node. + for (int i = 0; i < kDefaultUnmeasuredHoldTicks + 1; ++i) { + auto report = t.update(M{match("a", "/a", std::string("")), match("/a", "/a", std::nullopt)}); + if (i == kDefaultUnmeasuredHoldTicks) { + EXPECT_TRUE(report.unreadable_affected.count("/a")) << "unreadable did not win the tie-break over not-managed"; + EXPECT_TRUE(report.not_managed_affected.empty()); + } + } +} + +// ---- The unmeasured clock: UNREADABLE and NOT_MANAGED share one cause-blind clock ---- + +// Neither UNREADABLE nor NOT_MANAGED is ever a confirmed violation, however long the run: +// only a MEASURED not-active read ever advances the violation streak, and the unmeasured +// clock only ever feeds its OWN content maps, never `affected`. +TEST(LifecycleExpectation, UnreadableLabelNeverConfirmsInactiveWhileItsOwnClockClimbs) { + LifecycleExpectationTracker t({"a"}, /*grace=*/0); + for (int i = 0; i < kDefaultUnmeasuredHoldTicks; ++i) { + auto report = t.update(M{match("a", "/a", std::string(""))}); + EXPECT_TRUE(report.affected.empty()) << "iteration " << i; + EXPECT_TRUE(report.unreadable_affected.empty()) << "iteration " << i << ": still below the hold"; + EXPECT_EQ(report.pending_unreadable.count("/a"), 1u) << "iteration " << i; + } +} + +TEST(LifecycleExpectation, NotManagedNeverConfirmsInactiveWhileItsOwnClockClimbs) { + LifecycleExpectationTracker t({"a"}, /*grace=*/0); + for (int i = 0; i < kDefaultUnmeasuredHoldTicks; ++i) { + auto report = t.update(M{match("a", "/a", std::nullopt)}); + EXPECT_TRUE(report.affected.empty()) << "iteration " << i; + EXPECT_TRUE(report.not_managed_affected.empty()) << "iteration " << i << ": still below the hold"; + EXPECT_EQ(report.pending_not_managed.count("/a"), 1u) << "iteration " << i; + } +} + +// The instrument matters here (test-plan shape requirement 4): this asserts +// `state.has_value() && state->empty()` via the match builder's std::string("") overload, +// never merely "not active". +TEST(LifecycleExpectation, UnreadableMaturesIntoItsOwnFaultAtTheExactHoldBoundary) { + LifecycleExpectationTracker t({"a"}, /*grace=*/0); + for (int i = 0; i < kDefaultUnmeasuredHoldTicks; ++i) { + t.update(M{match("a", "/a", std::string(""))}); + } + auto report = t.update(M{match("a", "/a", std::string(""))}); // hold + 1: crosses THIS tick + ASSERT_EQ(report.unreadable_affected.size(), 1u); + EXPECT_TRUE(report.not_managed_affected.empty()); + EXPECT_TRUE(report.affected.empty()); + EXPECT_NE(report.unreadable_affected.at("/a").find("/a"), std::string::npos); + EXPECT_NE(report.unreadable_affected.at("/a").find("could not be read"), std::string::npos); + ASSERT_EQ(report.newly_unreadable.size(), 1u) << "crossed the hold on exactly this tick"; + EXPECT_EQ(report.newly_unreadable[0], "/a"); + EXPECT_TRUE(report.pending_unreadable.empty()) << "matured: no longer pending, ownership was taken"; +} + +// The instrument matters here too: `!state.has_value()`, never an empty string. +TEST(LifecycleExpectation, NotManagedMaturesIntoItsOwnFaultAtTheExactHoldBoundary) { + LifecycleExpectationTracker t({"a"}, /*grace=*/0); + for (int i = 0; i < kDefaultUnmeasuredHoldTicks; ++i) { + t.update(M{match("a", "/a", std::nullopt)}); + } + auto report = t.update(M{match("a", "/a", std::nullopt)}); // hold + 1: crosses THIS tick + ASSERT_EQ(report.not_managed_affected.size(), 1u); + EXPECT_TRUE(report.unreadable_affected.empty()); + EXPECT_TRUE(report.affected.empty()); + EXPECT_NE(report.not_managed_affected.at("/a").find("/a"), std::string::npos); + EXPECT_NE(report.not_managed_affected.at("/a").find("not a managed lifecycle node"), std::string::npos); + ASSERT_EQ(report.newly_not_managed.size(), 1u) << "crossed the hold on exactly this tick"; + EXPECT_EQ(report.newly_not_managed[0], "/a"); + EXPECT_TRUE(report.pending_not_managed.empty()) << "matured: no longer pending, ownership was taken"; +} + +// The redesign's whole point, proven directly: three review rounds each closed one +// alternation and revealed the next, because the design being replaced counted UNREADABLE +// and NOT_MANAGED on two SEPARATE counters, each reset by the other's tick. A node +// alternating between the two therefore never crossed either counter's hold, however long +// the run - invisible to both codes forever. Here the SAME alternation, against the new +// cause-blind clock, matures - proving the clock counts the ALTERNATION, not either cause +// alone. (This exact scenario was watched RED against the pre-redesign detector - see the +// integration test of the same shape for the end-to-end proof; this is the pure-logic proof +// at the level the fix actually lives.) +TEST(LifecycleExpectation, AlternatingUnreadableAndNotManagedStillMaturesTheClock) { + LifecycleExpectationTracker t({"a"}, /*grace=*/5); + bool matured = false; + for (int i = 0; i < kDefaultUnmeasuredHoldTicks + 5 && !matured; ++i) { + const auto state = (i % 2 == 0) ? std::optional("") : std::optional(std::nullopt); + auto report = t.update(M{match("a", "/a", state)}); + EXPECT_TRUE(report.affected.empty()) << "iteration " << i << ": never a confirmed violation"; + matured = !report.unreadable_affected.empty() || !report.not_managed_affected.empty(); + } + EXPECT_TRUE(matured) << "a node alternating between unreadable and not-managed never matured under " + "either code - exactly the failure this redesign closes"; +} + +// A node whose cause flips EVERY SINGLE TICK (the fastest possible alternation, faster than +// the pairwise sweep above which flips at the same cadence) - the degenerate case that most +// directly defeats a per-cause counter design (each tick would have zeroed the OTHER +// counter under the design being replaced). +TEST(LifecycleExpectation, AlternatingEveryTickStillMaturesTheClockEventually) { + LifecycleExpectationTracker t({"a"}, /*grace=*/0); + for (int i = 0; i < kDefaultUnmeasuredHoldTicks; ++i) { + const auto state = (i % 2 == 0) ? std::optional("") : std::optional(std::nullopt); + auto report = t.update(M{match("a", "/a", state)}); + EXPECT_TRUE(report.unreadable_affected.empty() && report.not_managed_affected.empty()) + << "iteration " << i << ": matured too early"; + } + const auto last_state = (kDefaultUnmeasuredHoldTicks % 2 == 0) ? std::optional("") + : std::optional(std::nullopt); + auto report = t.update(M{match("a", "/a", last_state)}); + EXPECT_TRUE(!report.unreadable_affected.empty() || !report.not_managed_affected.empty()) + << "the clock never matured despite " << (kDefaultUnmeasuredHoldTicks + 1) << " total unmeasured ticks"; +} + +// The pair the two clocks have to divide between them: a node alternating between a REAL +// not-active read and a not-managed one, with runs of ABSENCE longer than the absence grace +// in between - so every tick belongs to one of the three cases and the alternation crosses +// both ownership and presence. kNotManaged never touches the violation streak and kInactive +// always resets the unmeasured clock, so the streak is the only clock that can accumulate +// here, and it must: whichever way the node flaps, it was MEASURED not-active repeatedly. +TEST(LifecycleExpectation, InactiveAlternatingWithNotManagedAcrossAbsenceGapsStillConfirmsTheViolation) { + constexpr int kGap = kDefaultAbsenceGrace + 1; + LifecycleExpectationTracker t({"a"}, /*grace=*/5, kDefaultAbsenceGrace); + bool reported = false; + for (int cycle = 0; cycle < 20 && !reported; ++cycle) { + auto measured = t.update(M{match("a", "/a", "inactive")}); + reported = !measured.affected.empty(); + for (int i = 0; i < kGap && !reported; ++i) { + reported = !t.update(M{}).affected.empty(); + } + if (reported) { + break; + } + auto unmanaged = t.update(M{match("a", "/a", std::nullopt)}); + EXPECT_TRUE(unmanaged.affected.empty()) << "cycle " << cycle + << ": a node inside an unmeasured spell is not confirmed content, " + "however far its held streak had already climbed"; + EXPECT_EQ(unmanaged.pending_violation.count("/a"), 1u) + << "cycle " << cycle << ": the streak was RESET by a not-managed tick instead of merely held"; + for (int i = 0; i < kGap && !reported; ++i) { + reported = !t.update(M{}).affected.empty(); + } + } + EXPECT_TRUE(reported) << "a node alternating between a measured not-active read and a not-managed one, " + "separated by absence runs longer than the absence grace, was never confirmed - " + "invisible to GRAPH_NODE_INACTIVE and, since the unmeasured clock keeps being reset " + "by the real reads, to its siblings too"; +} + +// The violation streak resets ONLY on kActive. A mutant that zeroed it on every unmeasured +// tick would leave every other test in this suite green, because during the climbing window +// the ONLY field that differs is `pending_violation` - which nothing else reads there. This +// reads it, on every unmeasured tick, and then proves the streak resumed rather than +// restarted by counting the remaining ticks EXACTLY. +TEST(LifecycleExpectation, ViolationStreakSurvivesNonMaturingUnmeasuredTicksAndIsNotRestarted) { + constexpr int kGrace = 5; + LifecycleExpectationTracker t({"a"}, kGrace); + for (int i = 0; i < 3; ++i) { + auto climbing = t.update(M{match("a", "/a", "inactive")}); // streak 1..3, all <= grace + ASSERT_TRUE(climbing.affected.empty()) << "iteration " << i; + ASSERT_EQ(climbing.pending_violation.count("/a"), 1u) << "iteration " << i; + } + // A run of unmeasured ticks far too short to mature the unmeasured clock. The streak is + // HELD across all of them: not reported (nothing is measurable right now), not reset. + for (int i = 0; i < 4; ++i) { + auto unread = t.update(M{match("a", "/a", std::string(""))}); + EXPECT_EQ(unread.pending_violation.count("/a"), 1u) + << "unmeasured tick " << i << ": the violation streak was dropped by a tick that measured nothing"; + EXPECT_TRUE(unread.affected.empty()) << "unmeasured tick " << i; + } + // Exactly two more measured not-active ticks reach 5 == grace, and the third crosses it. + // Counted exactly: a streak that had restarted would need six here, not three. + EXPECT_TRUE(t.update(M{match("a", "/a", "inactive")}).affected.empty()) << "streak 4 <= grace"; + EXPECT_TRUE(t.update(M{match("a", "/a", "inactive")}).affected.empty()) << "streak 5 == grace"; + auto crossed = t.update(M{match("a", "/a", "inactive")}); + EXPECT_EQ(crossed.affected.count("/a"), 1u) + << "the streak restarted across the unmeasured run instead of resuming - a node that goes " + "briefly unreadable every few ticks would never be confirmed"; + EXPECT_EQ(crossed.newly_affected, (std::vector{"/a"})); +} + +// The clock resets on EITHER real measurement - kActive as well as kInactive - never on +// anything else. Proven directly rather than assumed from the maturity tests above. +TEST(LifecycleExpectation, UnmeasuredClockResetsOnlyOnARealMeasurement) { + LifecycleExpectationTracker t({"a"}, /*grace=*/5); + for (int i = 0; i < kDefaultUnmeasuredHoldTicks - 1; ++i) { + t.update(M{match("a", "/a", std::string(""))}); // climbing, not yet matured + } + // A real measurement (active) resets it entirely. + auto after_active = t.update(M{match("a", "/a", "active")}); + EXPECT_TRUE(after_active.pending_unreadable.empty()) << "an active read did not reset the unmeasured clock"; + + // Climb again, then reset via the OTHER real measurement (inactive). + for (int i = 0; i < kDefaultUnmeasuredHoldTicks - 1; ++i) { + t.update(M{match("a", "/a", std::nullopt)}); + } + auto after_inactive = t.update(M{match("a", "/a", "inactive")}); + EXPECT_TRUE(after_inactive.pending_not_managed.empty()) << "an inactive read did not reset the unmeasured clock"; + EXPECT_EQ(after_inactive.pending_violation.count("/a"), 1u) << "the inactive read itself starts a fresh streak"; +} + +// R28: ownership is exclusive, and taking it is not merely a bookkeeping detail - the +// violation streak is RELEASED (reset to 0), not just excluded from `affected` THIS tick. +// A node CONFIRMED inactive that then goes unreadable long enough to mature must stop being +// GRAPH_NODE_INACTIVE's concern entirely, so a later return to inactive re-earns `grace` +// from zero rather than re-crossing on the very next tick. +TEST(LifecycleExpectation, UnmeasuredClockMaturityReleasesTheViolationStreakEntirely) { + LifecycleExpectationTracker t({"a"}, /*grace=*/1); + t.update(M{match("a", "/a", "inactive")}); + auto confirmed = t.update(M{match("a", "/a", "inactive")}); // 2 > grace: confirmed + ASSERT_FALSE(confirmed.affected.empty()) << "sanity: must be confirmed before it can be released"; + + for (int i = 0; i < kDefaultUnmeasuredHoldTicks; ++i) { + auto report = t.update(M{match("a", "/a", std::string(""))}); + EXPECT_TRUE(report.affected.empty()) << "iteration " << i + << ": the held streak must not re-confirm while " + "unread - only maturity, or a real read, changes anything"; + } + auto matured = t.update(M{match("a", "/a", std::string(""))}); // crosses the hold: releases + ASSERT_FALSE(matured.unreadable_affected.empty()) << "sanity: must have matured"; + EXPECT_TRUE(matured.affected.empty()); + EXPECT_TRUE(matured.pending.count("/a") == 0) << "released: GRAPH_NODE_INACTIVE has nothing left to hold for it"; + + // The node returns to inactive: it must re-earn `grace` (1) from zero, not re-cross + // immediately on the strength of the old, released streak. + auto first_back = t.update(M{match("a", "/a", "inactive")}); + EXPECT_TRUE(first_back.affected.empty()) << "1 <= grace: the streak restarted at zero, not resumed"; + EXPECT_EQ(first_back.pending_violation.count("/a"), 1u); + auto second_back = t.update(M{match("a", "/a", "inactive")}); + EXPECT_FALSE(second_back.affected.empty()) << "2 > grace: re-earned normally from the fresh streak"; +} + +// The design decision this slice makes on its own, pinned directly: once matured, the fault +// code a node reports under is STICKY - it does not flip merely because a later tick's LIVE +// read shows the other cause, as long as the clock never resets via a real measurement. The +// rejected alternative (current-cause-wins) would flip `unreadable_affected` <-> +// `not_managed_affected` on every alternating tick below; sticky keeps it fixed on whichever +// cause was live AT THE MOMENT of maturity. +TEST(LifecycleExpectation, StickyCauseSurvivesACauseChangeAfterMaturity) { + LifecycleExpectationTracker t({"a"}, /*grace=*/5); + for (int i = 0; i < kDefaultUnmeasuredHoldTicks; ++i) { + t.update(M{match("a", "/a", std::string(""))}); // climbs as UNREADABLE + } + auto matured = t.update(M{match("a", "/a", std::string(""))}); // matures AS unreadable + ASSERT_EQ(matured.newly_unreadable.size(), 1u) << "sanity: must mature under kUnreadable first"; + + // The cause now flips to NOT_MANAGED for a long stretch, without ever crossing a real + // measurement. Sticky: the node must stay GRAPH_NODE_UNREADABLE's content throughout. + for (int i = 0; i < 20; ++i) { + auto report = t.update(M{match("a", "/a", std::nullopt)}); + EXPECT_TRUE(report.unreadable_affected.count("/a") == 1) << "iteration " << i << ": sticky cause must hold"; + EXPECT_TRUE(report.not_managed_affected.empty()) + << "iteration " << i << ": current-cause-wins would have flipped ownership here - rejected"; + EXPECT_TRUE(report.newly_not_managed.empty()) << "iteration " << i << ": no re-raise churn under a sticky cause"; + } + + // And the same the other way around: matured as NOT_MANAGED, flips to UNREADABLE. + LifecycleExpectationTracker t2({"b"}, /*grace=*/5); + for (int i = 0; i < kDefaultUnmeasuredHoldTicks; ++i) { + t2.update(M{match("b", "/b", std::nullopt)}); + } + auto matured2 = t2.update(M{match("b", "/b", std::nullopt)}); + ASSERT_EQ(matured2.newly_not_managed.size(), 1u) << "sanity: must mature under kNotManaged first"; + for (int i = 0; i < 20; ++i) { + auto report = t2.update(M{match("b", "/b", std::string(""))}); + EXPECT_TRUE(report.not_managed_affected.count("/b") == 1) << "iteration " << i; + EXPECT_TRUE(report.unreadable_affected.empty()) << "iteration " << i; + } +} + +// ---- Absence: held inside the blink tolerance, CONTINUED past it, never erased ---- + +// A blink well inside the absence grace must not disturb a still-climbing unmeasured clock +// at all - it neither advances it (a blink is tolerated, not counted) nor resets it (the +// same "held through a blink" contract every other clock in this tracker gets). +TEST(LifecycleExpectation, AbsentNodeWithAClimbingUnmeasuredClockIsHeldThroughABlink) { + LifecycleExpectationTracker t({"a"}, /*grace=*/5, /*absence_grace=*/3); + for (int i = 0; i < 10; ++i) { + t.update(M{match("a", "/a", std::string(""))}); + } + auto blink = t.update(M{}); // absent 1: well inside the absence grace + EXPECT_EQ(blink.pending_unreadable.count("/a"), 1u) << "held through the blink, not reset nor advanced"; + + // Resumes exactly where it left off: 10 climbing ticks before the blink, so exactly + // (kDefaultUnmeasuredHoldTicks - 10) more real unreadable ticks are needed to reach the + // hold itself (60 total) and one more past it matures - not kDefaultUnmeasuredHoldTicks + // more from scratch, which would prove the blink reset it instead of merely holding it. + for (int i = 0; i < kDefaultUnmeasuredHoldTicks - 10; ++i) { + auto report = t.update(M{match("a", "/a", std::string(""))}); + EXPECT_TRUE(report.unreadable_affected.empty()) << "iteration " << i; + } + auto matured = t.update(M{match("a", "/a", std::string(""))}); + EXPECT_FALSE(matured.unreadable_affected.empty()) + << "the blink must not have added extra ticks to the budget - the clock resumed, it did not reset"; +} + +// Past the absence grace the unmeasured clock KEEPS CLIMBING under the cause the node's +// last real observation set - it is neither reset nor frozen - so a node that vanishes +// mid-spell eventually matures on absence alone. The design this replaces discarded the +// whole thing here, which is what let a node that touches absence periodically evade every +// code forever. +TEST(LifecycleExpectation, AbsentPastGraceContinuesTheClimbingUnmeasuredClockAndEventuallyMaturesIt) { + LifecycleExpectationTracker t({"a"}, /*grace=*/5, /*absence_grace=*/2); + for (int i = 0; i < 10; ++i) { + t.update(M{match("a", "/a", std::string(""))}); // optional(""): a MANAGED, unread node + } + t.update(M{}); // absent 1 + auto blink = t.update(M{}); // absent 2 == absence_grace: still inside the blink tolerance + EXPECT_EQ(blink.pending_unreadable.count("/a"), 1u) << "absent 2 == absence_grace: held, unchanged"; + auto continued = t.update(M{}); // absent 3 > absence_grace: the clock resumes advancing + EXPECT_EQ(continued.pending_unreadable.count("/a"), 1u) + << "sustained absence discarded a climbing unmeasured clock instead of continuing it"; + EXPECT_TRUE(continued.unreadable_affected.empty()) << "sanity: nowhere near the hold yet"; + + // 10 climbing ticks before the blink, plus the one absent tick above: exactly + // (kDefaultUnmeasuredHoldTicks - 11) more absent ticks reach the hold, and one past it + // matures. Counted exactly, so a clock that merely stopped resetting - without actually + // advancing while gone - would fail here rather than passing on a generous budget. + for (int i = 0; i < kDefaultUnmeasuredHoldTicks - 11; ++i) { + EXPECT_TRUE(t.update(M{}).unreadable_affected.empty()) << "iteration " << i << ": matured too early"; + } + auto matured = t.update(M{}); + ASSERT_EQ(matured.unreadable_affected.size(), 1u) + << "absence never matured the clock it had been advancing - the node is invisible for as long " + "as it stays gone, which is exactly the evasion this closes"; + EXPECT_EQ(matured.newly_unreadable, (std::vector{"/a"})); + EXPECT_NE(matured.unreadable_affected.at("/a").find("has since left the graph"), std::string::npos) + << "a fault raised about a node that is no longer in the graph must say so, or the operator is " + "sent looking for it"; +} + +// A node that had ALREADY matured before it went absent KEEPS its fault: ownership is +// sticky and absence does not release it. The node was declared must-be-active, was never +// measured active, and is now gone - none of which is health. +TEST(LifecycleExpectation, AbsentPastGraceKeepsAnAlreadyMaturedUnmeasuredFault) { + LifecycleExpectationTracker t({"a"}, /*grace=*/5, /*absence_grace=*/2); + for (int i = 0; i <= kDefaultUnmeasuredHoldTicks; ++i) { + t.update(M{match("a", "/a", std::nullopt)}); // nullopt: no tracked lifecycle at all + } + ASSERT_EQ(t.update(M{match("a", "/a", std::nullopt)}).not_managed_affected.size(), 1u) << "sanity: matured"; + + t.update(M{}); // absent 1: content survives the blink + auto held = t.update(M{}); // absent 2 == absence_grace: still held + EXPECT_EQ(held.not_managed_affected.count("/a"), 1u); + EXPECT_EQ(held.not_managed_affected.at("/a").find("has since left the graph"), std::string::npos) + << "a blink inside the tolerance is not a departure and must not be described as one"; + for (int i = 0; i < 20; ++i) { + auto still = t.update(M{}); // absent 3.. > absence_grace + ASSERT_EQ(still.not_managed_affected.count("/a"), 1u) + << "iteration " << i << ": absence released a matured fault, discarding evidence the node earned"; + EXPECT_TRUE(still.newly_not_managed.empty()) << "iteration " << i << ": no re-raise churn while merely absent"; + } + EXPECT_NE(t.update(M{}).not_managed_affected.at("/a").find("has since left the graph"), std::string::npos); +} + +// The one departure that starts nothing: a node measured ACTIVE and then shut down. Both +// clocks are at zero, absence has nothing to continue, and the entry is idle so it is +// reclaimed silently. Raising for a HEALTHY node that left is GRAPH_NODE_DISAPPEARED's job, +// which has no detector in this package - the single gap left here. +TEST(LifecycleExpectation, HealthyNodeThatVanishesRaisesNothingAndIsReclaimed) { + LifecycleExpectationTracker t({"a"}, /*grace=*/0, /*absence_grace=*/2, kDefaultNoMatchWarnTicks, + /*prune_ticks=*/3); + for (int i = 0; i < 5; ++i) { + t.update(M{match("a", "/a", "active")}); + } + for (int i = 0; i < kDefaultUnmeasuredHoldTicks + 10; ++i) { + auto report = t.update(M{}); + EXPECT_TRUE(report.affected.empty()) << "iteration " << i << ": a healthy node that left was reported inactive"; + EXPECT_TRUE(report.unreadable_affected.empty() && report.not_managed_affected.empty()) + << "iteration " << i << ": absence STARTED an unmeasured clock for a node that was measured active"; + EXPECT_TRUE(report.pending.empty()) << "iteration " << i << ": a healthy departure must not withhold anything"; + } + EXPECT_EQ(t.tracked_count(), 0u) << "an idle entry past the prune horizon must be reclaimed"; +} + +// The same for a node that had been measured active AFTER an earlier bad spell: what +// absence continues is the LAST real observation, not the worst one ever seen. +TEST(LifecycleExpectation, AbsenceContinuesTheLastObservationNotAnEarlierWorseOne) { + LifecycleExpectationTracker t({"a"}, /*grace=*/1, /*absence_grace=*/1); + t.update(M{match("a", "/a", "inactive")}); + t.update(M{match("a", "/a", "inactive")}); // 2 > grace: confirmed + ASSERT_FALSE(t.update(M{match("a", "/a", "inactive")}).affected.empty()) << "sanity: confirmed first"; + t.update(M{match("a", "/a", "active")}); // healed: both clocks and the label cleared + + for (int i = 0; i < 10; ++i) { + auto report = t.update(M{}); + EXPECT_TRUE(report.affected.empty()) << "iteration " << i + << ": absence resumed a streak the node had already cleared by reading active"; + EXPECT_TRUE(report.pending.empty()) << "iteration " << i; + } +} + +// ---- Absence CONTINUES whatever the node was already doing ---- + +// The shape every one of the three tests below drives: `run_length` absent ticks between +// every real observation, forever. That is what a node in a restart loop looks like from +// the graph - start, crash, respawn delay, start - and it is the case this detector most +// exists to catch, so a clock that absence RESETS makes exactly that node invisible +// however long it runs. `run_length` is swept at the absence grace itself (a blink, which +// was already held) and past it (the run that used to wipe everything). +constexpr int kAbsenceRunLengths[] = {kDefaultAbsenceGrace, kDefaultAbsenceGrace + 2}; + +TEST(LifecycleExpectation, UnreadableInterleavedWithAbsenceRunsStillMaturesAndReports) { + for (const int run_length : kAbsenceRunLengths) { + LifecycleExpectationTracker t({"a"}, /*grace=*/5, kDefaultAbsenceGrace); + bool matured = false; + // Budget: every cycle contributes at least the one present tick, so this is far more + // than enough for a clock that only counts present ticks, let alone one that counts + // absent ones too. + for (int cycle = 0; cycle < kDefaultUnmeasuredHoldTicks + 5 && !matured; ++cycle) { + // The instrument is optional("") - a MANAGED node whose label never answered - not + // merely "not active". + matured = !t.update(M{match("a", "/a", std::string(""))}).unreadable_affected.empty(); + for (int i = 0; i < run_length && !matured; ++i) { + matured = !t.update(M{}).unreadable_affected.empty(); // ABSENT: not in the snapshot at all + } + } + EXPECT_TRUE(matured) << "run_length=" << run_length + << ": a node that is unreadable whenever it is present and absent the rest " + "of the time never matured - absence discarded the evidence, so the " + "restart loop this detector exists to catch evades it forever"; + } +} + +TEST(LifecycleExpectation, NotManagedInterleavedWithAbsenceRunsStillMaturesAndReports) { + for (const int run_length : kAbsenceRunLengths) { + LifecycleExpectationTracker t({"a"}, /*grace=*/5, kDefaultAbsenceGrace); + bool matured = false; + for (int cycle = 0; cycle < kDefaultUnmeasuredHoldTicks + 5 && !matured; ++cycle) { + // The instrument is nullopt - no tracked lifecycle at all - never an empty string. + matured = !t.update(M{match("a", "/a", std::nullopt)}).not_managed_affected.empty(); + for (int i = 0; i < run_length && !matured; ++i) { + matured = !t.update(M{}).not_managed_affected.empty(); + } + } + EXPECT_TRUE(matured) << "run_length=" << run_length + << ": a node that carries no tracked lifecycle whenever it is present and " + "is absent the rest of the time never matured"; + } +} + +TEST(LifecycleExpectation, InactiveInterleavedWithAbsenceRunsStillCrossesGraceAndReports) { + for (const int run_length : kAbsenceRunLengths) { + LifecycleExpectationTracker t({"a"}, /*grace=*/5, kDefaultAbsenceGrace); + bool reported = false; + for (int cycle = 0; cycle < kDefaultUnmeasuredHoldTicks + 5 && !reported; ++cycle) { + auto present = t.update(M{match("a", "/a", "inactive")}); + reported = !present.affected.empty(); + EXPECT_TRUE(present.unreadable_affected.empty() && present.not_managed_affected.empty()) + << "run_length=" << run_length << ": a MEASURED read must never feed an unmeasured code"; + for (int i = 0; i < run_length && !reported; ++i) { + reported = !t.update(M{}).affected.empty(); + } + } + EXPECT_TRUE(reported) << "run_length=" << run_length + << ": a node measured inactive whenever it is present and absent the rest " + "of the time never crossed grace - the violation streak was discarded " + "by every absence run"; + } +} + +// The tightest prune horizon the documented config space can produce (grace: 0 with +// prune_grace: 0). A node carrying a LIVE clock must survive it: reclaiming bookkeeping by +// age erases evidence, and at this endpoint the age is one tick. +TEST(LifecycleExpectation, NodeCarryingALiveClockSurvivesTheTightestPruneHorizon) { + LifecycleExpectationTracker t({"a"}, /*grace=*/0, kDefaultAbsenceGrace, kDefaultNoMatchWarnTicks, + /*prune_ticks=*/0); + auto confirmed = t.update(M{match("a", "/a", "inactive")}); // 1 > grace(0): confirmed at once + ASSERT_FALSE(confirmed.affected.empty()) << "sanity: the node must carry evidence before it vanishes"; + + for (int i = 0; i < 10; ++i) { + t.update(M{}); + EXPECT_EQ(t.tracked_count(), 1u) + << "iteration " << i + << ": a node carrying a confirmed violation was reclaimed by the prune horizon, so its " + "evidence is gone and it can never be reported again"; + } +} + +// ---- R11: new-first order, for all three fault-shaped maps ---- + +// A streak still below grace has not crossed anything; the tick it finally passes grace is +// the ONE tick it belongs in newly_affected; every tick after that it is old news. +TEST(LifecycleExpectation, NewlyAffectedNamesOnlyTheNodeThatCrossedGraceThisTick) { + LifecycleExpectationTracker t({"a"}, /*grace=*/1); + auto below = t.update(M{match("a", "/a", "inactive")}); // 1 <= grace + EXPECT_TRUE(below.newly_affected.empty()) << "below grace: nothing has crossed yet"; + + auto crossed = t.update(M{match("a", "/a", "inactive")}); // 2 > grace: crosses THIS tick + ASSERT_EQ(crossed.newly_affected.size(), 1u); + EXPECT_EQ(crossed.newly_affected[0], "/a"); + ASSERT_TRUE(crossed.affected.count("/a")); + + auto still = t.update(M{match("a", "/a", "inactive")}); // 3 > grace: still affected, NOT new + EXPECT_TRUE(still.newly_affected.empty()) << "a continuing violation must not re-announce as new"; + EXPECT_TRUE(still.affected.count("/a")) << "sanity: still reported, just not as new"; +} + +TEST(LifecycleExpectation, NewlyUnreadableNamesOnlyTheNodeThatMaturedThisTick) { + LifecycleExpectationTracker t({"a"}, /*grace=*/5); + for (int i = 0; i < kDefaultUnmeasuredHoldTicks; ++i) { + auto below = t.update(M{match("a", "/a", std::string(""))}); + EXPECT_TRUE(below.newly_unreadable.empty()) << "iteration " << i; + } + auto matured = t.update(M{match("a", "/a", std::string(""))}); + ASSERT_EQ(matured.newly_unreadable.size(), 1u); + EXPECT_EQ(matured.newly_unreadable[0], "/a"); + auto still = t.update(M{match("a", "/a", std::string(""))}); + EXPECT_TRUE(still.newly_unreadable.empty()) << "a continuing unreadable node must not re-announce as new"; +} + +TEST(LifecycleExpectation, NewlyNotManagedNamesOnlyTheNodeThatMaturedThisTick) { + LifecycleExpectationTracker t({"a"}, /*grace=*/5); + for (int i = 0; i < kDefaultUnmeasuredHoldTicks; ++i) { + auto below = t.update(M{match("a", "/a", std::nullopt)}); + EXPECT_TRUE(below.newly_not_managed.empty()) << "iteration " << i; + } + auto matured = t.update(M{match("a", "/a", std::nullopt)}); + ASSERT_EQ(matured.newly_not_managed.size(), 1u); + EXPECT_EQ(matured.newly_not_managed[0], "/a"); + auto still = t.update(M{match("a", "/a", std::nullopt)}); + EXPECT_TRUE(still.newly_not_managed.empty()) << "a continuing not-managed node must not re-announce as new"; +} + +// The degenerate case R11 calls out by name: grace=0 makes EVERY node cross on its first +// tick, so "new first" has nothing to distinguish them by. The tie-break must still be +// deterministic - here, the tracker's own internal std::map iteration, i.e. lexicographic +// by fqn. This is not a design choice this slice adds on top; it falls out of `nodes` being +// a std::map in update() - pinning it here so a future change to that container is caught. +TEST(LifecycleExpectation, AllNodesCrossingOnTheSameTickAreAllNewInLexicographicOrder) { + LifecycleExpectationTracker t({"z", "a", "m"}, /*grace=*/0); + auto report = t.update(M{match("z", "/z", "inactive"), match("a", "/a", "inactive"), match("m", "/m", "inactive")}); + ASSERT_EQ(report.newly_affected.size(), 3u) << "grace=0: every one of them crosses on its first tick"; + EXPECT_EQ(report.newly_affected, (std::vector{"/a", "/m", "/z"})) + << "when everything is equally new, the tie-break is lexicographic"; +} + +// Config sweep: the SAME degenerate case again, but through the bare-name (fleet-wide) form +// instead of one-entry-per-node, so "many entries" and "one entry matching many nodes" are +// both covered by the ordering guarantee, not just the latter. +TEST(LifecycleExpectation, AllNodesCrossingViaOneBareNameEntryAreAllNewInLexicographicOrder) { + LifecycleExpectationTracker t({"controller_server"}, /*grace=*/0); + auto report = t.update(M{match("controller_server", "/right/controller_server", "inactive"), + match("controller_server", "/left/controller_server", "inactive")}); + ASSERT_EQ(report.newly_affected.size(), 2u); + EXPECT_EQ(report.newly_affected, (std::vector{"/left/controller_server", "/right/controller_server"})); +} + +// Scale: many nodes already crossed and reported in an earlier tick, then one MORE crosses +// on a later tick. Only the fresh one is new - the earlier batch, however large, must not +// reappear. +TEST(LifecycleExpectation, ANodeCrossingLaterIsNamedNewEvenWithManyEarlierViolations) { + constexpr int kEarlierCount = 24; + std::set entries; + M first_batch; + for (int i = 0; i < kEarlierCount; ++i) { + const std::string fqn = "/a" + std::to_string(i); + entries.insert(fqn); + first_batch.push_back(match(fqn, fqn, "inactive")); + } + entries.insert("/z"); + LifecycleExpectationTracker t(entries, /*grace=*/0); + auto batch = t.update(first_batch); // all kEarlierCount cross together + ASSERT_EQ(batch.newly_affected.size(), static_cast(kEarlierCount)); + + M second_batch = first_batch; + second_batch.push_back(match("/z", "/z", "inactive")); // "/z" joins on this tick + auto later = t.update(second_batch); + ASSERT_EQ(later.newly_affected.size(), 1u) << "only the fresh node crosses grace on this tick"; + EXPECT_EQ(later.newly_affected[0], "/z"); + EXPECT_EQ(later.affected.size(), static_cast(kEarlierCount) + 1) + << "sanity: the earlier batch is still reported, just not as new"; +} + +// Change over time: a node LEAVING affected and another ENTERING on the same tick must be +// handled independently. +TEST(LifecycleExpectation, ANodeLeavingAndAnotherEnteringOnTheSameTickAreIndependent) { + LifecycleExpectationTracker t({"a", "b"}, /*grace=*/0); + auto first = t.update(M{match("a", "/a", "inactive")}); // "a" crosses tick 1 + ASSERT_EQ(first.newly_affected.size(), 1u); + + auto mixed = t.update(M{match("a", "/a", "active"), match("b", "/b", "inactive")}); + EXPECT_FALSE(mixed.affected.count("/a")) << "\"a\" healed and must not be reported"; + ASSERT_TRUE(mixed.affected.count("/b")); + ASSERT_EQ(mixed.newly_affected.size(), 1u) << "only the entering node is new"; + EXPECT_EQ(mixed.newly_affected[0], "/b"); +} + +// ---- R13: the remote-supplied label gets its own budget, before a whole-detail backstop ---- + +TEST(LifecycleExpectation, PathologicallyLongLabelIsTrimmedNotTheRestOfTheDetail) { + LifecycleExpectationTracker t({"a"}, /*grace=*/0); + const std::string garbage_label(5000, 'x'); // no real lifecycle implementation ever sends this + auto report = t.update(M{match("a", "/a", garbage_label)}); + ASSERT_EQ(report.affected.size(), 1u); + const std::string & detail = report.affected.at("/a"); + EXPECT_LE(detail.size(), kMaxLifecycleDetailChars) << "the whole-detail backstop must bound it regardless"; + EXPECT_NE(detail.find("/a"), std::string::npos) << "the node name must survive the trim"; + EXPECT_NE(detail.find("required by 'a'"), std::string::npos) << "the naming entry must survive the trim"; + EXPECT_EQ(detail.find(garbage_label), std::string::npos) << "the raw untrimmed label must not appear whole"; +} + +TEST(LifecycleExpectation, LabelAloneIsTrimmedEvenWhenTheWholeDetailWouldFitTheBackstop) { + LifecycleExpectationTracker t({"a"}, /*grace=*/0); + const std::string long_label(kMaxLifecycleLabelChars + 20, 'x'); + auto report = t.update(M{match("a", "/a", long_label)}); + ASSERT_EQ(report.affected.size(), 1u); + const std::string & detail = report.affected.at("/a"); + EXPECT_LT(detail.size(), kMaxLifecycleDetailChars) + << "sanity: fqn+entry are short here, so a total under the backstop proves the LABEL budget " + "did the trimming, not the whole-detail one"; + EXPECT_NE(detail.find("..."), std::string::npos) << "the oversized label must be visibly truncated"; +} + +TEST(LifecycleExpectation, PathologicallyLongFqnIsCappedByTheDetailBackstop) { + LifecycleExpectationTracker t({"a"}, /*grace=*/0); + const std::string huge_fqn = "/" + std::string(2000, 'n'); + auto report = t.update(M{match("a", huge_fqn, "inactive")}); + ASSERT_EQ(report.affected.size(), 1u); + EXPECT_LE(report.affected.at(huge_fqn).size(), kMaxLifecycleDetailChars) + << "a pathological fqn must not defeat the per-detail backstop"; +} + +// The unreadable/not-managed detail builders re-apply the SAME whole-detail backstop, for +// the same reason: the fqn and the "required by" list are graph- and config-controlled +// length, and neither detail carries a live label to add its own separate budget for. +TEST(LifecycleExpectation, PathologicalFqnOnAMaturedUnreadableNodeIsCappedByTheDetailBackstop) { + LifecycleExpectationTracker t({"a"}, /*grace=*/0); + const std::string huge_fqn = "/" + std::string(2000, 'n'); + for (int i = 0; i <= kDefaultUnmeasuredHoldTicks; ++i) { + t.update(M{match("a", huge_fqn, std::string(""))}); + } + auto report = t.update(M{match("a", huge_fqn, std::string(""))}); + ASSERT_EQ(report.unreadable_affected.size(), 1u); + EXPECT_LE(report.unreadable_affected.at(huge_fqn).size(), kMaxLifecycleDetailChars); + EXPECT_EQ(report.unreadable_affected.at(huge_fqn).find("node "), 0u) << "the fixed prefix must survive at the head"; +} + +TEST(LifecycleExpectation, PathologicalLabelOnOneNodeDoesNotConsumeTheWholeBudget) { + LifecycleExpectationTracker t({"a", "b"}, /*grace=*/0); + const std::string garbage_label(5000, 'y'); + auto report = t.update(M{match("a", "/a", garbage_label), match("b", "/b", "inactive")}); + ASSERT_EQ(report.affected.size(), 2u); + EXPECT_LE(report.affected.at("/a").size(), kMaxLifecycleDetailChars); + EXPECT_LE(report.affected.at("/a").size() + report.affected.at("/b").size(), 2 * kMaxLifecycleDetailChars); +} + +// C1/C3: the test the name above promises. Enough ORDINARY (short, capped-at-length-1) +// nodes to independently exceed AggregatedFault::kMaxDescriptionChars on their own, PLUS one +// pathological-label node, assembled through AggregatedFault::describe exactly as the real +// caller would. +TEST(LifecycleExpectation, PathologicalLabelAlongsideACapFillingBatchDoesNotCrowdOutTheOrdinaryNodes) { + LifecycleExpectationTracker probe({"n0000"}, /*grace=*/0); + auto probe_report = probe.update(M{match("n0000", "/n0000", std::string("inactive"))}); + const std::size_t entry_len = probe_report.affected.at("/n0000").size(); + constexpr std::size_t kJoinSep = 2; // "; " - AggregatedFault::describe_ordered's join separator + const std::size_t cap = AggregatedFault::kMaxDescriptionChars; + std::size_t ordinary_count = 1; + while (ordinary_count * entry_len + (ordinary_count - 1) * kJoinSep <= cap) { + ++ordinary_count; + } + + std::set entries; + M matches; + std::vector ordinary_fqns; + for (std::size_t i = 0; i < ordinary_count; ++i) { + std::string suffix = std::to_string(i); + suffix.insert(0, 4 - std::min(4, suffix.size()), '0'); // 4-digit zero pad + const std::string id = "n" + suffix; + entries.insert(id); + ordinary_fqns.push_back("/" + id); + matches.push_back(match(id, "/" + id, std::string("inactive"))); + } + // 'A' (0x41) sorts before 'n' (0x6E), so this entry is always first in describe()'s + // lexicographic join order - not by luck, by construction. + entries.insert("AAA_pathological"); + matches.push_back(match("AAA_pathological", "/AAA_pathological", std::string(5000, 'z'))); + + LifecycleExpectationTracker t(entries, /*grace=*/0); + auto report = t.update(matches); + ASSERT_EQ(report.affected.size(), ordinary_count + 1); + const std::string & pathological_entry = report.affected.at("/AAA_pathological"); + ASSERT_LE(pathological_entry.size(), kMaxLifecycleDetailChars) + << "sanity: the whole-detail backstop must bound the pathological entry regardless"; + EXPECT_LT(pathological_entry.size(), kMaxLifecycleDetailChars) + << "the pathological entry landed exactly ON the whole-detail cap instead of comfortably " + "under it - the label trim did not run"; + EXPECT_NE(pathological_entry.find("required by 'AAA_pathological'"), std::string::npos) + << "the \"required by\" suffix was cut off the pathological entry"; + + const std::string desc = AggregatedFault::describe(report.affected); + std::size_t named = 0; + for (const auto & fqn : ordinary_fqns) { + if (desc.find(fqn) != std::string::npos) { + ++named; + } + } + EXPECT_GT(named, 0u) << "every ordinary node's name was crowded out of the final description " + "by the pathological node's entry"; +} + +// ---- Pruning and bounded bookkeeping ---- + +// An entry currently reported (stuck inactive past grace) must never be pruned - it is +// present, not absent, so it can never accumulate absence in the first place. +TEST(LifecycleExpectation, ReportedInactiveEntryIsNeverPruned) { + LifecycleExpectationTracker t({"a"}, /*grace=*/1, /*absence_grace=*/3, kDefaultNoMatchWarnTicks, + /*prune_ticks=*/2); + t.update(M{match("a", "/a", "inactive")}); // miss 1 <= grace + auto report = t.update(M{match("a", "/a", "inactive")}); // miss 2 > grace -> reported + ASSERT_FALSE(report.affected.empty()); + for (int i = 0; i < 10; ++i) { + report = t.update(M{match("a", "/a", "inactive")}); // present + inactive the whole time + EXPECT_FALSE(report.affected.empty()) << "iteration " << i; + } + EXPECT_EQ(t.tracked_count(), 1u) << "a currently-reported node must never be pruned away"; +} + +// The prune bound reclaims the WHOLE node atomically - one map, one entry, one horizon - +// but only an IDLE one. A node measured ACTIVE and then gone carries nothing, so reclaiming +// it loses nothing; reclaiming the entry reclaims everything it held, in the same tick. +TEST(LifecycleExpectation, PruneTicksReclaimsTheWholeIdleNodeAtomically) { + LifecycleExpectationTracker t({"a"}, /*grace=*/1, /*absence_grace=*/3, kDefaultNoMatchWarnTicks, + /*prune_ticks=*/2); + t.update(M{match("a", "/a", "inactive")}); + t.update(M{match("a", "/a", "active")}); // healed: the entry is now idle + for (int i = 0; i < 6; ++i) { + t.update(M{}); // absent far past prune_ticks(2) + } + EXPECT_EQ(t.tracked_count(), 0u) << "the idle node must be fully reclaimed, not partially"; + EXPECT_TRUE(t.update(M{match("a", "/a", "inactive")}).affected.empty()) + << "nothing may have survived the reclaim - the returning node was reported on its first " + "present-and-inactive tick instead of after grace + 1, which would mean a stale streak " + "outlived the entry that was supposed to carry it"; +} + +// The other half of that rule, and the one that matters: a node carrying EVIDENCE is never +// reclaimed by age, at any horizon, however long it stays gone. prune_ticks SHORTER than +// the absence grace is reachable through the documented config (prune_grace: 2 is one under +// the default absence grace of 3), which is exactly where an age horizon used to erase a +// clock before the absence rules ever saw it. +TEST(LifecycleExpectation, ReportedNodeIsNeverPrunedEvenWhenThePruneBoundUndercutsTheAbsenceGrace) { + LifecycleExpectationTracker t({"a"}, /*grace=*/1, kDefaultAbsenceGrace, kDefaultNoMatchWarnTicks, + /*prune_ticks=*/2); + t.update(M{match("a", "/a", "inactive")}); // 1 <= grace + auto reported = t.update(M{match("a", "/a", "inactive")}); // 2 > grace: reported + ASSERT_FALSE(reported.affected.empty()); + + for (int i = 1; i <= 10; ++i) { + auto absent = t.update(M{}); // well past prune_ticks(2) AND past the absence grace + EXPECT_EQ(absent.affected.count("/a"), 1u) + << "absent tick " << i << ": the node's confirmed violation was erased by the prune horizon"; + EXPECT_EQ(t.tracked_count(), 1u) << "absent tick " << i; + } +} + +// Scale: many reported nodes blinking on the same tick. Content follows the clocks, not the +// snapshot, so every one of them stays reported rather than dropping into a withheld limbo. +TEST(LifecycleExpectation, ManyReportedNodesBlinkingTogetherAllStayReported) { + constexpr int kNodeCount = 20; + std::set entries; + M matches; + std::vector fqns; + for (int i = 0; i < kNodeCount; ++i) { + const std::string fqn = "/n" + std::to_string(i); + entries.insert(fqn); + fqns.push_back(fqn); + matches.push_back(match(fqn, fqn, "inactive")); + } + LifecycleExpectationTracker t(entries, /*grace=*/1, /*absence_grace=*/3); + t.update(matches); // 1 <= grace, all of them + auto reported = t.update(matches); // 2 > grace, all reported + ASSERT_EQ(reported.affected.size(), static_cast(kNodeCount)); + + auto blink = t.update(M{}); // every one of them vanishes from the snapshot on the same tick + ASSERT_EQ(blink.affected.size(), static_cast(kNodeCount)) + << "every one of them must keep its content, not just however many a description names"; + EXPECT_TRUE(blink.pending.empty()) << "a node still in `affected` has nothing left to withhold for"; + for (const auto & fqn : fqns) { + EXPECT_EQ(blink.affected.count(fqn), 1u) << fqn; + EXPECT_TRUE(blink.newly_affected.empty()) << "a blink is not a fresh crossing"; + } +} + +// Change: one node crosses grace on the SAME tick another is deep inside its own absence. +TEST(LifecycleExpectation, NodeCrossingGraceDoesNotDisturbAnothersAbsenceBookkeeping) { + LifecycleExpectationTracker t({"a", "b"}, /*grace=*/1, /*absence_grace=*/3); + t.update(M{match("b", "/b", "inactive")}); // b: 1 <= grace + auto b_reported = t.update(M{match("b", "/b", "inactive")}); // b: 2 > grace -> reported + ASSERT_TRUE(b_reported.affected.count("/b")); + + auto mixed = t.update(M{match("a", "/a", "inactive")}); // a: 1 <= grace; b: absent 1 + EXPECT_TRUE(mixed.affected.count("/a") == 0) << "a is only at 1 <= grace"; + EXPECT_EQ(mixed.affected.count("/b"), 1u) << "b, already reported, keeps its content through the blink"; + EXPECT_EQ(mixed.pending.count("/a"), 1u) << "a's own below-grace streak"; + EXPECT_TRUE(mixed.newly_affected.empty()) << "nothing crossed on this tick"; + + auto crossed = t.update(M{match("a", "/a", "inactive")}); // a: 2 > grace -> reported; b: absent 2 + ASSERT_EQ(crossed.affected.size(), 2u) << "a joins b, which never left"; + ASSERT_EQ(crossed.newly_affected.size(), 1u) << "only a crossed grace on this tick"; + EXPECT_EQ(crossed.newly_affected[0], "/a"); +} + +// Sustained absence CONTINUES the streak instead of discarding it: the node was measured +// not-active, nothing has said otherwise, and being gone is not an answer. +TEST(LifecycleExpectation, SustainedAbsenceContinuesTheStreakAndEventuallyConfirmsIt) { + LifecycleExpectationTracker t({"a"}, /*grace=*/2, /*absence_grace=*/1); + t.update(M{match("a", "/a", "inactive")}); // streak 1 <= grace + EXPECT_TRUE(t.update(M{}).affected.empty()) << "absent 1 == absence_grace: held, not advanced"; + EXPECT_TRUE(t.update(M{}).affected.empty()) << "absent 2 > absence_grace: streak 2 == grace, not past it"; + auto crossed = t.update(M{}); // absent 3: streak 3 > grace + ASSERT_EQ(crossed.affected.count("/a"), 1u) + << "absence discarded the streak instead of continuing it, so a node that leaves while " + "violating is never confirmed"; + EXPECT_EQ(crossed.newly_affected, (std::vector{"/a"})); + EXPECT_NE(crossed.affected.at("/a").find("inactive"), std::string::npos) + << "the detail must still name the state the node was last measured in"; + EXPECT_NE(crossed.affected.at("/a").find("has since left the graph"), std::string::npos); +} + +// `pending` gives way to CONTENT, never to silence: the withheld-clear hold ends because +// the tracker finally settled the node's status, not because it gave up on it. +TEST(LifecycleExpectation, PendingBecomesContentWhenAbsenceOutlivesTheAbsenceGrace) { + LifecycleExpectationTracker t({"a"}, /*grace=*/3, /*absence_grace=*/2); + EXPECT_EQ(t.update(M{match("a", "/a", "inactive")}).pending.count("/a"), 1u); // streak 1 + EXPECT_EQ(t.update(M{}).pending.count("/a"), 1u) << "absent 1: inside the blink tolerance, held"; + EXPECT_EQ(t.update(M{}).pending.count("/a"), 1u) << "absent 2 == absence_grace: still held"; + EXPECT_EQ(t.update(M{}).pending.count("/a"), 1u) << "absent 3: streak 2, still below grace"; + EXPECT_EQ(t.update(M{}).pending.count("/a"), 1u) << "absent 4: streak 3 == grace, still below"; + auto settled = t.update(M{}); // absent 5: streak 4 > grace + EXPECT_TRUE(settled.pending.empty()) << "the hold must end by SETTLING, not by discarding"; + EXPECT_EQ(settled.affected.count("/a"), 1u); +} + +// A node the detector already reported keeps its content through a blink AND past it - it +// is never silently cleared at any point, which is what a level-triggered emitter needs to +// avoid a raise/clear/raise churn on a node that is simply gone. +TEST(LifecycleExpectation, ReportedNodeKeepsItsContentThroughAndPastTheAbsenceGrace) { + LifecycleExpectationTracker t({"a"}, /*grace=*/1, /*absence_grace=*/2); + t.update(M{match("a", "/a", "inactive")}); // 1 <= grace + auto reported = t.update(M{match("a", "/a", "inactive")}); // 2 > grace: reported + ASSERT_FALSE(reported.affected.empty()) << "sanity: the node must already be reported before it blinks"; + + for (int i = 1; i <= 8; ++i) { + auto absent = t.update(M{}); + EXPECT_EQ(absent.affected.count("/a"), 1u) << "absent tick " << i << ": content was dropped"; + EXPECT_TRUE(absent.pending.empty()) << "absent tick " << i << ": content and a withhold are exclusive"; + EXPECT_TRUE(absent.newly_affected.empty()) << "absent tick " << i << ": no re-announcement as new"; + } +} + +// R10: the re-raise on return is CORRECT and must be preserved, not "fixed". +TEST(LifecycleExpectation, ReportedNodeReRaisesOnReturnWithoutReEarningGrace) { + LifecycleExpectationTracker t({"a"}, /*grace=*/1, /*absence_grace=*/3); + t.update(M{match("a", "/a", "inactive")}); // 1 <= grace + auto reported = t.update(M{match("a", "/a", "inactive")}); // 2 > grace: reported + ASSERT_FALSE(reported.affected.empty()); + + t.update(M{}); // absent 1: held, not cleared + + auto back = t.update(M{match("a", "/a", "inactive")}); // still inactive on return + EXPECT_FALSE(back.affected.empty()) << "the streak survived the blink - no fresh grace to re-earn"; + EXPECT_TRUE(back.pending.empty()) << "past grace again immediately, not a fresh below-grace streak"; +} + +// An IDLE node absent past prune_ticks is reclaimed and the map shrinks - the age horizon +// still does its job, on the only entries that have nothing to lose. +TEST(LifecycleExpectation, IdleNodeAbsentPastPruneTicksIsPrunedAndMapShrinks) { + LifecycleExpectationTracker t({"a"}, /*grace=*/2, /*absence_grace=*/1, kDefaultNoMatchWarnTicks, + /*prune_ticks=*/3); + t.update(M{match("a", "/a", "active")}); // measured healthy: idle from the first tick + EXPECT_EQ(t.tracked_count(), 1u); + t.update(M{}); // absent 1 + t.update(M{}); // absent 2 + t.update(M{}); // absent 3 (== prune_ticks, not past it yet) + EXPECT_EQ(t.tracked_count(), 1u); + t.update(M{}); // absent 4 (> prune_ticks) -> reclaimed + EXPECT_EQ(t.tracked_count(), 0u) << "an idle node absent past prune_ticks must be reclaimed"; +} + +// ---- The tracked-node cap: what bounds the map now that evidence outlives absence ---- + +// Identity churn where every respawn is HEALTHY: each entry goes idle immediately, so the +// age horizon alone keeps the map small and the cap is never even approached. +TEST(LifecycleExpectation, ChurningHealthyIdentitiesAreReclaimedByTheAgeHorizon) { + constexpr int kPruneTicks = 2; + LifecycleExpectationTracker t({"a"}, /*grace=*/1, /*absence_grace=*/1, kDefaultNoMatchWarnTicks, kPruneTicks); + for (int i = 0; i < 50; ++i) { + t.update(M{match("a", "/ns" + std::to_string(i) + "/a", "active")}); + EXPECT_LE(t.tracked_count(), static_cast(kPruneTicks) + 1) + << "iteration " << i << ": an idle entry is retained for every fqn ever seen"; + } + for (int i = 0; i < 5; ++i) { + t.update(M{}); + } + EXPECT_EQ(t.tracked_count(), 0u) << "every churned identity must be reclaimed once it is gone"; +} + +// Identity churn where every respawn is VIOLATING: nothing is ever idle, so the age horizon +// reclaims nothing (by design - that horizon is what a restart loop evaded) and the CAP is +// the only bound. The PRESENT node always wins a slot: the entries for the identities that +// are GONE are collapsed into a count instead, so the one node actually in the graph is +// always checked and nothing is refused. The count keeps the fault's content non-empty, so +// freeing those slots heals nothing. +TEST(LifecycleExpectation, ChurningViolatingIdentitiesAreCollapsedSoThePresentNodeIsAlwaysTracked) { + constexpr int kCap = 8; + LifecycleExpectationTracker t({"a"}, /*grace=*/0, /*absence_grace=*/1, kDefaultNoMatchWarnTicks, + /*prune_ticks=*/2, kDefaultUnmeasuredHoldTicks, kCap); + int saturations = 0; + for (int i = 0; i < 50; ++i) { + const std::string fqn = "/ns" + std::to_string(i) + "/a"; + auto report = t.update(M{match("a", fqn, "inactive")}); + saturations += report.tracking_saturated ? 1 : 0; + ASSERT_LE(t.tracked_count(), static_cast(kCap)) + << "iteration " << i << ": the map grew past the cap, so nothing bounds it at all"; + ASSERT_EQ(report.affected.count(fqn), 1u) + << "iteration " << i + << ": the PRESENT violating node was not reported - a cap full of entries for nodes that are " + "GONE refused the only node actually in the graph, so the detector reports health it " + "declined to check"; + } + EXPECT_EQ(saturations, 0) + << "a present node was refused while the cap held entries for departed nodes, which can never " + "become idle again - the refusal would last for the life of the process"; + + // Nothing was thrown away to make that room: the departed identities are still content, + // as a count. + auto steady = t.update(M{}); + ASSERT_FALSE(steady.affected.empty()) << "collapsing the departed entries healed the fault outright"; + const std::string description = AggregatedFault::describe(steady.affected); + EXPECT_NE(description.find("more required node(s) left the graph"), std::string::npos) + << "the collapsed identities left no trace in what the operator reads, so their evidence was " + "silently discarded: " + << description; +} + +// The other side of the same cap: when every tracked node is PRESENT there is nothing to +// collapse, so the newcomer genuinely IS refused - and that must be reported on every such +// tick, not once, because the caller has to withhold GRAPH_NODE_INACTIVE's clear for as long +// as a required node is going unchecked. +TEST(LifecycleExpectation, AllPresentAtTheCapRefusesTheNewcomerAndSaysSoOnEveryTick) { + constexpr int kCap = 2; + LifecycleExpectationTracker t({"/a", "/b", "/c"}, /*grace=*/0, /*absence_grace=*/1, kDefaultNoMatchWarnTicks, + LifecycleExpectationTracker::kNoPrune, kDefaultUnmeasuredHoldTicks, kCap); + const M all{match("/a", "/a", "inactive"), match("/b", "/b", "inactive"), match("/c", "/c", "inactive")}; + for (int i = 0; i < 5; ++i) { + auto report = t.update(all); + EXPECT_TRUE(report.tracking_saturated) + << "tick " << i << ": a required node is present and going unchecked, and the report does not say so"; + EXPECT_EQ(report.saturation_started, i == 0) + << "tick " << i << ": the saturation EDGE must be the first tick of the episode, and only that one"; + EXPECT_EQ(t.tracked_count(), static_cast(kCap)); + EXPECT_EQ(report.affected.count("/c"), 0u) << "the refused node must not be reported as measured"; + } + + // The episode ends when the refused node leaves, and a LATER one is reported again rather + // than silenced by the first. + const M two{match("/a", "/a", "inactive"), match("/b", "/b", "inactive")}; + auto ended = t.update(two); + EXPECT_FALSE(ended.tracking_saturated) << "nothing was refused, so nothing is saturated"; + auto recurred = t.update(all); + EXPECT_TRUE(recurred.tracking_saturated); + EXPECT_TRUE(recurred.saturation_started) + << "a second, real saturation was not surfaced because the first episode spent the latch"; +} + +// At the cap, IDLE entries are reclaimed FIRST - they carry nothing, so freeing them costs +// nothing and the newcomer gets in without anything being refused. +TEST(LifecycleExpectation, IdleEntriesAreReclaimedFirstAtTheCapSoTheNewcomerIsAdmitted) { + constexpr int kCap = 4; + std::set entries{"/keep", "/idle0", "/idle1", "/idle2", "/newcomer"}; + LifecycleExpectationTracker t(entries, /*grace=*/0, /*absence_grace=*/1, kDefaultNoMatchWarnTicks, + LifecycleExpectationTracker::kNoPrune, kDefaultUnmeasuredHoldTicks, kCap); + // One entry carrying a confirmed violation, three measured healthy: the map is exactly full. + t.update(M{match("/keep", "/keep", "inactive"), match("/idle0", "/idle0", "active"), + match("/idle1", "/idle1", "active"), match("/idle2", "/idle2", "active")}); + ASSERT_EQ(t.tracked_count(), static_cast(kCap)); + + auto report = t.update(M{match("/newcomer", "/newcomer", "inactive")}); + EXPECT_FALSE(report.tracking_saturated) << "idle entries were available - nothing should have been refused"; + EXPECT_EQ(report.affected.count("/newcomer"), 1u) << "the newcomer was refused despite reclaimable idle entries"; + EXPECT_EQ(report.affected.count("/keep"), 1u) << "the entry carrying evidence must survive the reclaim"; + EXPECT_LE(t.tracked_count(), static_cast(kCap)); +} + +// Scale, at the REAL shipped cap rather than a test-shrunk one: kDefaultTrackedNodeCap +// violating identities fit, and the very next one is refused. +TEST(LifecycleExpectation, TheShippedCapAdmitsExactlyItsOwnCountOfViolatingNodes) { + std::set entries; + M matches; + for (int i = 0; i < kDefaultTrackedNodeCap; ++i) { + const std::string fqn = "/n" + std::to_string(i); + entries.insert(fqn); + matches.push_back(match(fqn, fqn, "inactive")); + } + entries.insert("/one_too_many"); + LifecycleExpectationTracker t(entries, /*grace=*/0); + auto full = t.update(matches); + ASSERT_EQ(full.affected.size(), static_cast(kDefaultTrackedNodeCap)); + EXPECT_FALSE(full.tracking_saturated) << "exactly the cap must fit without saturating"; + + matches.push_back(match("/one_too_many", "/one_too_many", "inactive")); + auto over = t.update(matches); + EXPECT_TRUE(over.tracking_saturated) << "one node past the cap must be refused, and said so"; + EXPECT_EQ(over.affected.count("/one_too_many"), 0u) << "the refused node must not be reported"; + EXPECT_EQ(t.tracked_count(), static_cast(kDefaultTrackedNodeCap)); +} + +// ---- Settling: what absence is allowed to continue ---- + +// The transient the settling rule exists for: a present, HEALTHY, managed node whose +// get_state path is missing from one sweep reads "no tracked lifecycle" for a tick. If that +// is the last thing seen before a clean shutdown, continuing it literally matures a healthy +// departure into a permanent fault. Swept across every uncorroborated run length, so an +// off-by-one at the bound is caught rather than one arbitrary length being pinned. +TEST(LifecycleExpectation, UncorroboratedUnmeasuredRunBeforeAHealthyDepartureRaisesNothing) { + for (int blink = 1; blink < kDefaultObservationSettleTicks; ++blink) { + LifecycleExpectationTracker t({"a"}, /*grace=*/1); + for (int i = 0; i < 10; ++i) { + t.update(M{match("a", "/a", "active")}); // measured healthy, over and over + } + for (int i = 0; i < blink; ++i) { + t.update(M{match("a", "/a", std::nullopt)}); // the missed sweep(s) + } + for (int i = 0; i < kDefaultUnmeasuredHoldTicks + kDefaultAbsenceGrace + 5; ++i) { + auto report = t.update(M{}); // and then it is gone, cleanly + ASSERT_TRUE(report.not_managed_affected.empty()) + << "blink=" << blink << ", absent tick " << i + << ": a healthy managed node whose lifecycle services were missing from " << blink + << " sweep(s) before a clean shutdown was reported as not managed"; + ASSERT_TRUE(report.unreadable_affected.empty()) << "blink=" << blink; + ASSERT_TRUE(report.affected.empty()) << "blink=" << blink; + } + EXPECT_TRUE(t.update(M{}).pending.empty()) + << "blink=" << blink + << ": the released node is still UNSETTLED, so it withholds GRAPH_NODE_INACTIVE's clear for " + "every other node for the life of the process"; + } +} + +// The other side of the same bound, and the reason it cannot simply be "ignore unmeasured +// readings before a departure": a node that is GENUINELY unmeasurable when it leaves must +// still be reported. Same shape as the test above, one tick longer. +TEST(LifecycleExpectation, CorroboratedUnmeasuredRunBeforeADepartureIsStillReported) { + LifecycleExpectationTracker t({"a"}, /*grace=*/1); + for (int i = 0; i < 10; ++i) { + t.update(M{match("a", "/a", "active")}); + } + for (int i = 0; i < kDefaultObservationSettleTicks; ++i) { + t.update(M{match("a", "/a", std::nullopt)}); + } + bool reported = false; + for (int i = 0; i < kDefaultUnmeasuredHoldTicks + kDefaultAbsenceGrace + 5 && !reported; ++i) { + reported = !t.update(M{}).not_managed_affected.empty(); + } + EXPECT_TRUE(reported) << "a node observed not-managed for exactly the settling budget before it left " + "the graph was never reported - corroboration is not supposed to be a way " + "for a genuinely unmeasurable node to leave quietly"; +} + +// A real measurement needs no corroboration at all: one not-active read is a fact about the +// node, so a node that departs immediately after it still confirms. Without this the +// settling rule would swallow the very case the detector exists for at grace: 0. +TEST(LifecycleExpectation, OneMeasuredNotActiveReadBeforeADepartureStillConfirms) { + LifecycleExpectationTracker t({"a"}, /*grace=*/2); + t.update(M{match("a", "/a", "inactive")}); // exactly one real measurement, then gone + bool confirmed = false; + for (int i = 0; i < kDefaultAbsenceGrace + 10 && !confirmed; ++i) { + confirmed = !t.update(M{}).affected.empty(); + } + EXPECT_TRUE(confirmed) << "a node measured not-active once and then gone was never confirmed - a " + "lifecycle label is a measurement, not something a sweep can invent, so it " + "needs no corroborating"; +} + +// ---- The cap: a present node always wins a slot ---- + +// A cap held entirely by entries for DEPARTED nodes must not refuse a PRESENT one. Those +// entries can never become idle again (becoming idle needs a real measurement of a node that +// is gone), so without collapsing them the refusal lasts for the life of the process - and a +// refused node never enters `affected` or `pending`, which makes GRAPH_NODE_INACTIVE emit a +// level-triggered CLEAR every tick while that node reads not-active. +TEST(LifecycleExpectation, DepartedEntriesAreCollapsedSoAPresentBrokenNodeIsStillReported) { + constexpr int kCap = 2; + LifecycleExpectationTracker t({"/d0", "/d1", "/live"}, /*grace=*/0, /*absence_grace=*/1, kDefaultNoMatchWarnTicks, + LifecycleExpectationTracker::kNoPrune, kDefaultUnmeasuredHoldTicks, kCap); + t.update(M{match("/d0", "/d0", "inactive"), match("/d1", "/d1", "inactive")}); + ASSERT_EQ(t.tracked_count(), 2u) << "the cap must be full of departed-to-be entries, or nothing is tested"; + for (int i = 0; i < 3; ++i) { + t.update(M{}); // both leave for good, past the absence grace + } + + auto report = t.update(M{match("/live", "/live", "inactive")}); + EXPECT_FALSE(report.tracking_saturated) + << "a present, broken, required node was refused by entries for two nodes that are gone"; + EXPECT_EQ(report.affected.count("/live"), 1u) + << "the present node was admitted but not reported - it is the one node actually in the graph"; + // And the departed evidence survived being collapsed, as content. + EXPECT_NE(AggregatedFault::describe(report.affected).find("more required node(s) left the graph"), std::string::npos) + << "collapsing the departed entries discarded their evidence: " << AggregatedFault::describe(report.affected); +} + +// The bound on how many departed entries stay individually NAMED. Three, because three +// maximally-long details are all one description holds - a fourth name could never be shown +// however the ordering fell out, so keeping it only costs a slot a present node may need. +TEST(LifecycleExpectation, AtMostThreeDepartedEntriesStayNamedWhenTheCapMakesRoom) { + constexpr int kCap = 8; + std::set entries; + M departing; + for (int i = 0; i < kCap; ++i) { + const std::string fqn = "/d" + std::to_string(i); + entries.insert(fqn); + departing.push_back(match(fqn, fqn, "inactive")); + } + entries.insert("/live"); + LifecycleExpectationTracker t(entries, /*grace=*/0, /*absence_grace=*/1, kDefaultNoMatchWarnTicks, + LifecycleExpectationTracker::kNoPrune, kDefaultUnmeasuredHoldTicks, kCap); + t.update(departing); + ASSERT_EQ(t.tracked_count(), static_cast(kCap)); + for (int i = 0; i < 3; ++i) { + t.update(M{}); // every one of them leaves for good + } + + t.update(M{match("/live", "/live", "inactive")}); + EXPECT_EQ(t.tracked_count(), static_cast(kMaxNamedDepartedEntries) + 1) + << "the departed entries were not collapsed down to the named bound, so they go on holding " + "slots that can never appear in any description"; +} + +// Change in the other direction: a node whose entry was collapsed and which then COMES BACK. +// It is tracked and measured afresh - the tracker no longer knows which fqn it collapsed - and +// the count that stands in for the identity it lost keeps the fault raised. That is the +// documented behaviour, not an accident, so it is pinned rather than left to be discovered. +TEST(LifecycleExpectation, ANodeReturningAfterItsEntryWasCollapsedIsMeasuredAfresh) { + constexpr int kCap = 1; + LifecycleExpectationTracker t({"/gone", "/live"}, /*grace=*/0, /*absence_grace=*/1, kDefaultNoMatchWarnTicks, + LifecycleExpectationTracker::kNoPrune, kDefaultUnmeasuredHoldTicks, kCap); + t.update(M{match("/gone", "/gone", "inactive")}); + for (int i = 0; i < 3; ++i) { + t.update(M{}); // "/gone" leaves for good, carrying a confirmed violation + } + auto collapsed = t.update(M{match("/live", "/live", "inactive")}); // its slot goes to the live node + ASSERT_EQ(collapsed.affected.count("/live"), 1u); + + // "/gone" comes back HEALTHY. It cannot have the slot ("/live" is present and broken), and + // even once it does it starts from zero - nothing remembers the entry that was collapsed. + auto returned = t.update(M{match("/live", "/live", "active"), match("/gone", "/gone", "active")}); + EXPECT_EQ(returned.affected.count("/gone"), 0u) << "a returning node was reported without being measured"; + EXPECT_NE(AggregatedFault::describe(returned.affected).find("more required node(s) left the graph"), + std::string::npos) + << "the collapsed count was dropped when the node came back, so a fault that a departure must " + "never heal was healed by one: " + << AggregatedFault::describe(returned.affected); +} + +// ---- Entries matching nothing: unrelated per-entry mechanism, unaffected by this slice ---- + +TEST(LifecycleExpectation, EntryMatchingNothingIsReportedOnce) { + LifecycleExpectationTracker t({"typoed_name"}, /*grace=*/0, /*absence_grace=*/3, + /*no_match_warn_ticks=*/2); + EXPECT_TRUE(t.update(M{}).entries_matching_nothing.empty()); // 1 + EXPECT_TRUE(t.update(M{}).entries_matching_nothing.empty()); // 2 == threshold + auto report = t.update(M{}); // 3 > threshold + ASSERT_EQ(report.entries_matching_nothing.size(), 1u); + EXPECT_EQ(report.entries_matching_nothing[0], "typoed_name"); + EXPECT_TRUE(t.update(M{}).entries_matching_nothing.empty()) << "reported once, not every tick"; +} + +TEST(LifecycleExpectation, EntryThatMatchesIsNeverReportedAsMatchingNothing) { + LifecycleExpectationTracker t({"a"}, /*grace=*/0, /*absence_grace=*/3, /*no_match_warn_ticks=*/1); + for (int i = 0; i < 5; ++i) { + EXPECT_TRUE(t.update(M{match("a", "/a", "active")}).entries_matching_nothing.empty()); + } +} +} // namespace diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_watcher.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_watcher.cpp index e4e9798b7..77906d0cd 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_watcher.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_watcher.cpp @@ -14,10 +14,15 @@ #include #include +#include +#include +#include #include #include +#include #include +#include #include #include "ros2_medkit_gateway/core/providers/introspection_provider.hpp" // App, IntrospectionInput, ServiceInfo @@ -84,6 +89,27 @@ ros2_medkit_gateway::IntrospectionInput managed_node_snapshot(const std::string return in; } +// Same shape, but with App::id decoupled from the node it is bound to - the re-bind +// cases below move `fqn` between snapshots while `id` stays put. +ros2_medkit_gateway::IntrospectionInput managed_app_snapshot(const std::string & id, const std::string & fqn) { + auto in = managed_node_snapshot(fqn); + in.apps.front().id = id; + return in; +} + +// Same App::id and same fqn, but the lifecycle services live somewhere else - a +// `~/get_state` remap in the launch file, or a manifest-bound app re-pointed at another +// node's services. The ~/transition_event topic is DERIVED from that path, so this is a +// different node behind an unchanged name: the half of the binding identity that no other +// test moves on its own. +ros2_medkit_gateway::IntrospectionInput remapped_service_snapshot(const std::string & fqn, + const std::string & service_prefix) { + auto in = managed_node_snapshot(service_prefix); + in.apps.front().id = fqn; + in.apps.front().bound_fqn = fqn; + return in; +} + // N managed nodes in one snapshot, so a single update() queues more GetState work than the // per-tick blocking budget can possibly run. ros2_medkit_gateway::IntrospectionInput managed_nodes_snapshot(const std::vector & ids) { @@ -102,6 +128,28 @@ constexpr auto kPumpBudget = std::chrono::milliseconds(100); // matching, and the watcher's subscriptions are in a callback group NO node-wide executor // collects - pump_events() is the only thing that runs them, exactly as the plugin's tick // thread does in production. +// Cancels the executor and joins its spin thread on every exit path, including the one a +// fatal gtest assertion takes. +class ExecutorJoin { + public: + ExecutorJoin(rclcpp::executors::SingleThreadedExecutor & exec, std::thread & spin) : exec_(exec), spin_(spin) { + } + ExecutorJoin(const ExecutorJoin &) = delete; + ExecutorJoin & operator=(const ExecutorJoin &) = delete; + ExecutorJoin(ExecutorJoin &&) = delete; + ExecutorJoin & operator=(ExecutorJoin &&) = delete; + ~ExecutorJoin() { + exec_.cancel(); + if (spin_.joinable()) { + spin_.join(); + } + } + + private: + rclcpp::executors::SingleThreadedExecutor & exec_; + std::thread & spin_; +}; + bool publish_until_label(ros2_medkit_graph_watchdog::LifecycleWatcher & w, const rclcpp::Publisher::SharedPtr & pub, const lifecycle_msgs::msg::TransitionEvent & msg, const std::string & id, @@ -358,3 +406,170 @@ TEST_F(LifecycleWatcherTest, DepartedNodeRetainsLastLabelByFqnThenPrunes) { w.update(ros2_medkit_gateway::IntrospectionInput{}, /*tick=*/10 + kRetentionTicks + 1); EXPECT_FALSE(w.departed_state_of(fqn).has_value()); } + +// A tracked id whose BINDING moves (same App::id, different fqn / get_state path) is a +// different node wearing the same name: the old node's label and ~/transition_event +// subscription must be dropped and the id re-seeded through the new binding's GetState. +// The old label here is a KNOWN "inactive" with the self-heal budget already spent, so +// nothing else can heal it - an entry kept across the re-bind would keep gating the id +// on a node that is no longer behind it. +TEST_F(LifecycleWatcherTest, RebindToALiveNodeDropsTheOldLabelAndSeedsTheNewBinding) { + const std::string id = "/appk"; + ros2_medkit_graph_watchdog::LifecycleWatcher w(node_.get(), &mtx_); + + // The OLD binding: no GetState service (seeds fail), label driven over a live + // ~/transition_event. The NEW binding: a GetState service that answers "active". + auto old_pub = node_->create_publisher( + "/rb_old/transition_event", rclcpp::QoS(rclcpp::KeepLast(10)).reliable()); + auto new_srv = node_->create_service( + "/rb_new/get_state", [](const std::shared_ptr & /*req*/, + const std::shared_ptr & resp) { + resp->current_state.id = lifecycle_msgs::msg::State::PRIMARY_STATE_ACTIVE; + resp->current_state.label = "active"; + }); + + w.update(managed_app_snapshot(id, "/rb_old"), /*tick=*/1); + ASSERT_TRUE(w.state_of(id).has_value()); + const int fresh_budget = w.reseeds_remaining_for_test(id); + ASSERT_GT(fresh_budget, 0); + + // The NEW binding's GetState service lives on node_, so node_ has to be spun for the + // watcher's reader to get an answer out of it. That executor never runs the watcher's + // own subscriptions - they sit in a callback group it does not collect - so the + // transition events below still go through pump_events(), exactly as in production. + rclcpp::executors::SingleThreadedExecutor exec; + exec.add_node(node_); + std::thread spin([&exec]() { + exec.spin(); + }); + // A fatal assertion below returns straight out of the test body, and a joinable + // std::thread destroyed that way calls std::terminate - which aborts the whole binary + // and takes every remaining case in this file with it, hiding the failure that started + // it. The guard makes the join happen on that path too. + ExecutorJoin join_guard{exec, spin}; + + lifecycle_msgs::msg::TransitionEvent msg; + msg.goal_state.label = "inactive"; + ASSERT_TRUE(publish_until_label(w, old_pub, msg, id, "inactive")); + + // Drain the self-heal budget against the OLD (serviceless) binding, so the assertion + // below cannot be satisfied by a leftover re-seed happening to hit the new path - the + // re-bind handling itself has to do the healing. + for (std::uint64_t tick = 2; w.reseeds_remaining_for_test(id) > 0 && tick < 10; ++tick) { + w.update(managed_app_snapshot(id, "/rb_old"), tick); + } + ASSERT_EQ(w.reseeds_remaining_for_test(id), 0); + ASSERT_EQ(w.state_of(id).value_or(""), "inactive"); + + // The re-bind: same id, now bound to the live "/rb_new" node. + w.update(managed_app_snapshot(id, "/rb_new"), /*tick=*/10); + EXPECT_EQ(w.state_of(id).value_or(""), "active") + << "after a re-bind the id must carry the NEW binding's seeded state, not the old node's label"; + EXPECT_TRUE(w.node_ok(id)) << "the old binding's non-active label must stop gating the id"; + EXPECT_EQ(w.reseeds_remaining_for_test(id), fresh_budget) + << "a re-bound id must be re-seeded as a NEW entry (fresh self-heal budget), not healed in place"; + + // The old subscription must be gone with the old entry: the old node's transitions + // must no longer reach this id. + for (int i = 0; i < 10; ++i) { + old_pub->publish(msg); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + w.pump_events(kPumpBudget); + } + EXPECT_EQ(w.state_of(id).value_or(""), "active") + << "a transition published by the OLD binding after the re-bind overwrote the new binding's state"; +} + +// Re-bind to a binding with NO live node behind it: the id follows the new-binding +// seeding semantics - tracked with an empty (unknown, ungated) label after the failed +// GetState - and must NOT keep the old node's label. +TEST_F(LifecycleWatcherTest, RebindToADeadBindingLeavesStateUnknownNotTheOldLabel) { + const std::string id = "/appd"; + ros2_medkit_graph_watchdog::LifecycleWatcher w(node_.get(), &mtx_); + + w.update(managed_app_snapshot(id, "/rbd_old"), /*tick=*/1); + ASSERT_TRUE(w.state_of(id).has_value()); + w.set_state_for_test(id, "inactive"); // the OLD binding's observed label + + w.update(managed_app_snapshot(id, "/rbd_new"), /*tick=*/2); + EXPECT_EQ(w.state_of(id).value_or(""), "") + << "a re-bind to a dead binding must leave the id unknown (empty seed), not wearing the old label"; + EXPECT_TRUE(w.node_ok(id)) << "unknown must not gate - only a KNOWN non-active label does"; +} + +// The guard against overcorrection: an UNCHANGED binding across updates must never be +// treated as a re-bind. The instrument is the self-heal budget, which counts what the +// remote actually pays (GetState round trips): a drop + re-create would reset it to the +// fresh value on every tick and record a phantom departure. +TEST_F(LifecycleWatcherTest, SameBindingAcrossUpdatesIsNeverTreatedAsARebind) { + const std::string fqn = "/stable"; + ros2_medkit_graph_watchdog::LifecycleWatcher w(node_.get(), &mtx_); + + w.update(managed_node_snapshot(fqn), /*tick=*/1); + const int fresh_budget = w.reseeds_remaining_for_test(fqn); + ASSERT_GT(fresh_budget, 0); + + // Non-active label + budget left -> exactly one charged re-seed per update. + w.update(managed_node_snapshot(fqn), /*tick=*/2); + EXPECT_EQ(w.reseeds_remaining_for_test(fqn), fresh_budget - 1) + << "one update with the same binding must charge exactly one re-seed - a re-created entry " + "would reset the budget to " + << fresh_budget; + w.update(managed_node_snapshot(fqn), /*tick=*/3); + w.update(managed_node_snapshot(fqn), /*tick=*/4); + EXPECT_EQ(w.reseeds_remaining_for_test(fqn), 0) + << "the budget must drain monotonically across same-binding updates and stay drained"; + EXPECT_TRUE(w.state_of(fqn).has_value()) << "the entry itself must survive every same-binding update"; + EXPECT_FALSE(w.departed_state_of(fqn).has_value()) << "an unchanged binding must never be recorded as a departure"; +} + +// The OTHER half of the binding identity, on its own. Every re-bind test above moves the +// fqn and the get_state path together (the snapshot builder derives one from the other), so +// the path arm of the drop condition is never the thing that fires. Here only the SERVICE +// path moves - the id and the fqn are unchanged - and it is still a different node: the +// ~/transition_event subscription is derived from that path, so an entry kept across this +// would keep enforcing the old node's label and listening on the old node's topic. +TEST_F(LifecycleWatcherTest, MovingOnlyTheServicePathIsStillARebind) { + const std::string fqn = "/remapped"; + ros2_medkit_graph_watchdog::LifecycleWatcher w(node_.get(), &mtx_); + + w.update(remapped_service_snapshot(fqn, "/svc_old"), /*tick=*/1); + ASSERT_TRUE(w.state_of(fqn).has_value()); + const int fresh_budget = w.reseeds_remaining_for_test(fqn); + ASSERT_GT(fresh_budget, 0); + w.set_state_for_test(fqn, "inactive"); // the OLD binding's observed label + + // Same id, same fqn - only the services moved. + w.update(remapped_service_snapshot(fqn, "/svc_new"), /*tick=*/2); + EXPECT_EQ(w.state_of(fqn).value_or(""), "") + << "a node whose lifecycle services moved kept the old binding's label - the get_state path is half " + "the binding identity, and the transition_event topic is derived from it"; + EXPECT_EQ(w.reseeds_remaining_for_test(fqn), fresh_budget) + << "the entry must be re-created (fresh self-heal budget), not healed in place"; +} + +// What a re-bind means for departed_state_of(): the OLD binding effectively departed, so +// its last observed state is recorded under ITS fqn - same record shape and same +// retention mechanics as a node vanishing from the snapshot. +TEST_F(LifecycleWatcherTest, RebindRecordsTheOldBindingAsDepartedUnderItsOwnFqn) { + const std::string id = "/appr"; + constexpr int kRetentionTicks = 3; + ros2_medkit_graph_watchdog::LifecycleWatcher w(node_.get(), &mtx_, kRetentionTicks); + + w.update(managed_app_snapshot(id, "/rbr_old"), /*tick=*/1); + ASSERT_TRUE(w.state_of(id).has_value()); + w.set_state_for_test(id, "inactive"); + + w.update(managed_app_snapshot(id, "/rbr_new"), /*tick=*/5); + const auto departed = w.departed_state_of("/rbr_old"); + ASSERT_TRUE(departed.has_value()) << "a re-bind must record the OLD binding as departed under its fqn"; + EXPECT_EQ(departed->label, "inactive"); + EXPECT_TRUE(departed->saw_transition); + EXPECT_FALSE(departed->error_terminated); + EXPECT_FALSE(w.departed_state_of("/rbr_new").has_value()) + << "the NEW binding is present, not departed - only the old one may be recorded"; + + // The record ages out through the ordinary retention prune. + w.update(managed_app_snapshot(id, "/rbr_new"), /*tick=*/5 + kRetentionTicks + 1); + EXPECT_FALSE(w.departed_state_of("/rbr_old").has_value()); +} diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_orphan_integration.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_orphan_integration.cpp index 67092d185..c370b2308 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_orphan_integration.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_orphan_integration.cpp @@ -80,7 +80,7 @@ class OrphanIntegrationTest : public ::testing::Test { sink_ = std::make_shared("orphan_it_sink"); srv_ = sink_->create_service( "/fault_manager/report_fault", - [this](const std::shared_ptr req, std::shared_ptr resp) { + [this](const std::shared_ptr & req, const std::shared_ptr & resp) { { std::lock_guard lk(mtx_); received_.push_back(*req); diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_qos_mismatch_integration.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_qos_mismatch_integration.cpp index 8be794a42..b2074a7a4 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_qos_mismatch_integration.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_qos_mismatch_integration.cpp @@ -80,7 +80,7 @@ class QosMismatchIntegrationTest : public ::testing::Test { sink_ = std::make_shared("qm_it_sink"); srv_ = sink_->create_service( "/fault_manager/report_fault", - [this](const std::shared_ptr req, std::shared_ptr resp) { + [this](const std::shared_ptr & req, const std::shared_ptr & resp) { { std::lock_guard lk(mtx_); received_.push_back(*req); diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_reliability_gate.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_reliability_gate.cpp index 74f7e62f1..d9da17948 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_reliability_gate.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_reliability_gate.cpp @@ -35,9 +35,9 @@ class ReliabilityGateTest : public ::testing::Test { void TearDown() override { node_.reset(); } - static IntrospectionInput snap(std::vector app_ids) { + static IntrospectionInput snap(const std::vector & app_ids) { IntrospectionInput in; - for (auto & id : app_ids) { + for (const auto & id : app_ids) { App a; a.id = id; in.apps.push_back(a);