From eeb01717931a75154b9b53d4a05e77e4ca3e30cf Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Tue, 28 Jul 2026 17:56:43 -0500 Subject: [PATCH 1/2] fix(neutron-understack): avoid setting HCG on centralized router ports link_vxlan_network_ha_chassis_group anchored the internal router-interface Logical_Router_Port to the network's unified HA_Chassis_Group. Every vxlan gateway router is centralized (options:chassis set), and anchoring one of its ports as a distributed gateway port makes ovn-northd log "Bad configuration: distributed gateway port configured on port ... on L3 gateway router" and ignore the setting anyway. Skip that anchor whenever the owning router is centralized; the network HCG population (the part that actually fixes external/baremetal port routing) is untouched. Also fix scripts/cleanup_dead_ovn_ha_chassis.py to match: - Drop the now-invalid LRP anchor from Phase 2's repopulation action. - Phase 2 previously derived the gateway chassis solely from the router's own per-router HA_Chassis_Group (neutron-). That group is only ever created once, at gateway-port-creation time, and nothing recreates it if deleted -- several routers in the fleet had options:chassis set but no such group, so Phase 2 skipped them forever. Now reads Logical_Router.options.chassis directly as the primary, always-reliable source, falling back to the per-router HCG only when the router isn't centralized. --- .../neutron_understack/routers.py | 64 ++++--- .../neutron_understack/tests/test_routers.py | 50 ++++- scripts/cleanup_dead_ovn_ha_chassis.py | 178 +++++++++--------- 3 files changed, 180 insertions(+), 112 deletions(-) diff --git a/python/neutron-understack/neutron_understack/routers.py b/python/neutron-understack/neutron_understack/routers.py index 3b7dab422..80d51b523 100644 --- a/python/neutron-understack/neutron_understack/routers.py +++ b/python/neutron-understack/neutron_understack/routers.py @@ -204,27 +204,33 @@ def create_uplink_port(segment: NetworkSegment, network_id: str, txn=None) -> No def link_vxlan_network_ha_chassis_group(_resource, _event, _trigger, payload) -> None: - """Populate the unified network HCG (and anchor the internal LRP) for vxlan. - - Workaround for a neutron bug exposed in 2026.1. For a router with a vxlan-type - external gateway, neutron pins the Logical_Router to a single - chassis via ``options:chassis`` and creates a per-router HA_Chassis_Group - (neutron-) carrying that chassis, but it never sets - ha_chassis_group on the gateway LRP. neutron's link_network_ha_chassis_group - (fired when the internal LRP is created) bails out at its - ``if not gw_lrps[0].ha_chassis_group`` check, so it never copies the chassis - into the per-network unified HCG (neutron-). External/baremetal - ports on that network reference the empty network HCG, so no chassis owns - them and routing/ARP breaks. - - We do what link_network_ha_chassis_group would have done: populate the unified - network HCG with sync_ha_chassis_group_network_unified, then anchor the internal - router-interface LRP to that same HCG. The gateway chassis is sourced from the - global HA_Chassis table (all *live* records must share one chassis_name) so the - fix fires even before the external gateway port is attached. Rows pointing at a - chassis no longer present in the Southbound DB (e.g. left behind by a - decommissioned/replaced host) are excluded before checking for uniqueness, so a - single stale row elsewhere in the fleet doesn't block every vxlan network. + """Populate the unified network HCG for vxlan external gateways. + + Workaround for a neutron bug exposed in 2026.1. For a vxlan-type external + gateway, neutron pins the Logical_Router to a chassis via ``options:chassis`` + but never populates ``gateway_chassis`` on the gateway LRP (that's only done + for VLAN/FLAT via the OVN L3 scheduler). neutron's own + ``link_network_ha_chassis_group`` builds its chassis list from that empty + ``gateway_chassis``, so it syncs the per-network unified HCG + (neutron-) with zero members, leaving external/baremetal ports on + that network with no owning chassis and broken routing/ARP. + + We populate that same unified HCG ourselves via + sync_ha_chassis_group_network_unified. The chassis comes from the global + HA_Chassis table (all *live* rows must share one chassis_name), so this works + even before the gateway port is attached. Rows for chassis no longer in the + Southbound DB (decommissioned/replaced hosts) are excluded so one stale row + doesn't block every vxlan network. + + We do NOT set ha_chassis_group on the internal router-interface LRP. + Upstream never does this on a Logical_Router_Port (only on external + Logical_Switch_Ports), and doing so on a router that already has + options:chassis set -- true for every vxlan gateway -- makes ovn-northd log + "Bad configuration: distributed gateway port configured on ... L3 gateway + router" and ignore it anyway, since the router is already centralized on one + chassis. We guard on options:chassis rather than dropping the anchor + outright in case this fixup is ever reused for a genuinely distributed + router. For VLAN/FLAT networks neutron's handler already populates the network HCG correctly; we detect that and return early. @@ -301,8 +307,20 @@ def link_vxlan_network_ha_chassis_group(_resource, _event, _trigger, payload) -> txn, ) - # Anchor the internal router-interface LRP to the same unified HCG. - if nb_idl.lookup("Logical_Router_Port", lrp_name, default=None): + # Anchor the internal LRP to the HCG, unless the router is + # centralized (options:chassis set) -- see docstring. + lr = nb_idl.lookup( + "Logical_Router", ovn_utils.ovn_name(router_id), default=None + ) + router_is_centralized = bool(lr and lr.options.get("chassis")) + if router_is_centralized: + LOG.debug( + "Not anchoring LRP %(lrp)s to HCG %(hcg)s: router " + "%(router)s is centralized (options:chassis set), so " + "northd would reject a distributed gateway port on it", + {"lrp": lrp_name, "hcg": hcg, "router": router_id}, + ) + elif nb_idl.lookup("Logical_Router_Port", lrp_name, default=None): txn.add( nb_idl.db_set( "Logical_Router_Port", diff --git a/python/neutron-understack/neutron_understack/tests/test_routers.py b/python/neutron-understack/neutron_understack/tests/test_routers.py index 1379150e0..d4e4ccca6 100644 --- a/python/neutron-understack/neutron_understack/tests/test_routers.py +++ b/python/neutron-understack/neutron_understack/tests/test_routers.py @@ -407,7 +407,9 @@ def _payload(mocker, router_id="router-1", port_id="port-1", network_id="net-1") ) @staticmethod - def _client(mocker, ha_chassis_rows, lrp, network_hcg=None, live_chassis=None): + def _client( + mocker, ha_chassis_rows, lrp, network_hcg=None, live_chassis=None, router=None + ): nb_idl = mocker.MagicMock() nb_idl.db_list_rows.return_value.execute.return_value = ha_chassis_rows @@ -416,6 +418,8 @@ def lookup(table, _name, default=None): return network_hcg # None by default (unpopulated vxlan HCG) if table == "Logical_Router_Port": return lrp + if table == "Logical_Router": + return router # None by default (not centralized) return default nb_idl.lookup.side_effect = lookup @@ -541,3 +545,47 @@ def test_lrp_missing_still_populates_network_hcg(self, mocker): # The network HCG is still populated even if the LRP is not found yet. sync.assert_called_once() nb_idl.db_set.assert_not_called() + + def test_skips_lrp_anchor_when_router_is_centralized(self, mocker): + # Every vxlan gateway router has options:chassis set (that's what makes + # it vxlan rather than VLAN/FLAT). Anchoring one of its ports as a + # distributed gateway port makes ovn-northd log "Bad configuration: + # distributed gateway port configured on ... L3 gateway router" and + # ignore the setting anyway, so we must not attempt it. + hc = mocker.Mock(chassis_name="chassis-1") + lrp = mocker.Mock(ha_chassis_group=[]) + router = mocker.Mock(options={"chassis": "some-chassis-uuid"}) + client, nb_idl = self._client( + mocker, ha_chassis_rows=[hc], lrp=lrp, router=router + ) + sync = self._patch_sync(mocker) + mocker.patch("neutron_understack.routers.ovn_client", return_value=client) + + link_vxlan_network_ha_chassis_group(None, None, None, self._payload(mocker)) + + # The network HCG is still populated (that's what actually fixes the + # external/baremetal ports)... + sync.assert_called_once() + # ...but the internal LRP is left alone. + nb_idl.db_set.assert_not_called() + + def test_anchors_lrp_when_router_is_not_centralized(self, mocker): + # Defensive case: if this fixup is ever reused for a genuinely + # distributed router (no options:chassis), the old anchoring behavior + # still applies. + hc = mocker.Mock(chassis_name="chassis-1") + lrp = mocker.Mock(ha_chassis_group=[]) + router = mocker.Mock(options={}) + client, nb_idl = self._client( + mocker, ha_chassis_rows=[hc], lrp=lrp, router=router + ) + self._patch_sync(mocker) + mocker.patch("neutron_understack.routers.ovn_client", return_value=client) + + link_vxlan_network_ha_chassis_group(None, None, None, self._payload(mocker)) + + nb_idl.db_set.assert_called_once_with( + "Logical_Router_Port", + "lrp-port-1", + ("ha_chassis_group", "net-hcg-uuid"), + ) diff --git a/scripts/cleanup_dead_ovn_ha_chassis.py b/scripts/cleanup_dead_ovn_ha_chassis.py index 32154d19d..361298119 100755 --- a/scripts/cleanup_dead_ovn_ha_chassis.py +++ b/scripts/cleanup_dead_ovn_ha_chassis.py @@ -15,13 +15,27 @@ 2. Empty per-network unified HA_Chassis_Groups. Once (1) has happened (or for any other reason a network's unified group ended up empty), its - external/baremetal ports have no chassis to claim them. Normally you'd - fix this per-router by detaching/reattaching a subnet to re-fire - link_vxlan_network_ha_chassis_group. This script instead derives the - right chassis directly from the router's own HA_Chassis_Group and writes - it to the network's group (plus anchors the internal router-interface - LRP to it), mirroring what that function does — for every affected - network in one run, no manual per-router action needed. + external/baremetal ports have no chassis to claim them. This script + repopulates it for every affected network in one run, no manual + per-router action needed. The chassis is sourced two ways, tried in + order: + a. The router's own options:chassis, if set (every centralized/vxlan + gateway router has this set directly -- it's what makes it + centralized). This is the primary source and is always reliable: + the router-scoped HA_Chassis_Group (neutron-) that + link_vxlan_network_ha_chassis_group would otherwise derive this + from is only ever created once, at gateway-port-creation time, and + nothing recreates it if it's later deleted. Several routers in this + fleet have options:chassis set but no such group anymore. + b. Falling back to that per-router HA_Chassis_Group only when the + router isn't centralized (no options:chassis). Routers created long + enough ago may have neither -- those are reported as skipped, not + guessed at. + It does not touch any Logical_Router_Port: routers.py no longer anchors + the internal router-interface LRP to the network HCG either, since that + trips ovn-northd's "distributed gateway port configured on ... L3 + gateway router" warning on every vxlan gateway router for no operational + benefit. 3. Leftover networks from a half-deleted network. When a neutron network is deleted but OVN is left with a stale Logical_Switch (neutron-), @@ -44,10 +58,11 @@ Caveat: repopulation here is a direct, minimal OVN write. It does not replicate neutron's candidate filtering (chassis-as-gw eligibility, physnet connectivity, priority ordering across multiple chassis) that -sync_ha_chassis_group_network_unified performs. It only picks a chassis -if the router's own HA_Chassis_Group resolves to exactly one distinct live -chassis. This is safe for the common single-gateway-chassis case; in a -multi-chassis HA setup, prefer re-triggering the real neutron code path. +sync_ha_chassis_group_network_unified performs. When falling back to a +router's own HA_Chassis_Group (2b above), it only picks a chassis if that +group resolves to exactly one distinct live chassis. This is safe for the +common single-gateway-chassis case; in a multi-chassis HA setup, prefer +re-triggering the real neutron code path. """ import argparse @@ -61,6 +76,7 @@ SB_POD = "ovn-ovsdb-sb-0" OVN_NAMESPACE = "openstack" +NEUTRON_PREFIX = "neutron-" OVN_NETWORK_ID_EXT_ID_KEY = "neutron:network_id" OVN_ROUTER_ID_EXT_ID_KEY = "neutron:router_id" HA_CHASSIS_GROUP_HIGHEST_PRIORITY = 32767 @@ -147,11 +163,15 @@ def get_all_ha_chassis_groups(kubectl_base: list[str]) -> list[dict]: ) -def get_all_router_ports(kubectl_base: list[str]) -> list[dict]: - """Every Logical_Router_Port row in the Northbound DB.""" - return _ovn_list( - kubectl_base, NB_POD, "ovn-nbctl", "Logical_Router_Port", "name,external_ids" +def get_router_chassis_by_id(kubectl_base: list[str]) -> dict[str, str | None]: + """Map neutron router_id -> its Logical_Router options:chassis, if set.""" + rows = _ovn_list( + kubectl_base, NB_POD, "ovn-nbctl", "Logical_Router", "name,options" ) + return { + r["name"].removeprefix(NEUTRON_PREFIX): (r.get("options") or {}).get("chassis") + for r in rows + } # --- Phase 1: stale HA_Chassis cleanup ------------------------------------- @@ -256,24 +276,23 @@ def plan_repopulation( groups: list[dict], all_ha_chassis_by_uuid: dict[str, str], stale_uuids: set[str], - router_ports: list[dict], + live_chassis: set[str], + router_chassis_by_id: dict[str, str | None], ) -> tuple[list[dict], list[tuple[str, str]]]: """Work out which empty per-network HCGs can be safely repopulated. - A group is a candidate if: it's a per-network unified HCG (has - neutron:network_id in external_ids), it has no members once stale ones - are excluded, and its associated router's own HCG resolves to exactly - one distinct live chassis. + A group is a candidate if it's a per-network unified HCG (has + neutron:network_id in external_ids) with no members once stale ones are + excluded. The chassis to (re)add is sourced from, in order: + + 1. The router's own options:chassis, if set and still live. This is the + primary, most reliable source (see module docstring point 2). + 2. The router's own per-router HA_Chassis_Group (neutron-), + only tried when the router isn't centralized (no options:chassis), + and only if it resolves to exactly one distinct live chassis. """ groups_by_name = {g["name"]: g for g in groups} - lrps_by_network: dict[str, list[dict]] = {} - for lrp in router_ports: - ext = lrp.get("external_ids") or {} - net_name = ext.get("neutron:network_name") - if net_name: - lrps_by_network.setdefault(net_name, []).append(lrp) - plan = [] skipped = [] for g in groups: @@ -292,52 +311,44 @@ def plan_repopulation( skipped.append((g["name"], "no router_id in HCG external_ids")) continue - router_group = groups_by_name.get(f"neutron-{router_id}") - if not router_group: - skipped.append((g["name"], f"router HCG neutron-{router_id} not found")) - continue - - router_members = _as_list(router_group.get("ha_chassis")) - router_remaining = [m for m in router_members if m not in stale_uuids] - chassis_names = { - all_ha_chassis_by_uuid[m] - for m in router_remaining - if m in all_ha_chassis_by_uuid - } - if len(chassis_names) != 1: - skipped.append( - ( - g["name"], - f"router HCG resolves to {len(chassis_names)} distinct " - "live chassis (expected exactly 1)", + target_chassis = None + skip_reason = None + + centralized_chassis = router_chassis_by_id.get(router_id) + if centralized_chassis: + if centralized_chassis in live_chassis: + target_chassis = centralized_chassis + else: + skip_reason = ( + f"router's options:chassis ({centralized_chassis}) is " + "not a live chassis" ) - ) - continue - target_chassis = next(iter(chassis_names)) - - switch_name = f"neutron-{network_id}" - candidates = [ - lrp - for lrp in lrps_by_network.get(switch_name, []) - if (lrp.get("external_ids") or {}).get("neutron:router_name") - == f"neutron-{router_id}" - ] - if not candidates: - skipped.append( - ( - g["name"], - f"no internal Logical_Router_Port found for network " - f"{network_id} on router {router_id}", + else: + router_group = groups_by_name.get(f"neutron-{router_id}") + if not router_group: + skip_reason = ( + f"router is not centralized and its HCG " + f"neutron-{router_id} was not found" ) - ) + else: + router_members = _as_list(router_group.get("ha_chassis")) + router_remaining = [m for m in router_members if m not in stale_uuids] + chassis_names = { + all_ha_chassis_by_uuid[m] + for m in router_remaining + if m in all_ha_chassis_by_uuid + } + if len(chassis_names) == 1: + target_chassis = next(iter(chassis_names)) + else: + skip_reason = ( + f"router HCG resolves to {len(chassis_names)} " + "distinct live chassis (expected exactly 1)" + ) + + if target_chassis is None: + skipped.append((g["name"], skip_reason)) continue - if len(candidates) > 1: - log.warning( - "Multiple internal LRPs found for network %s on router %s; using %s", - network_id, - router_id, - candidates[0]["name"], - ) plan.append( { @@ -346,7 +357,6 @@ def plan_repopulation( "network_id": network_id, "router_id": router_id, "target_chassis": target_chassis, - "lrp_name": candidates[0]["name"], } ) return plan, skipped @@ -362,13 +372,11 @@ def print_repopulation_report(plan: list[dict], skipped: list[tuple[str, str]]) print(f"[DRY-RUN] Network HCG : {p['group_name']}") print(f"[DRY-RUN] Router : {p['router_id']}") print(f"[DRY-RUN] Chassis : {p['target_chassis']}") - print(f"[DRY-RUN] Anchor LRP : {p['lrp_name']}") print( f"[DRY-RUN] Action : create HA_Chassis " f"chassis_name={p['target_chassis']} " f"priority={HA_CHASSIS_GROUP_HIGHEST_PRIORITY}; add to " - f"{p['group_uuid']}; set {p['lrp_name']} " - f"ha_chassis_group={p['group_uuid']}" + f"{p['group_uuid']}" ) print() @@ -386,10 +394,7 @@ def execute_repopulation( print(f"About to repopulate {len(plan)} network HCG(s):") for p in plan: - print( - f" - {p['group_name']}: chassis {p['target_chassis']}, " - f"anchor LRP {p['lrp_name']}" - ) + print(f" - {p['group_name']}: chassis {p['target_chassis']}") if not _confirm("Apply these Phase 2 changes?", assume_yes): print("Skipped Phase 2 — no changes made.\n") return @@ -410,11 +415,6 @@ def execute_repopulation( p["group_uuid"], "ha_chassis", hc_id, - "--", - "set", - "Logical_Router_Port", - p["lrp_name"], - f"ha_chassis_group={p['group_uuid']}", ] print(f"Repopulating {len(plan)} network HCG(s) in one transaction …") @@ -429,8 +429,6 @@ def execute_repopulation( # --- Phase 3: delete truly-orphaned leftover networks --------------------- -NEUTRON_PREFIX = "neutron-" - # Safety tripwire: if more than this fraction of OVN networks look orphaned, # the OpenStack credentials almost certainly aren't admin (or can't list every # network) rather than the fleet being mostly dead. Abort unless --force. @@ -924,10 +922,14 @@ def main() -> None: execute_cleanup(cleanup_records, kubectl_base, args.assume_yes) if not args.skip_repopulate: - log.info("Fetching Logical_Router_Port rows …") - router_ports = get_all_router_ports(kubectl_base) + log.info("Fetching Logical_Router options:chassis …") + router_chassis_by_id = get_router_chassis_by_id(kubectl_base) plan, skipped = plan_repopulation( - groups, all_ha_chassis_by_uuid, stale_uuids, router_ports + groups, + all_ha_chassis_by_uuid, + stale_uuids, + live_chassis, + router_chassis_by_id, ) if not args.execute: From 575418c7baac47b5f7305046be03d8d4eb02228b Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Tue, 28 Jul 2026 18:32:33 -0500 Subject: [PATCH 2/2] fix(neutron): rebase to latest upstream stable/2026.1 We need the fix in https://review.opendev.org/c/openstack/neutron/+/998088 to ensure that our router ports get properly scheduled and not incorrectly pinned which breaks their traffic. Plus this backports https://review.opendev.org/c/openstack/neutron/+/999158 which at this time is proposed upstream. --- containers/neutron/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/neutron/Dockerfile b/containers/neutron/Dockerfile index 303437d99..6f2c67f91 100644 --- a/containers/neutron/Dockerfile +++ b/containers/neutron/Dockerfile @@ -6,7 +6,7 @@ FROM quay.io/airshipit/neutron:${OPENSTACK_VERSION}-ubuntu_noble AS build COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ # renovate: name=openstack/neutron repo=https://github.com/rackerlabs/neutron.git branch=understack/2026.1 -ARG NEUTRON_GIT_REF=51f968afde5eb80c04e0bd01391e9e3344812c9a +ARG NEUTRON_GIT_REF=7cad1b179a9645f9cd0256a5d90a97ebfb95d8d0 ADD --keep-git-dir=true https://github.com/rackerlabs/neutron.git#${NEUTRON_GIT_REF} /src/neutron RUN git -C /src/neutron fetch --unshallow --tags