From f57c290d4b0f34f19595ffc1c7fb97978319180a Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 14 Aug 2026 08:26:01 -0500 Subject: [PATCH 1/2] feat(e2e): let a network's presence reach a POP cell locally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The harness installed NSO's CRDs on the control plane and both POP cells but not on the Karmada hub, and installed no NSO propagation policy at all. A NetworkContext is written on the hub and propagated to the cells serving its location, so there was no local path by which one could reach a cell and no way to test the claim path end to end. Install the NSO CRDs on the hub too, apply NSO's own federation propagation policy there, and label the member clusters infra.datum.net/gateways=enabled so that policy actually selects them — it matches nothing without the label, and does so silently. Also grant the hub identity networkbindings on networking.datumapis.com. Compute asks for a network in a location by writing a binding on the hub, and the hub role granted nothing in that group at all. --- Taskfile.yaml | 46 +++++++++++++++++++++++++-- config/base/downstream-rbac/rbac.yaml | 12 +++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/Taskfile.yaml b/Taskfile.yaml index 2ae2eab1..828cfa94 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -517,13 +517,20 @@ tasks: {{.KUBECONFIG_DIR}}/karmada.yaml \ {{.CLUSTER_NAME}} \ {{.INTERNAL_KUBECONFIG}} - # ── Apply city-code label ────────────────────────────────────────── + # ── Apply cluster labels ─────────────────────────────────────────── + # city-code is what compute's federator places deployments by. + # + # infra.datum.net/gateways=enabled is what NSO's propagation policy selects + # on, and every cell in infra carries it. Without it that policy matches no + # cluster and propagates nothing — silently, with no error and no event — + # so a NetworkContext would never reach a POP cell here. - | kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \ label cluster {{.CLUSTER_NAME}} \ topology.datum.net/city-code={{.CITY_CODE}} \ + infra.datum.net/gateways=enabled \ --overwrite - echo "Labeled cluster '{{.CLUSTER_NAME}}' with topology.datum.net/city-code={{.CITY_CODE}}" + echo "Labeled cluster '{{.CLUSTER_NAME}}' with topology.datum.net/city-code={{.CITY_CODE}} and infra.datum.net/gateways=enabled" # ════════════════════════════════════════════════════════════════════════ # CRD installation @@ -535,6 +542,8 @@ tasks: - task: _e2e:crds:compute - task: _e2e:crds:nso - task: _e2e:crds:quota + # Depends on _e2e:crds:nso having registered the kinds it selects. + - task: _e2e:federation:nso-policy _e2e:crds:compute: internal: true @@ -555,12 +564,17 @@ tasks: _e2e:crds:nso: internal: true - desc: "Apply NSO CRDs to control-plane and POP cell clusters" + desc: "Apply NSO CRDs to all clusters and the Karmada API server" cmds: # NSO CRDs (NetworkBinding, SubnetClaim, etc.) are installed on the # control-plane as well as POP cells. The control-plane operator needs them # so that Subnet/SubnetClaim informer watches can start without cache errors, # even though NSO controllers themselves only run on POP cells. + # + # The Karmada API server needs them too. A network's presence in a location + # is a NetworkContext that is written on the hub and propagated to the cells + # serving that location, so without these CRDs the hub cannot store one and + # nothing reaches a cell. - | go mod download go.datum.net/network-services-operator NSO_VERSION=$(go list -m -json go.datum.net/network-services-operator \ @@ -569,12 +583,38 @@ tasks: echo "NSO CRDs from: ${NSO_CRD_PATH}" for KC in \ {{.KUBECONFIG_DIR}}/control-plane.yaml \ + {{.KUBECONFIG_DIR}}/karmada.yaml \ {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ {{.KUBECONFIG_DIR}}/pop-ord.yaml; do echo "Installing NSO CRDs → $(basename $KC .yaml)..." kubectl --kubeconfig="$KC" apply -k "${NSO_CRD_PATH}" --server-side --validate=false done + _e2e:federation:nso-policy: + internal: true + desc: "Apply NSO's federation propagation policy to the Karmada API server" + cmds: + # NSO ships the policy that carries its own resources to cells. Applying the + # real file, rather than a compute-authored stand-in, is what makes a local + # NetworkContext-reaches-a-cell test meaningful: it exercises the same + # selectors production uses. + # + # config/federation has no kustomization.yaml, and resourceinterpreters.yaml + # only aggregates Gateway API status, so apply just the policy. + # + # This runs after _e2e:crds:nso so the selected kinds are discoverable when + # Karmada's resource detector picks the policy up. Selectors for kinds this + # environment does not install (Gateway API, Envoy Gateway, cert-manager, + # external-dns) simply never match. + - | + NSO_VERSION=$(go list -m -json go.datum.net/network-services-operator \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['Version'])") + NSO_FEDERATION_PATH="$(go env GOMODCACHE)/go.datum.net/network-services-operator@${NSO_VERSION}/config/federation" + echo "NSO federation policy from: ${NSO_FEDERATION_PATH}" + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply \ + -f "${NSO_FEDERATION_PATH}/clusterpropagationpolicy.yaml" \ + --server-side --validate=false + _e2e:crds:quota: internal: true desc: "Apply Milo quota CRDs to all clusters and the Karmada API server" diff --git a/config/base/downstream-rbac/rbac.yaml b/config/base/downstream-rbac/rbac.yaml index f2a94a37..7ad15ef5 100644 --- a/config/base/downstream-rbac/rbac.yaml +++ b/config/base/downstream-rbac/rbac.yaml @@ -16,6 +16,18 @@ rules: - apiGroups: ["compute.datumapis.com"] resources: ["workloaddeployments", "workloaddeployments/status", "instances", "instances/status"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + # Compute asks for a network's presence in a location by writing a + # NetworkBinding on the hub. NSO turns every binding for the same network and + # location into one shared NetworkContext and propagates it to the cells + # serving that location, where the claim path can use it. Compute owns the + # bindings it creates, including releasing them, but only reads the contexts + # NSO derives from them. + - apiGroups: ["networking.datumapis.com"] + resources: ["networkbindings", "networkbindings/status"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["networking.datumapis.com"] + resources: ["networkcontexts"] + verbs: ["get", "list", "watch"] - apiGroups: ["policy.karmada.io"] resources: ["propagationpolicies", "clusterpropagationpolicies"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] From fd3e10457f5596b14434d94fa8e49d92d93e5df0 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 14 Aug 2026 09:24:46 -0500 Subject: [PATCH 2/2] chore(e2e): drop the Python interpreter requirement from the e2e environment Running the local e2e environment required Python plus the PyYAML module, an interpreter dependency unrelated to anything the environment actually tests. Every use is expressible with tools the Taskfile already depends on, so a contributor now needs only go, kubectl, docker, and a POSIX shell. Co-Authored-By: Claude Opus 5 --- Taskfile.yaml | 46 ++++++++++++++++------------------------------ 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/Taskfile.yaml b/Taskfile.yaml index 828cfa94..f2af01e0 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -410,29 +410,18 @@ tasks: get secret karmada-kubeconfig \ -n karmada-system \ -o jsonpath='{.data.kubeconfig}' \ - | base64 -d > {{.KUBECONFIG_DIR}}/karmada-raw.yaml - # Rewrite the server address to the NodePort exposed on localhost - python3 - {{.KUBECONFIG_DIR}}/karmada-raw.yaml {{.KUBECONFIG_DIR}}/karmada.yaml 127.0.0.1 {{.KARMADA_API_NODEPORT}} << 'PYEOF' - import sys, yaml - - src, dst, host, port = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] - - with open(src) as f: - cfg = yaml.safe_load(f) - - for cluster in cfg.get('clusters', []): - old = cluster['cluster'].get('server', '') - cluster['cluster']['server'] = f'https://{host}:{port}' - # The cert is for the internal cluster IP, so skip TLS verification. - # This is a local dev-only environment. - cluster['cluster']['insecure-skip-tls-verify'] = True - cluster['cluster'].pop('certificate-authority-data', None) - print(f" karmada server: {old} → https://{host}:{port}", file=sys.stderr) - - with open(dst, 'w') as f: - yaml.dump(cfg, f, default_flow_style=False) - PYEOF - rm {{.KUBECONFIG_DIR}}/karmada-raw.yaml + | base64 -d > {{.KUBECONFIG_DIR}}/karmada.yaml + # Rewrite the server address to the NodePort exposed on localhost. The + # embedded cert is for the internal cluster IP, so verification has to be + # dropped as well; --insecure-skip-tls-verify also clears the embedded CA. + # This is a local dev-only environment. + for CLUSTER in $(kubectl config get-clusters --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml | tail -n +2); do + echo " karmada server: ${CLUSTER} → https://127.0.0.1:{{.KARMADA_API_NODEPORT}}" + kubectl config set-cluster "${CLUSTER}" \ + --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \ + --server=https://127.0.0.1:{{.KARMADA_API_NODEPORT}} \ + --insecure-skip-tls-verify=true + done # Copy karmada.yaml → downstream.yaml so Chainsaw tests that declare # cluster: downstream can load it (chainsaw-config.yaml references this path). - cp {{.KUBECONFIG_DIR}}/karmada.yaml {{.KUBECONFIG_DIR}}/downstream.yaml @@ -577,8 +566,7 @@ tasks: # nothing reaches a cell. - | go mod download go.datum.net/network-services-operator - NSO_VERSION=$(go list -m -json go.datum.net/network-services-operator \ - | python3 -c "import sys, json; print(json.load(sys.stdin)['Version'])") + NSO_VERSION=$(go list -m -f '{{`{{.Version}}`}}' go.datum.net/network-services-operator) NSO_CRD_PATH="$(go env GOMODCACHE)/go.datum.net/network-services-operator@${NSO_VERSION}/config/crd" echo "NSO CRDs from: ${NSO_CRD_PATH}" for KC in \ @@ -607,8 +595,7 @@ tasks: # environment does not install (Gateway API, Envoy Gateway, cert-manager, # external-dns) simply never match. - | - NSO_VERSION=$(go list -m -json go.datum.net/network-services-operator \ - | python3 -c "import sys, json; print(json.load(sys.stdin)['Version'])") + NSO_VERSION=$(go list -m -f '{{`{{.Version}}`}}' go.datum.net/network-services-operator) NSO_FEDERATION_PATH="$(go env GOMODCACHE)/go.datum.net/network-services-operator@${NSO_VERSION}/config/federation" echo "NSO federation policy from: ${NSO_FEDERATION_PATH}" kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply \ @@ -624,8 +611,7 @@ tasks: # against project Milo control planes without cache startup errors. - | go mod download go.miloapis.com/milo - MILO_VERSION=$(go list -m -json go.miloapis.com/milo \ - | python3 -c "import sys, json; print(json.load(sys.stdin)['Version'])") + MILO_VERSION=$(go list -m -f '{{`{{.Version}}`}}' go.miloapis.com/milo) QUOTA_CRD_PATH="$(go env GOMODCACHE)/go.miloapis.com/milo@${MILO_VERSION}/config/crd/bases/quota" echo "Milo quota CRDs from: ${QUOTA_CRD_PATH}" for KC in \ @@ -754,7 +740,7 @@ tasks: # NodePort rather than the in-cluster Service the management pod uses. - | CP_IP=$(docker inspect {{.KIND_CONTROL_PLANE}}-control-plane \ - | python3 -c "import sys,json; print(list(json.load(sys.stdin)[0]['NetworkSettings']['Networks'].values())[0]['IPAddress'])") + -f '{{`{{range .NetworkSettings.Networks}}{{.IPAddress}}{{"\n"}}{{end}}`}}' | head -n1) echo "Cell operators reach the Karmada hub via ${CP_IP}:{{.KARMADA_API_NODEPORT}}" TOKEN=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml -n {{.COMPUTE_NAMESPACE}} \ create token compute-manager --duration=720h)