feat: test that a network reaches the cell a workload runs in

Taskfile.yaml

@@ -410,29 +410,18 @@ tasks:
           get secret karmada-kubeconfig \
           -n karmada-system \
           -o jsonpath='{.data.kubeconfig}' \
-          | base64 -d > {{.KUBECONFIG_DIR}}/karmada-raw.yaml
-        # Rewrite the server address to the NodePort exposed on localhost
-        python3 - {{.KUBECONFIG_DIR}}/karmada-raw.yaml {{.KUBECONFIG_DIR}}/karmada.yaml 127.0.0.1 {{.KARMADA_API_NODEPORT}} << 'PYEOF'
-        import sys, yaml
-
-        src, dst, host, port = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
-
-        with open(src) as f:
-            cfg = yaml.safe_load(f)
-
-        for cluster in cfg.get('clusters', []):
-            old = cluster['cluster'].get('server', '')
-            cluster['cluster']['server'] = f'https://{host}:{port}'
-            # The cert is for the internal cluster IP, so skip TLS verification.
-            # This is a local dev-only environment.
-            cluster['cluster']['insecure-skip-tls-verify'] = True
-            cluster['cluster'].pop('certificate-authority-data', None)
-            print(f"  karmada server: {old} → https://{host}:{port}", file=sys.stderr)
-
-        with open(dst, 'w') as f:
-            yaml.dump(cfg, f, default_flow_style=False)
-        PYEOF
-        rm {{.KUBECONFIG_DIR}}/karmada-raw.yaml
+          | base64 -d > {{.KUBECONFIG_DIR}}/karmada.yaml
+        # Rewrite the server address to the NodePort exposed on localhost. The
+        # embedded cert is for the internal cluster IP, so verification has to be
+        # dropped as well; --insecure-skip-tls-verify also clears the embedded CA.
+        # This is a local dev-only environment.
+        for CLUSTER in $(kubectl config get-clusters --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml | tail -n +2); do
+          echo "  karmada server: ${CLUSTER} → https://127.0.0.1:{{.KARMADA_API_NODEPORT}}"
+          kubectl config set-cluster "${CLUSTER}" \
+            --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \
+            --server=https://127.0.0.1:{{.KARMADA_API_NODEPORT}} \
+            --insecure-skip-tls-verify=true
+        done
       # Copy karmada.yaml → downstream.yaml so Chainsaw tests that declare
       # cluster: downstream can load it (chainsaw-config.yaml references this path).
       - cp {{.KUBECONFIG_DIR}}/karmada.yaml {{.KUBECONFIG_DIR}}/downstream.yaml
@@ -517,13 +506,20 @@ tasks:
           {{.KUBECONFIG_DIR}}/karmada.yaml \
           {{.CLUSTER_NAME}} \
           {{.INTERNAL_KUBECONFIG}}
-      # ── Apply city-code label ──────────────────────────────────────────
+      # ── Apply cluster labels ───────────────────────────────────────────
+      # city-code is what compute's federator places deployments by.
+      #
+      # infra.datum.net/gateways=enabled is what NSO's propagation policy selects
+      # on, and every cell in infra carries it. Without it that policy matches no
+      # cluster and propagates nothing — silently, with no error and no event —
+      # so a NetworkContext would never reach a POP cell here.
       - |
         kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \
           label cluster {{.CLUSTER_NAME}} \
           topology.datum.net/city-code={{.CITY_CODE}} \
+          infra.datum.net/gateways=enabled \
           --overwrite
-        echo "Labeled cluster '{{.CLUSTER_NAME}}' with topology.datum.net/city-code={{.CITY_CODE}}"
+        echo "Labeled cluster '{{.CLUSTER_NAME}}' with topology.datum.net/city-code={{.CITY_CODE}} and infra.datum.net/gateways=enabled"
 
   # ════════════════════════════════════════════════════════════════════════
   #  CRD installation
@@ -535,6 +531,8 @@ tasks:
       - task: _e2e:crds:compute
       - task: _e2e:crds:nso
       - task: _e2e:crds:quota
+      # Depends on _e2e:crds:nso having registered the kinds it selects.
+      - task: _e2e:federation:nso-policy
 
   _e2e:crds:compute:
     internal: true
@@ -555,26 +553,55 @@ tasks:
 
   _e2e:crds:nso:
     internal: true
-    desc: "Apply NSO CRDs to control-plane and POP cell clusters"
+    desc: "Apply NSO CRDs to all clusters and the Karmada API server"
     cmds:
       # NSO CRDs (NetworkBinding, SubnetClaim, etc.) are installed on the
       # control-plane as well as POP cells. The control-plane operator needs them
       # so that Subnet/SubnetClaim informer watches can start without cache errors,
       # even though NSO controllers themselves only run on POP cells.
+      #
+      # The Karmada API server needs them too. A network's presence in a location
+      # is a NetworkContext that is written on the hub and propagated to the cells
+      # serving that location, so without these CRDs the hub cannot store one and
+      # nothing reaches a cell.
       - |
         go mod download go.datum.net/network-services-operator
-        NSO_VERSION=$(go list -m -json go.datum.net/network-services-operator \
-          | python3 -c "import sys, json; print(json.load(sys.stdin)['Version'])")
+        NSO_VERSION=$(go list -m -f '{{`{{.Version}}`}}' go.datum.net/network-services-operator)
         NSO_CRD_PATH="$(go env GOMODCACHE)/go.datum.net/network-services-operator@${NSO_VERSION}/config/crd"
         echo "NSO CRDs from: ${NSO_CRD_PATH}"
         for KC in \
           {{.KUBECONFIG_DIR}}/control-plane.yaml \
+          {{.KUBECONFIG_DIR}}/karmada.yaml \
           {{.KUBECONFIG_DIR}}/pop-dfw.yaml \
           {{.KUBECONFIG_DIR}}/pop-ord.yaml; do
           echo "Installing NSO CRDs → $(basename $KC .yaml)..."
           kubectl --kubeconfig="$KC" apply -k "${NSO_CRD_PATH}" --server-side --validate=false
         done
 
+  _e2e:federation:nso-policy:
+    internal: true
+    desc: "Apply NSO's federation propagation policy to the Karmada API server"
+    cmds:
+      # NSO ships the policy that carries its own resources to cells. Applying the
+      # real file, rather than a compute-authored stand-in, is what makes a local
+      # NetworkContext-reaches-a-cell test meaningful: it exercises the same
+      # selectors production uses.
+      #
+      # config/federation has no kustomization.yaml, and resourceinterpreters.yaml
+      # only aggregates Gateway API status, so apply just the policy.
+      #
+      # This runs after _e2e:crds:nso so the selected kinds are discoverable when
+      # Karmada's resource detector picks the policy up. Selectors for kinds this
+      # environment does not install (Gateway API, Envoy Gateway, cert-manager,
+      # external-dns) simply never match.
+      - |
+        NSO_VERSION=$(go list -m -f '{{`{{.Version}}`}}' go.datum.net/network-services-operator)
+        NSO_FEDERATION_PATH="$(go env GOMODCACHE)/go.datum.net/network-services-operator@${NSO_VERSION}/config/federation"
+        echo "NSO federation policy from: ${NSO_FEDERATION_PATH}"
+        kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply \
+          -f "${NSO_FEDERATION_PATH}/clusterpropagationpolicy.yaml" \
+          --server-side --validate=false
+
   _e2e:crds:quota:
     internal: true
     desc: "Apply Milo quota CRDs to all clusters and the Karmada API server"
@@ -584,8 +611,7 @@ tasks:
       # against project Milo control planes without cache startup errors.
       - |
         go mod download go.miloapis.com/milo
-        MILO_VERSION=$(go list -m -json go.miloapis.com/milo \
-          | python3 -c "import sys, json; print(json.load(sys.stdin)['Version'])")
+        MILO_VERSION=$(go list -m -f '{{`{{.Version}}`}}' go.miloapis.com/milo)
         QUOTA_CRD_PATH="$(go env GOMODCACHE)/go.miloapis.com/milo@${MILO_VERSION}/config/crd/bases/quota"
         echo "Milo quota CRDs from: ${QUOTA_CRD_PATH}"
         for KC in \
@@ -714,7 +740,7 @@ tasks:
       # NodePort rather than the in-cluster Service the management pod uses.
       - |
         CP_IP=$(docker inspect {{.KIND_CONTROL_PLANE}}-control-plane \
-          | python3 -c "import sys,json; print(list(json.load(sys.stdin)[0]['NetworkSettings']['Networks'].values())[0]['IPAddress'])")
+          -f '{{`{{range .NetworkSettings.Networks}}{{.IPAddress}}{{"\n"}}{{end}}`}}' | head -n1)
         echo "Cell operators reach the Karmada hub via ${CP_IP}:{{.KARMADA_API_NODEPORT}}"
         TOKEN=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml -n {{.COMPUTE_NAMESPACE}} \
           create token compute-manager --duration=720h)

config/base/downstream-rbac/rbac.yaml

@@ -16,6 +16,18 @@ rules:
   - apiGroups: ["compute.datumapis.com"]
     resources: ["workloaddeployments", "workloaddeployments/status", "instances", "instances/status"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  # Compute asks for a network's presence in a location by writing a
+  # NetworkBinding on the hub. NSO turns every binding for the same network and
+  # location into one shared NetworkContext and propagates it to the cells
+  # serving that location, where the claim path can use it. Compute owns the
+  # bindings it creates, including releasing them, but only reads the contexts
+  # NSO derives from them.
+  - apiGroups: ["networking.datumapis.com"]
+    resources: ["networkbindings", "networkbindings/status"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["networking.datumapis.com"]
+    resources: ["networkcontexts"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: ["policy.karmada.io"]
     resources: ["propagationpolicies", "clusterpropagationpolicies"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]