diff --git a/Taskfile.yaml b/Taskfile.yaml index 2ae2eab1..f2af01e0 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -410,29 +410,18 @@ tasks: get secret karmada-kubeconfig \ -n karmada-system \ -o jsonpath='{.data.kubeconfig}' \ - | base64 -d > {{.KUBECONFIG_DIR}}/karmada-raw.yaml - # Rewrite the server address to the NodePort exposed on localhost - python3 - {{.KUBECONFIG_DIR}}/karmada-raw.yaml {{.KUBECONFIG_DIR}}/karmada.yaml 127.0.0.1 {{.KARMADA_API_NODEPORT}} << 'PYEOF' - import sys, yaml - - src, dst, host, port = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] - - with open(src) as f: - cfg = yaml.safe_load(f) - - for cluster in cfg.get('clusters', []): - old = cluster['cluster'].get('server', '') - cluster['cluster']['server'] = f'https://{host}:{port}' - # The cert is for the internal cluster IP, so skip TLS verification. - # This is a local dev-only environment. - cluster['cluster']['insecure-skip-tls-verify'] = True - cluster['cluster'].pop('certificate-authority-data', None) - print(f" karmada server: {old} → https://{host}:{port}", file=sys.stderr) - - with open(dst, 'w') as f: - yaml.dump(cfg, f, default_flow_style=False) - PYEOF - rm {{.KUBECONFIG_DIR}}/karmada-raw.yaml + | base64 -d > {{.KUBECONFIG_DIR}}/karmada.yaml + # Rewrite the server address to the NodePort exposed on localhost. The + # embedded cert is for the internal cluster IP, so verification has to be + # dropped as well; --insecure-skip-tls-verify also clears the embedded CA. + # This is a local dev-only environment. + for CLUSTER in $(kubectl config get-clusters --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml | tail -n +2); do + echo " karmada server: ${CLUSTER} → https://127.0.0.1:{{.KARMADA_API_NODEPORT}}" + kubectl config set-cluster "${CLUSTER}" \ + --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \ + --server=https://127.0.0.1:{{.KARMADA_API_NODEPORT}} \ + --insecure-skip-tls-verify=true + done # Copy karmada.yaml → downstream.yaml so Chainsaw tests that declare # cluster: downstream can load it (chainsaw-config.yaml references this path). - cp {{.KUBECONFIG_DIR}}/karmada.yaml {{.KUBECONFIG_DIR}}/downstream.yaml @@ -517,13 +506,20 @@ tasks: {{.KUBECONFIG_DIR}}/karmada.yaml \ {{.CLUSTER_NAME}} \ {{.INTERNAL_KUBECONFIG}} - # ── Apply city-code label ────────────────────────────────────────── + # ── Apply cluster labels ─────────────────────────────────────────── + # city-code is what compute's federator places deployments by. + # + # infra.datum.net/gateways=enabled is what NSO's propagation policy selects + # on, and every cell in infra carries it. Without it that policy matches no + # cluster and propagates nothing — silently, with no error and no event — + # so a NetworkContext would never reach a POP cell here. - | kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml \ label cluster {{.CLUSTER_NAME}} \ topology.datum.net/city-code={{.CITY_CODE}} \ + infra.datum.net/gateways=enabled \ --overwrite - echo "Labeled cluster '{{.CLUSTER_NAME}}' with topology.datum.net/city-code={{.CITY_CODE}}" + echo "Labeled cluster '{{.CLUSTER_NAME}}' with topology.datum.net/city-code={{.CITY_CODE}} and infra.datum.net/gateways=enabled" # ════════════════════════════════════════════════════════════════════════ # CRD installation @@ -535,6 +531,8 @@ tasks: - task: _e2e:crds:compute - task: _e2e:crds:nso - task: _e2e:crds:quota + # Depends on _e2e:crds:nso having registered the kinds it selects. + - task: _e2e:federation:nso-policy _e2e:crds:compute: internal: true @@ -555,26 +553,55 @@ tasks: _e2e:crds:nso: internal: true - desc: "Apply NSO CRDs to control-plane and POP cell clusters" + desc: "Apply NSO CRDs to all clusters and the Karmada API server" cmds: # NSO CRDs (NetworkBinding, SubnetClaim, etc.) are installed on the # control-plane as well as POP cells. The control-plane operator needs them # so that Subnet/SubnetClaim informer watches can start without cache errors, # even though NSO controllers themselves only run on POP cells. + # + # The Karmada API server needs them too. A network's presence in a location + # is a NetworkContext that is written on the hub and propagated to the cells + # serving that location, so without these CRDs the hub cannot store one and + # nothing reaches a cell. - | go mod download go.datum.net/network-services-operator - NSO_VERSION=$(go list -m -json go.datum.net/network-services-operator \ - | python3 -c "import sys, json; print(json.load(sys.stdin)['Version'])") + NSO_VERSION=$(go list -m -f '{{`{{.Version}}`}}' go.datum.net/network-services-operator) NSO_CRD_PATH="$(go env GOMODCACHE)/go.datum.net/network-services-operator@${NSO_VERSION}/config/crd" echo "NSO CRDs from: ${NSO_CRD_PATH}" for KC in \ {{.KUBECONFIG_DIR}}/control-plane.yaml \ + {{.KUBECONFIG_DIR}}/karmada.yaml \ {{.KUBECONFIG_DIR}}/pop-dfw.yaml \ {{.KUBECONFIG_DIR}}/pop-ord.yaml; do echo "Installing NSO CRDs → $(basename $KC .yaml)..." kubectl --kubeconfig="$KC" apply -k "${NSO_CRD_PATH}" --server-side --validate=false done + _e2e:federation:nso-policy: + internal: true + desc: "Apply NSO's federation propagation policy to the Karmada API server" + cmds: + # NSO ships the policy that carries its own resources to cells. Applying the + # real file, rather than a compute-authored stand-in, is what makes a local + # NetworkContext-reaches-a-cell test meaningful: it exercises the same + # selectors production uses. + # + # config/federation has no kustomization.yaml, and resourceinterpreters.yaml + # only aggregates Gateway API status, so apply just the policy. + # + # This runs after _e2e:crds:nso so the selected kinds are discoverable when + # Karmada's resource detector picks the policy up. Selectors for kinds this + # environment does not install (Gateway API, Envoy Gateway, cert-manager, + # external-dns) simply never match. + - | + NSO_VERSION=$(go list -m -f '{{`{{.Version}}`}}' go.datum.net/network-services-operator) + NSO_FEDERATION_PATH="$(go env GOMODCACHE)/go.datum.net/network-services-operator@${NSO_VERSION}/config/federation" + echo "NSO federation policy from: ${NSO_FEDERATION_PATH}" + kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml apply \ + -f "${NSO_FEDERATION_PATH}/clusterpropagationpolicy.yaml" \ + --server-side --validate=false + _e2e:crds:quota: internal: true desc: "Apply Milo quota CRDs to all clusters and the Karmada API server" @@ -584,8 +611,7 @@ tasks: # against project Milo control planes without cache startup errors. - | go mod download go.miloapis.com/milo - MILO_VERSION=$(go list -m -json go.miloapis.com/milo \ - | python3 -c "import sys, json; print(json.load(sys.stdin)['Version'])") + MILO_VERSION=$(go list -m -f '{{`{{.Version}}`}}' go.miloapis.com/milo) QUOTA_CRD_PATH="$(go env GOMODCACHE)/go.miloapis.com/milo@${MILO_VERSION}/config/crd/bases/quota" echo "Milo quota CRDs from: ${QUOTA_CRD_PATH}" for KC in \ @@ -714,7 +740,7 @@ tasks: # NodePort rather than the in-cluster Service the management pod uses. - | CP_IP=$(docker inspect {{.KIND_CONTROL_PLANE}}-control-plane \ - | python3 -c "import sys,json; print(list(json.load(sys.stdin)[0]['NetworkSettings']['Networks'].values())[0]['IPAddress'])") + -f '{{`{{range .NetworkSettings.Networks}}{{.IPAddress}}{{"\n"}}{{end}}`}}' | head -n1) echo "Cell operators reach the Karmada hub via ${CP_IP}:{{.KARMADA_API_NODEPORT}}" TOKEN=$(kubectl --kubeconfig={{.KUBECONFIG_DIR}}/karmada.yaml -n {{.COMPUTE_NAMESPACE}} \ create token compute-manager --duration=720h) diff --git a/config/base/downstream-rbac/rbac.yaml b/config/base/downstream-rbac/rbac.yaml index f2a94a37..7ad15ef5 100644 --- a/config/base/downstream-rbac/rbac.yaml +++ b/config/base/downstream-rbac/rbac.yaml @@ -16,6 +16,18 @@ rules: - apiGroups: ["compute.datumapis.com"] resources: ["workloaddeployments", "workloaddeployments/status", "instances", "instances/status"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + # Compute asks for a network's presence in a location by writing a + # NetworkBinding on the hub. NSO turns every binding for the same network and + # location into one shared NetworkContext and propagates it to the cells + # serving that location, where the claim path can use it. Compute owns the + # bindings it creates, including releasing them, but only reads the contexts + # NSO derives from them. + - apiGroups: ["networking.datumapis.com"] + resources: ["networkbindings", "networkbindings/status"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["networking.datumapis.com"] + resources: ["networkcontexts"] + verbs: ["get", "list", "watch"] - apiGroups: ["policy.karmada.io"] resources: ["propagationpolicies", "clusterpropagationpolicies"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]