From 247fdd64161b2dc5ca3a557fc735b7fa1a037efb Mon Sep 17 00:00:00 2001 From: Ondra Kupka Date: Thu, 30 Jul 2026 15:07:46 +0200 Subject: [PATCH 1/9] Add component-scoped proxy e2e tests Add end-to-end tests that validate the Authentication operator's component-scoped proxy support. Three test cases cover: - OIDC IdP discovery through an HTTP forward proxy - OIDC IdP discovery through an HTTPS forward proxy with trustedCA - Fallback behavior when spec.proxy is removed (deletes the proxy namespace to prove the operator no longer depends on it) The tests deploy a Squid forward proxy and Keycloak in ephemeral namespaces, configure the Authentication operator to use the proxy, register Keycloak as an OIDC IdP, and verify: - The operator discovers the IdP and stabilizes - The OAuth server deployment has the correct proxy env vars - Proxy traffic from the operator appears in structured Squid access logs (custom logformat for precise source IP matching) - The trustedCA ConfigMap is synced (HTTPS proxy case) Test infrastructure includes helpers for deploying Squid with self-signed TLS (using library-go/pkg/crypto), deploying and configuring Keycloak, saving/restoring Authentication and OAuth state, and structured proxy log verification. Extends the keycloak client with methods for client secret regeneration, access token timeout configuration, and raw client updates needed by the proxy test setup. --- .../authentication/component_proxy.go | 182 +++++ .../authentication/component_proxy_helpers.go | 687 ++++++++++++++++++ .../authentication/keycloak_client.go | 97 ++- .../authentication/operator_status_helpers.go | 20 + 4 files changed, 984 insertions(+), 2 deletions(-) create mode 100644 test/extended/authentication/component_proxy.go create mode 100644 test/extended/authentication/component_proxy_helpers.go create mode 100644 test/extended/authentication/operator_status_helpers.go diff --git a/test/extended/authentication/component_proxy.go b/test/extended/authentication/component_proxy.go new file mode 100644 index 000000000000..de2b37eb277c --- /dev/null +++ b/test/extended/authentication/component_proxy.go @@ -0,0 +1,182 @@ +package authentication + +import ( + "context" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + operatorv1 "github.com/openshift/api/operator/v1" + + exutil "github.com/openshift/origin/test/extended/util" + operator "github.com/openshift/origin/test/extended/util/operator" +) + +var _ = g.Describe("[sig-auth][Suite:openshift/conformance/serial][OCPFeatureGate:AuthenticationComponentProxy][Serial]", func() { + oc := exutil.NewCLIWithoutNamespace("component-proxy") + + var ( + ctx context.Context + httpProxyURL string + httpsProxyURL string + caCertPEM []byte + proxyNamespace string + kcSetup *keycloakProxySetup + cleanups []removalFunc + ) + + g.BeforeEach(func() { + ctx = context.Background() + cleanups = nil + + g.By("Saving auth state for restore after test") + authRestore, err := saveAndRestoreAuthState(ctx, oc) + cleanups = append(cleanups, authRestore) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Deploying Squid forward proxy") + var proxyCleanup removalFunc + httpProxyURL, httpsProxyURL, caCertPEM, proxyNamespace, proxyCleanup, err = deploySquidProxy(ctx, oc) + cleanups = append(cleanups, proxyCleanup) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Deploying Keycloak (without registering IdP yet)") + var kcCleanups []removalFunc + kcSetup, kcCleanups, err = deployKeycloakForProxy(ctx, oc) + cleanups = append(cleanups, kcCleanups...) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for operators to be stable before test") + err = operator.WaitForOperatorsToSettle(ctx, oc.AdminConfigClient(), 10) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.GinkgoWriter.Printf("Squid proxy URL: http=%s https=%s\n", httpProxyURL, httpsProxyURL) + g.GinkgoWriter.Printf("Keycloak issuer URL: %s\n", kcSetup.issuerURL) + g.GinkgoWriter.Printf("Keycloak namespace: %s\n", kcSetup.namespace) + }) + + g.AfterEach(func() { + _ = removeResources(ctx, cleanups...) + + g.By("Waiting for operators to be stable after test") + err := operator.WaitForOperatorsToSettle(ctx, oc.AdminConfigClient(), 10) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + g.It("operator should validate OIDC IdP through component proxy", func() { + testOIDCIdPThroughComponentProxy(ctx, oc, kcSetup, httpProxyURL, nil, proxyNamespace) + }) + g.It("operator should validate OIDC IdP through component proxy with trustedCA", func() { + testOIDCIdPThroughComponentProxy(ctx, oc, kcSetup, httpsProxyURL, caCertPEM, proxyNamespace) + }) + g.It("operator should fall back to original configuration on spec.proxy removal", func() { + testFallbackOnProxyRemoval(ctx, oc, kcSetup, httpProxyURL, proxyNamespace) + }) +}) + +func testOIDCIdPThroughComponentProxy(ctx context.Context, oc *exutil.CLI, kcSetup *keycloakProxySetup, proxyURL string, trustedCACertPEM []byte, proxyNamespace string) { + withTrustedCA := len(trustedCACertPEM) > 0 + + const trustedCAConfigMapName = "e2e-proxy-ca" + if withTrustedCA { + g.By("Creating trustedCA ConfigMap in openshift-config") + _, err := oc.AdminKubeClient().CoreV1().ConfigMaps("openshift-config").Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: trustedCAConfigMapName, + Labels: componentProxyTestLabels(), + }, + Data: map[string]string{ + "ca-bundle.crt": string(trustedCACertPEM), + }, + }, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + g.DeferCleanup(func(ctx context.Context) error { + return oc.AdminKubeClient().CoreV1().ConfigMaps("openshift-config").Delete(ctx, trustedCAConfigMapName, metav1.DeleteOptions{}) + }) + } + + proxyTrafficStart := time.Now() + + g.By("Setting component-scoped proxy") + proxyConfig := operatorv1.AuthenticationProxyConfig{ + HTTPSProxy: proxyURL, + } + if withTrustedCA { + proxyConfig.TrustedCA = operatorv1.AuthenticationConfigMapReference{Name: trustedCAConfigMapName} + } + err := updateAuthenticationProxy(ctx, oc, proxyConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + if withTrustedCA { + g.By("Waiting for trustedCA ConfigMap to be synced before registering IdP") + err = verifyTrustedCAConfigMapSynced(ctx, oc) + o.Expect(err).NotTo(o.HaveOccurred()) + } + + g.By("Registering Keycloak as OIDC IdP (operator discovers it through the proxy)") + idpCleanups, err := addKeycloakOIDCIdPForProxy(ctx, oc, kcSetup) + g.DeferCleanup(func() { + _ = removeResources(ctx, idpCleanups...) + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for operator to pick up IdP changes and stabilize") + err = waitForOperatorToPickUpChanges(ctx, oc, "authentication") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verifying OAuth server deployment has proxy env vars and trustedCA volume/mount") + err = verifyOAuthServerDeploymentProxyConfig(ctx, oc, "", proxyURL, ".cluster.local,.svc,127.0.0.1,localhost", withTrustedCA) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Looking up operator pod IP") + operatorPods, err := oc.AdminKubeClient().CoreV1().Pods("openshift-authentication-operator").List(ctx, metav1.ListOptions{ + LabelSelector: "app=authentication-operator", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(operatorPods.Items).NotTo(o.BeEmpty()) + operatorIP := operatorPods.Items[0].Status.PodIP + o.Expect(operatorIP).NotTo(o.BeEmpty()) + + g.By("Verifying operator traffic went through the Squid proxy") + err = waitForProxyTrafficFrom(ctx, oc, proxyNamespace, operatorIP, proxyTrafficStart, 5*time.Minute) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func testFallbackOnProxyRemoval(ctx context.Context, oc *exutil.CLI, kcSetup *keycloakProxySetup, httpProxyURL string, proxyNamespace string) { + g.By("Setting component-scoped proxy") + err := updateAuthenticationProxy(ctx, oc, operatorv1.AuthenticationProxyConfig{ + HTTPSProxy: httpProxyURL, + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Registering Keycloak as OIDC IdP") + idpCleanups, err := addKeycloakOIDCIdPForProxy(ctx, oc, kcSetup) + g.DeferCleanup(func() { + _ = removeResources(ctx, idpCleanups...) + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for operator to pick up IdP changes and stabilize") + err = waitForOperatorToPickUpChanges(ctx, oc, "authentication") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Removing spec.proxy from Authentication CR") + err = updateAuthenticationProxy(ctx, oc, operatorv1.AuthenticationProxyConfig{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Deleting Squid to prove the operator no longer routes through it") + err = oc.AdminKubeClient().CoreV1().Namespaces().Delete(ctx, proxyNamespace, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for operator to pick up proxy removal and stabilize") + err = waitForOperatorToPickUpChanges(ctx, oc, "authentication") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verifying proxy env vars are no longer set on OAuth server deployment") + err = verifyOAuthServerDeploymentProxyConfig(ctx, oc, "", "", "", false) + o.Expect(err).NotTo(o.HaveOccurred()) +} diff --git a/test/extended/authentication/component_proxy_helpers.go b/test/extended/authentication/component_proxy_helpers.go new file mode 100644 index 000000000000..c7e047b4a0cc --- /dev/null +++ b/test/extended/authentication/component_proxy_helpers.go @@ -0,0 +1,687 @@ +package authentication + +import ( + "context" + "fmt" + "net" + "reflect" + "strconv" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + configv1 "github.com/openshift/api/config/v1" + operatorv1 "github.com/openshift/api/operator/v1" + libcrypto "github.com/openshift/library-go/pkg/crypto" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/tools/cache" + watchtools "k8s.io/client-go/tools/watch" + "k8s.io/client-go/util/retry" + + exutil "github.com/openshift/origin/test/extended/util" + "github.com/openshift/origin/test/extended/util/image" +) + +const ( + squidImage = "registry.redhat.io/rhel10/squid:10.2-1784702318" + squidHTTPPort = int32(3128) + squidHTTPSPort = int32(3129) + squidServiceName = "squid-proxy" + + componentProxyCAConfigMapName = "v4-0-config-system-auth-proxy-ca" +) + +func componentProxyTestLabels() map[string]string { + return map[string]string{ + "e2e-test": "openshift-authentication-operator", + } +} + +// saveAndRestoreAuthState snapshots the Authentication operator CR and +// oauth/cluster, returning a cleanup function that restores both. +// If either resource was modified, it waits for the operator to stabilize. +func saveAndRestoreAuthState(ctx context.Context, oc *exutil.CLI) (removalFunc, error) { + operatorClient := oc.AdminOperatorClient() + oauthClient := oc.AdminConfigClient().ConfigV1().OAuths() + + auth, err := operatorClient.OperatorV1().Authentications().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("getting authentication/cluster: %w", err) + } + originalAuthSpec := auth.Spec.DeepCopy() + + oauth, err := oauthClient.Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("getting oauth/cluster: %w", err) + } + originalOAuthSpec := oauth.Spec.DeepCopy() + + return func(ctx context.Context) error { + var changed bool + + g.GinkgoWriter.Println("cleanup: restoring authentication/cluster") + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + fresh, err := operatorClient.OperatorV1().Authentications().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return err + } + if reflect.DeepEqual(fresh.Spec, *originalAuthSpec) { + return nil + } + changed = true + fresh.Spec = *originalAuthSpec + _, err = operatorClient.OperatorV1().Authentications().Update(ctx, fresh, metav1.UpdateOptions{}) + return err + }); err != nil { + g.GinkgoWriter.Printf("cleanup: failed to restore Authentication CR: %v\n", err) + } + + g.GinkgoWriter.Println("cleanup: restoring oauth/cluster") + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + fresh, err := oauthClient.Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return err + } + if reflect.DeepEqual(fresh.Spec, *originalOAuthSpec) { + return nil + } + changed = true + fresh.Spec = *originalOAuthSpec + _, err = oauthClient.Update(ctx, fresh, metav1.UpdateOptions{}) + return err + }); err != nil { + g.GinkgoWriter.Printf("cleanup: failed to restore oauth/cluster: %v\n", err) + } + + if changed { + g.GinkgoWriter.Println("cleanup: waiting for operator to stabilize") + if err := waitForOperatorToPickUpChanges(ctx, oc, "authentication"); err != nil { + g.GinkgoWriter.Printf("cleanup: operator did not recover: %v\n", err) + } + } + return nil + }, nil +} + +// deploySquidProxy deploys a Squid forward proxy listening on HTTP (3128) and +// HTTPS (3129) with a self-signed CA and serving certificate. +func deploySquidProxy(ctx context.Context, oc *exutil.CLI) (httpProxyURL, httpsProxyURL string, caCertPEM []byte, namespace string, cleanup removalFunc, err error) { + kubeClient := oc.AdminKubeClient() + + nsLabels := componentProxyTestLabels() + nsLabels["pod-security.kubernetes.io/enforce"] = "baseline" + nsLabels["security.openshift.io/scc.podSecurityLabelSync"] = "false" + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "e2e-proxy-", + Labels: nsLabels, + }, + } + created, err := kubeClient.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + if err != nil { + return "", "", nil, "", nil, fmt.Errorf("creating Squid proxy namespace: %w", err) + } + namespace = created.Name + cleanup = func(ctx context.Context) error { + g.GinkgoWriter.Println("cleanup: removing Squid proxy namespace") + err := kubeClient.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + return err + } + + caConfig, err := libcrypto.MakeSelfSignedCAConfigForDuration("squid-proxy-ca", 2*time.Hour) + if err != nil { + return "", "", nil, "", cleanup, fmt.Errorf("creating proxy CA: %w", err) + } + ca := &libcrypto.CA{Config: caConfig, SerialGenerator: &libcrypto.RandomSerialGenerator{}} + + serviceDNS := fmt.Sprintf("%s.%s.svc.cluster.local", squidServiceName, namespace) + serverCertConfig, err := ca.MakeServerCert(sets.New(serviceDNS), 2*time.Hour) + if err != nil { + return "", "", nil, "", cleanup, fmt.Errorf("creating proxy server cert: %w", err) + } + + caCertPEM, _, err = caConfig.GetPEMBytes() + if err != nil { + return "", "", nil, "", cleanup, fmt.Errorf("encoding proxy CA cert: %w", err) + } + serverCertPEM, serverKeyPEM, err := serverCertConfig.GetPEMBytes() + if err != nil { + return "", "", nil, "", cleanup, fmt.Errorf("encoding proxy server cert: %w", err) + } + + squidConfig := fmt.Sprintf(`http_port %d +https_port %d tls-cert=/etc/squid/tls/tls.crt tls-key=/etc/squid/tls/tls.key +pid_filename /tmp/squid.pid +acl all src all +http_access allow all +logformat proxytest %%rm %%ru %%>a %%Ss %%>Hs +access_log stdio:/dev/stdout proxytest +cache_log stdio:/dev/stderr +cache deny all +buffered_logs off +`, squidHTTPPort, squidHTTPSPort) + + _, err = kubeClient.CoreV1().ConfigMaps(namespace).Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "squid-config"}, + Data: map[string]string{"squid.conf": squidConfig}, + }, metav1.CreateOptions{}) + if err != nil { + return "", "", nil, "", cleanup, fmt.Errorf("creating Squid config: %w", err) + } + + _, err = kubeClient.CoreV1().Secrets(namespace).Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "squid-tls"}, + Data: map[string][]byte{ + "tls.crt": serverCertPEM, + "tls.key": serverKeyPEM, + }, + }, metav1.CreateOptions{}) + if err != nil { + return "", "", nil, "", cleanup, fmt.Errorf("creating Squid TLS secret: %w", err) + } + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: squidServiceName, + Labels: map[string]string{"app": squidServiceName}, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: new(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": squidServiceName}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": squidServiceName}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "squid", + Image: image.LocationFor(squidImage), + Ports: []corev1.ContainerPort{ + {ContainerPort: squidHTTPPort, Protocol: corev1.ProtocolTCP}, + {ContainerPort: squidHTTPSPort, Protocol: corev1.ProtocolTCP}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "squid-config", MountPath: "/etc/squid/squid.conf", SubPath: "squid.conf"}, + {Name: "squid-tls", MountPath: "/etc/squid/tls", ReadOnly: true}, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt32(squidHTTPPort), + }, + }, + InitialDelaySeconds: 5, + PeriodSeconds: 5, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "squid-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "squid-config"}, + }, + }, + }, + { + Name: "squid-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: "squid-tls"}, + }, + }, + }, + }, + }, + }, + } + + _, err = kubeClient.AppsV1().Deployments(namespace).Create(ctx, deployment, metav1.CreateOptions{}) + if err != nil { + return "", "", nil, "", cleanup, fmt.Errorf("creating Squid deployment: %w", err) + } + + _, err = kubeClient.CoreV1().Services(namespace).Create(ctx, &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: squidServiceName, + Labels: map[string]string{"app": squidServiceName}, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"app": squidServiceName}, + Ports: []corev1.ServicePort{ + {Name: "http", Port: squidHTTPPort, TargetPort: intstr.FromInt32(squidHTTPPort), Protocol: corev1.ProtocolTCP}, + {Name: "https", Port: squidHTTPSPort, TargetPort: intstr.FromInt32(squidHTTPSPort), Protocol: corev1.ProtocolTCP}, + }, + }, + }, metav1.CreateOptions{}) + if err != nil { + return "", "", nil, "", cleanup, fmt.Errorf("creating Squid service: %w", err) + } + + g.GinkgoWriter.Printf("waiting for Squid proxy deployment in %s to be ready\n", namespace) + timeLimitedCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + _, err = watchtools.UntilWithSync(timeLimitedCtx, + cache.NewListWatchFromClient( + kubeClient.AppsV1().RESTClient(), "deployments", namespace, + fields.OneTermEqualSelector("metadata.name", squidServiceName)), + &appsv1.Deployment{}, + nil, + func(event watch.Event) (bool, error) { + if event.Type == watch.Error { + return false, fmt.Errorf("Squid deployment watch error: %v", event.Object) + } + if event.Type == watch.Bookmark { + return false, nil + } + d, ok := event.Object.(*appsv1.Deployment) + if !ok { + return false, nil + } + return d.Status.ReadyReplicas > 0, nil + }, + ) + if err != nil { + return "", "", nil, "", cleanup, fmt.Errorf("Squid proxy deployment did not become ready: %w", err) + } + + serviceHost := fmt.Sprintf("%s.%s.svc.cluster.local", squidServiceName, namespace) + httpProxyURL = "http://" + net.JoinHostPort(serviceHost, strconv.Itoa(int(squidHTTPPort))) + httpsProxyURL = "https://" + net.JoinHostPort(serviceHost, strconv.Itoa(int(squidHTTPSPort))) + g.GinkgoWriter.Printf("Squid proxy deployed: http=%s https=%s\n", httpProxyURL, httpsProxyURL) + return httpProxyURL, httpsProxyURL, caCertPEM, namespace, cleanup, nil +} + +func getSquidProxyLogs(ctx context.Context, oc *exutil.CLI, namespace string) (string, error) { + return getSquidProxyLogsSince(ctx, oc, namespace, time.Time{}) +} + +func getSquidProxyLogsSince(ctx context.Context, oc *exutil.CLI, namespace string, since time.Time) (string, error) { + kubeClient := oc.AdminKubeClient() + + pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", squidServiceName), + }) + if err != nil { + return "", fmt.Errorf("listing squid pods in %s: %w", namespace, err) + } + if len(pods.Items) == 0 { + return "", fmt.Errorf("no squid proxy pods found in namespace %s", namespace) + } + + logOpts := &corev1.PodLogOptions{Container: "squid"} + if !since.IsZero() { + t := metav1.NewTime(since) + logOpts.SinceTime = &t + } + logBytes, err := kubeClient.CoreV1().Pods(namespace).GetLogs(pods.Items[0].Name, logOpts).DoRaw(ctx) + if err != nil { + return "", fmt.Errorf("getting logs from squid container: %w", err) + } + + return string(logBytes), nil +} + +// waitForProxyTrafficFrom polls the squid access log until a CONNECT entry from +// the given source IP appears, confirming that the source is routing traffic +// through the proxy. The log format is set by the proxytest logformat directive +// in the squid config: "method url sourceIP status httpCode". +func waitForProxyTrafficFrom(ctx context.Context, oc *exutil.CLI, proxyNamespace, sourceIP string, since time.Time, timeout time.Duration) error { + g.GinkgoWriter.Printf("waiting up to %s for proxy traffic from %s\n", timeout, sourceIP) + return wait.PollUntilContextTimeout(ctx, 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + logs, err := getSquidProxyLogsSince(ctx, oc, proxyNamespace, since) + if err != nil { + g.GinkgoWriter.Printf("failed to read squid logs: %v\n", err) + return false, nil + } + for line := range strings.SplitSeq(logs, "\n") { + parts := strings.Fields(line) + if len(parts) >= 4 && parts[0] == "CONNECT" && parts[2] == sourceIP && parts[3] == "TCP_TUNNEL" { + g.GinkgoWriter.Printf("confirmed proxy traffic from %s: %s\n", sourceIP, line) + return true, nil + } + } + return false, nil + }) +} + +// keycloakProxySetup holds the results of deploying Keycloak for proxy tests, +// before the IdP is registered in OpenShift. +type keycloakProxySetup struct { + client *keycloakClient + idpName string + namespace string + clientID string + clientSecret string + issuerURL string +} + +func deployKeycloakForProxy(ctx context.Context, oc *exutil.CLI) (*keycloakProxySetup, []removalFunc, error) { + namespace := fmt.Sprintf("e2e-proxy-kc-%s", rand.String(8)) + cleanups, err := deployKeycloak(ctx, oc, namespace, g.GinkgoLogr) + if err != nil { + return nil, cleanups, fmt.Errorf("deploying keycloak: %w", err) + } + + setup := &keycloakProxySetup{ + idpName: fmt.Sprintf("keycloak-proxy-test-%s", namespace), + namespace: namespace, + } + + // Use the route for admin API calls (the test runner may be external). + routeURL, err := admittedURLForRoute(ctx, oc, keycloakResourceName, namespace) + if err != nil { + return nil, cleanups, fmt.Errorf("getting keycloak route URL: %w", err) + } + + kcClient, err := keycloakClientFor(routeURL) + if err != nil { + return nil, cleanups, fmt.Errorf("creating keycloak client: %w", err) + } + setup.client = kcClient + setup.issuerURL = routeURL + "/realms/master" + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + err := kcClient.Authenticate("admin-cli", keycloakAdminUsername, keycloakAdminPassword) + if err != nil { + g.GinkgoWriter.Printf("failed to authenticate to Keycloak: %v\n", err) + return false, nil + } + return true, nil + }) + if err != nil { + return nil, cleanups, fmt.Errorf("authenticating to keycloak: %w", err) + } + + clientList, err := kcClient.ListClients() + if err != nil { + return nil, cleanups, fmt.Errorf("listing keycloak clients: %w", err) + } + + var adminClientID, passwdClientID string + for _, c := range clientList { + if c.ClientID == "admin-cli" { + adminClientID = c.ID + } else if len(c.RedirectURIs) > 0 { + passwdClientID = c.ID + setup.clientID = c.ClientID + } + if len(passwdClientID) > 0 && len(adminClientID) > 0 { + break + } + } + + if adminClientID == "" { + return nil, cleanups, fmt.Errorf("admin-cli client not found in keycloak") + } + if passwdClientID == "" { + return nil, cleanups, fmt.Errorf("password-grant client (with redirectUris) not found in keycloak") + } + + // Extend admin-cli token lifetime to 30 minutes so the token doesn't + // expire during subsequent Keycloak API calls. + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + err := kcClient.UpdateClientAccessTokenTimeout(adminClientID, 60*30) + if err != nil { + g.GinkgoWriter.Printf("failed to update client access token timeout: %v, retrying\n", err) + if authErr := kcClient.Authenticate("admin-cli", keycloakAdminUsername, keycloakAdminPassword); authErr != nil { + g.GinkgoWriter.Printf("failed to re-authenticate: %v\n", authErr) + } + return false, nil + } + return true, nil + }) + if err != nil { + return nil, cleanups, fmt.Errorf("updating admin-cli access token timeout: %w", err) + } + + err = kcClient.Authenticate("admin-cli", keycloakAdminUsername, keycloakAdminPassword) + if err != nil { + return nil, cleanups, fmt.Errorf("re-authenticating to keycloak: %w", err) + } + + // Regenerate the client secret so we have a known value to pass to the + // OAuth IdP configuration — the initial secret is auto-generated by Keycloak. + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + var err error + setup.clientSecret, err = kcClient.RegenerateClientSecret(passwdClientID) + if err != nil { + g.GinkgoWriter.Printf("failed to regenerate client secret: %v, retrying\n", err) + if authErr := kcClient.Authenticate("admin-cli", keycloakAdminUsername, keycloakAdminPassword); authErr != nil { + g.GinkgoWriter.Printf("failed to re-authenticate: %v\n", authErr) + } + return false, nil + } + return true, nil + }) + if err != nil { + return nil, cleanups, fmt.Errorf("regenerating client secret: %w", err) + } + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + err := kcClient.CreateClientGroupMapper(passwdClientID, "test-groups-mapper", "groups") + if err != nil { + g.GinkgoWriter.Printf("failed to create client group mapper: %v, retrying\n", err) + if authErr := kcClient.Authenticate("admin-cli", keycloakAdminUsername, keycloakAdminPassword); authErr != nil { + g.GinkgoWriter.Printf("failed to re-authenticate: %v\n", authErr) + } + return false, nil + } + return true, nil + }) + if err != nil { + return nil, cleanups, fmt.Errorf("creating client group mapper: %w", err) + } + + return setup, cleanups, nil +} + +func addKeycloakOIDCIdPForProxy(ctx context.Context, oc *exutil.CLI, setup *keycloakProxySetup) ([]removalFunc, error) { + var cleanups []removalFunc + kubeClient := oc.AdminKubeClient() + + secretName := setup.idpName + "-secret" + _, err := kubeClient.CoreV1().Secrets("openshift-config").Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Labels: componentProxyTestLabels(), + }, + Data: map[string][]byte{ + "clientSecret": []byte(setup.clientSecret), + }, + }, metav1.CreateOptions{}) + if err != nil { + return cleanups, fmt.Errorf("creating keycloak client secret: %w", err) + } + cleanups = append(cleanups, func(ctx context.Context) error { + return kubeClient.CoreV1().Secrets("openshift-config").Delete(ctx, secretName, metav1.DeleteOptions{}) + }) + + caCMName := setup.idpName + "-ca" + caCleanup, err := syncDefaultIngressCAToConfig(ctx, oc, caCMName) + if err != nil { + return cleanups, fmt.Errorf("syncing default ingress CA: %w", err) + } + cleanups = append(cleanups, caCleanup) + + err = addIdentityProvider(ctx, oc, configv1.IdentityProvider{ + Name: setup.idpName, + MappingMethod: configv1.MappingMethodClaim, + IdentityProviderConfig: configv1.IdentityProviderConfig{ + Type: configv1.IdentityProviderTypeOpenID, + OpenID: &configv1.OpenIDIdentityProvider{ + ClientID: setup.clientID, + ClientSecret: configv1.SecretNameReference{ + Name: secretName, + }, + ExtraScopes: []string{"profile", "email"}, + Claims: configv1.OpenIDClaims{ + PreferredUsername: []string{"preferred_username"}, + Groups: []configv1.OpenIDClaim{"groups"}, + }, + Issuer: setup.issuerURL, + CA: configv1.ConfigMapNameReference{ + Name: caCMName, + }, + }, + }, + }) + if err != nil { + return cleanups, fmt.Errorf("adding identity provider: %w", err) + } + + return cleanups, nil +} + +// syncDefaultIngressCAToConfig copies the default ingress CA into a new +// ConfigMap in openshift-config so it can be referenced by an IdP's CA field. +func syncDefaultIngressCAToConfig(ctx context.Context, oc *exutil.CLI, name string) (removalFunc, error) { + kubeClient := oc.AdminKubeClient() + + ca, err := kubeClient.CoreV1().ConfigMaps("openshift-config-managed").Get(ctx, "default-ingress-cert", metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("getting openshift-config-managed/default-ingress-cert: %w", err) + } + + _, err = kubeClient.CoreV1().ConfigMaps("openshift-config").Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: componentProxyTestLabels(), + }, + Data: map[string]string{ + "ca.crt": ca.Data["ca-bundle.crt"], + }, + }, metav1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("creating configmap openshift-config/%s: %w", name, err) + } + + return func(ctx context.Context) error { + return kubeClient.CoreV1().ConfigMaps("openshift-config").Delete(ctx, name, metav1.DeleteOptions{}) + }, nil +} + +func updateAuthenticationProxy(ctx context.Context, oc *exutil.CLI, proxy operatorv1.AuthenticationProxyConfig) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + auth, err := oc.AdminOperatorClient().OperatorV1().Authentications().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return err + } + auth.Spec.Proxy = proxy + _, err = oc.AdminOperatorClient().OperatorV1().Authentications().Update(ctx, auth, metav1.UpdateOptions{}) + return err + }) +} + +func addIdentityProvider(ctx context.Context, oc *exutil.CLI, idp configv1.IdentityProvider) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + oauth, err := oc.AdminConfigClient().ConfigV1().OAuths().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return err + } + oauth.Spec.IdentityProviders = append(oauth.Spec.IdentityProviders, idp) + _, err = oc.AdminConfigClient().ConfigV1().OAuths().Update(ctx, oauth, metav1.UpdateOptions{}) + return err + }) +} + +func verifyOAuthServerDeploymentProxyConfig(ctx context.Context, oc *exutil.CLI, expectedHTTPProxy, expectedHTTPSProxy, expectedNoProxy string, expectTrustedCAVolume bool) error { + kubeClient := oc.AdminKubeClient() + + return wait.PollUntilContextTimeout(ctx, 10*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + deployment, err := kubeClient.AppsV1().Deployments("openshift-authentication").Get(ctx, "oauth-openshift", metav1.GetOptions{}) + if err != nil { + g.GinkgoWriter.Printf("failed to get oauth-openshift deployment: %v\n", err) + return false, nil + } + + envVars := make(map[string]string) + for _, container := range deployment.Spec.Template.Spec.Containers { + for _, env := range container.Env { + switch env.Name { + case "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY": + envVars[env.Name] = env.Value + } + } + } + + if envVars["HTTP_PROXY"] != expectedHTTPProxy || envVars["HTTPS_PROXY"] != expectedHTTPSProxy { + g.GinkgoWriter.Printf("proxy env mismatch: HTTP_PROXY=%q (want %q), HTTPS_PROXY=%q (want %q)\n", + envVars["HTTP_PROXY"], expectedHTTPProxy, envVars["HTTPS_PROXY"], expectedHTTPSProxy) + return false, nil + } + if expectedNoProxy == "" { + if envVars["NO_PROXY"] != "" { + g.GinkgoWriter.Printf("proxy env mismatch: NO_PROXY=%q (want empty)\n", envVars["NO_PROXY"]) + return false, nil + } + } else { + // Use IsSuperset rather than exact match because the operator appends + // the apiserver IP to NO_PROXY beyond the entries we configure. + actualNoProxy := sets.New[string](strings.Split(envVars["NO_PROXY"], ",")...) + expectedNoProxyEntries := sets.New[string](strings.Split(expectedNoProxy, ",")...) + if !actualNoProxy.IsSuperset(expectedNoProxyEntries) { + g.GinkgoWriter.Printf("proxy env mismatch: NO_PROXY=%q does not contain all of %q\n", envVars["NO_PROXY"], expectedNoProxy) + return false, nil + } + } + + if !matchTrustedCAVolume(deployment, expectTrustedCAVolume) { + g.GinkgoWriter.Printf("trustedCA volume/mount present=%v (want present=%v)\n", !expectTrustedCAVolume, expectTrustedCAVolume) + return false, nil + } + return true, nil + }) +} + +func matchTrustedCAVolume(deployment *appsv1.Deployment, expectPresent bool) bool { + foundVolume := false + for _, vol := range deployment.Spec.Template.Spec.Volumes { + if vol.ConfigMap != nil && vol.ConfigMap.Name == componentProxyCAConfigMapName { + foundVolume = true + break + } + } + + foundMount := false + for _, container := range deployment.Spec.Template.Spec.Containers { + for _, mount := range container.VolumeMounts { + if mount.Name == componentProxyCAConfigMapName { + foundMount = true + break + } + } + } + + if expectPresent { + return foundVolume && foundMount + } + return !foundVolume && !foundMount +} + +func verifyTrustedCAConfigMapSynced(ctx context.Context, oc *exutil.CLI) error { + kubeClient := oc.AdminKubeClient() + + return wait.PollUntilContextTimeout(ctx, 10*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + cm, err := kubeClient.CoreV1().ConfigMaps("openshift-authentication").Get(ctx, componentProxyCAConfigMapName, metav1.GetOptions{}) + if err != nil { + return false, nil + } + return len(cm.Data) > 0, nil + }) +} diff --git a/test/extended/authentication/keycloak_client.go b/test/extended/authentication/keycloak_client.go index a91283139f30..aff858b6be28 100644 --- a/test/extended/authentication/keycloak_client.go +++ b/test/extended/authentication/keycloak_client.go @@ -6,8 +6,10 @@ import ( "encoding/json" "fmt" "io" + "maps" "net/http" "net/url" + "strconv" "k8s.io/apimachinery/pkg/runtime" ) @@ -194,6 +196,96 @@ func (kc *keycloakClient) DoRequest(method, url, contentType string, authenticat return kc.client.Do(req) } +func (kc *keycloakClient) RegenerateClientSecret(id string) (string, error) { + regenURL := *kc.adminURL + regenURL.Path += fmt.Sprintf("/clients/%s/client-secret", id) + + resp, err := kc.DoRequest(http.MethodPost, regenURL.String(), runtime.ContentTypeJSON, true, nil) + if err != nil { + return "", err + } + defer resp.Body.Close() + + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("regenerating client %q secret failed: %s - %s", id, resp.Status, respBytes) + } + + secret := map[string]string{} + if err = json.Unmarshal(respBytes, &secret); err != nil { + return "", err + } + + secretVal, ok := secret["value"] + if !ok { + return "", fmt.Errorf("failed to retrieve new secret for client %q", id) + } + + return secretVal, nil +} + +func (kc *keycloakClient) UpdateClientAccessTokenTimeout(id string, timeout int32) error { + return kc.UpdateClientRaw(id, map[string]any{ + "attributes": map[string]any{ + "access.token.lifespan": strconv.FormatInt(int64(timeout), 10), + }, + }) +} + +func (kc *keycloakClient) UpdateClientRaw(id string, changes map[string]any) error { + existing, err := kc.GetClientRaw(id) + if err != nil { + return err + } + + maps.Copy(existing, changes) + + var body bytes.Buffer + if err := json.NewEncoder(&body).Encode(existing); err != nil { + return err + } + + clientURL := *kc.adminURL + clientURL.Path += fmt.Sprintf("/clients/%s", id) + resp, err := kc.DoRequest(http.MethodPut, clientURL.String(), runtime.ContentTypeJSON, true, &body) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + respBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed updating client %q: %s - %s", id, resp.Status, respBytes) + } + return nil +} + +func (kc *keycloakClient) GetClientRaw(id string) (map[string]any, error) { + clientURL := *kc.adminURL + clientURL.Path += fmt.Sprintf("/clients/%s", id) + + resp, err := kc.DoRequest(http.MethodGet, clientURL.String(), runtime.ContentTypeJSON, true, nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("getting client %q failed: %s - %s", id, resp.Status, respBytes) + } + + result := map[string]any{} + err = json.Unmarshal(respBytes, &result) + return result, err +} + func (kc *keycloakClient) AccessToken() string { return kc.accessToken } @@ -345,8 +437,9 @@ func (kc *keycloakClient) CreateClientAudienceMapper(clientId, name string) erro } type client struct { - ClientID string `json:"clientID"` - ID string `json:"id"` + ClientID string `json:"clientId"` + ID string `json:"id"` + RedirectURIs []string `json:"redirectUris"` } // ListClients retrieves all clients diff --git a/test/extended/authentication/operator_status_helpers.go b/test/extended/authentication/operator_status_helpers.go new file mode 100644 index 000000000000..4b92f0de0c4a --- /dev/null +++ b/test/extended/authentication/operator_status_helpers.go @@ -0,0 +1,20 @@ +package authentication + +import ( + "context" + "time" + + g "github.com/onsi/ginkgo/v2" + + exutil "github.com/openshift/origin/test/extended/util" + operator "github.com/openshift/origin/test/extended/util/operator" +) + +func waitForOperatorToPickUpChanges(ctx context.Context, oc *exutil.CLI, name string) error { + progressCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + if err := exutil.WaitForOperatorProgressingTrue(progressCtx, oc.AdminConfigClient(), name); err != nil { + g.GinkgoWriter.Printf("operator %s did not become Progressing=True (may have reconciled quickly): %v\n", name, err) + } + return operator.WaitForOperatorsToSettle(ctx, oc.AdminConfigClient(), 10) +} From 9c7ac0b1449977940f0b9df1cefd81a87330729c Mon Sep 17 00:00:00 2001 From: Ondra Kupka Date: Fri, 7 Aug 2026 08:52:51 +0200 Subject: [PATCH 2/9] Wait for namespace deletion and extract trustedCA helper Make testFallbackOnProxyRemoval robust by using deleteNamespaceSync which issues a foreground-propagation delete and polls until the namespace is fully gone, eliminating the race where the Squid proxy could still be serving traffic when the operator reconciles. Extract createTrustedCAConfigMap helper to reduce inline resource creation in the test function. --- .../authentication/component_proxy.go | 20 ++------ .../authentication/component_proxy_helpers.go | 51 +++++++++++++++++++ 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/test/extended/authentication/component_proxy.go b/test/extended/authentication/component_proxy.go index de2b37eb277c..0f2a73f03463 100644 --- a/test/extended/authentication/component_proxy.go +++ b/test/extended/authentication/component_proxy.go @@ -7,7 +7,6 @@ import ( g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" operatorv1 "github.com/openshift/api/operator/v1" @@ -81,22 +80,13 @@ var _ = g.Describe("[sig-auth][Suite:openshift/conformance/serial][OCPFeatureGat func testOIDCIdPThroughComponentProxy(ctx context.Context, oc *exutil.CLI, kcSetup *keycloakProxySetup, proxyURL string, trustedCACertPEM []byte, proxyNamespace string) { withTrustedCA := len(trustedCACertPEM) > 0 - const trustedCAConfigMapName = "e2e-proxy-ca" + var trustedCAConfigMapName string if withTrustedCA { g.By("Creating trustedCA ConfigMap in openshift-config") - _, err := oc.AdminKubeClient().CoreV1().ConfigMaps("openshift-config").Create(ctx, &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: trustedCAConfigMapName, - Labels: componentProxyTestLabels(), - }, - Data: map[string]string{ - "ca-bundle.crt": string(trustedCACertPEM), - }, - }, metav1.CreateOptions{}) + cmName, cmCleanup, err := createTrustedCAConfigMap(ctx, oc, trustedCACertPEM) + g.DeferCleanup(cmCleanup) o.Expect(err).NotTo(o.HaveOccurred()) - g.DeferCleanup(func(ctx context.Context) error { - return oc.AdminKubeClient().CoreV1().ConfigMaps("openshift-config").Delete(ctx, trustedCAConfigMapName, metav1.DeleteOptions{}) - }) + trustedCAConfigMapName = cmName } proxyTrafficStart := time.Now() @@ -169,7 +159,7 @@ func testFallbackOnProxyRemoval(ctx context.Context, oc *exutil.CLI, kcSetup *ke o.Expect(err).NotTo(o.HaveOccurred()) g.By("Deleting Squid to prove the operator no longer routes through it") - err = oc.AdminKubeClient().CoreV1().Namespaces().Delete(ctx, proxyNamespace, metav1.DeleteOptions{}) + err = deleteNamespaceSync(ctx, oc, proxyNamespace, 5*time.Minute) o.Expect(err).NotTo(o.HaveOccurred()) g.By("Waiting for operator to pick up proxy removal and stabilize") diff --git a/test/extended/authentication/component_proxy_helpers.go b/test/extended/authentication/component_proxy_helpers.go index c7e047b4a0cc..b22308c77ff5 100644 --- a/test/extended/authentication/component_proxy_helpers.go +++ b/test/extended/authentication/component_proxy_helpers.go @@ -112,6 +112,31 @@ func saveAndRestoreAuthState(ctx context.Context, oc *exutil.CLI) (removalFunc, }, nil } +func createTrustedCAConfigMap(ctx context.Context, oc *exutil.CLI, caCertPEM []byte) (string, removalFunc, error) { + const configMapName = "e2e-proxy-trusted-ca" + kubeClient := oc.AdminKubeClient() + _, err := kubeClient.CoreV1().ConfigMaps("openshift-config").Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Labels: componentProxyTestLabels(), + }, + Data: map[string]string{ + "ca-bundle.crt": string(caCertPEM), + }, + }, metav1.CreateOptions{}) + if err != nil { + return "", nil, fmt.Errorf("failed to create trusted CA configmap \"openshift-config/%s\": %w", configMapName, err) + } + + return configMapName, func(ctx context.Context) error { + g.GinkgoWriter.Println("cleanup: removing trustedCA configmap") + if err := kubeClient.CoreV1().ConfigMaps("openshift-config").Delete(ctx, configMapName, metav1.DeleteOptions{}); err != nil { + g.GinkgoWriter.Printf("failed to clean up configmap \"openshift-config/%s\": %v\n", configMapName, err) + } + return nil + }, nil +} + // deploySquidProxy deploys a Squid forward proxy listening on HTTP (3128) and // HTTPS (3129) with a self-signed CA and serving certificate. func deploySquidProxy(ctx context.Context, oc *exutil.CLI) (httpProxyURL, httpsProxyURL string, caCertPEM []byte, namespace string, cleanup removalFunc, err error) { @@ -685,3 +710,29 @@ func verifyTrustedCAConfigMapSynced(ctx context.Context, oc *exutil.CLI) error { return len(cm.Data) > 0, nil }) } + +// deleteNamespaceSync deletes a namespace and polls until it is fully removed. +// Foreground delete propagation is being used, so all namespace resources are deleted by the time this function unblocks. +func deleteNamespaceSync(ctx context.Context, oc *exutil.CLI, namespace string, timeout time.Duration) error { + kubeClient := oc.AdminKubeClient() + if err := kubeClient.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{ + PropagationPolicy: new(metav1.DeletePropagationForeground), + }); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("deleting namespace %s: %w", namespace, err) + } + + g.GinkgoWriter.Printf("waiting up to %s for namespace %s to be fully deleted\n", timeout, namespace) + return wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + _, err := kubeClient.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return true, nil + } + if err != nil { + g.GinkgoWriter.Printf("error checking namespace %s: %v\n", namespace, err) + } + return false, nil + }) +} From 76503f3e579172847d309f75330776be97d82f1b Mon Sep 17 00:00:00 2001 From: Ondra Kupka Date: Fri, 7 Aug 2026 08:58:18 +0200 Subject: [PATCH 3/9] Fix attribute clobbering and trustedCA volume diagnostic Deep-merge the attributes map in UpdateClientRaw so setting a single attribute like access.token.lifespan does not wipe unrelated client attributes. Return actual volume/mount state from trustedCAVolumeState so the diagnostic log shows what was observed rather than inferring it from the expected value. --- .../authentication/component_proxy_helpers.go | 14 +++++--------- test/extended/authentication/keycloak_client.go | 8 ++++++++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/test/extended/authentication/component_proxy_helpers.go b/test/extended/authentication/component_proxy_helpers.go index b22308c77ff5..feb316a45e10 100644 --- a/test/extended/authentication/component_proxy_helpers.go +++ b/test/extended/authentication/component_proxy_helpers.go @@ -666,16 +666,16 @@ func verifyOAuthServerDeploymentProxyConfig(ctx context.Context, oc *exutil.CLI, } } - if !matchTrustedCAVolume(deployment, expectTrustedCAVolume) { - g.GinkgoWriter.Printf("trustedCA volume/mount present=%v (want present=%v)\n", !expectTrustedCAVolume, expectTrustedCAVolume) + foundVolume, foundMount := trustedCAVolumeState(deployment) + if expectTrustedCAVolume != (foundVolume && foundMount) { + g.GinkgoWriter.Printf("trustedCA volume=%v mount=%v (want present=%v)\n", foundVolume, foundMount, expectTrustedCAVolume) return false, nil } return true, nil }) } -func matchTrustedCAVolume(deployment *appsv1.Deployment, expectPresent bool) bool { - foundVolume := false +func trustedCAVolumeState(deployment *appsv1.Deployment) (foundVolume, foundMount bool) { for _, vol := range deployment.Spec.Template.Spec.Volumes { if vol.ConfigMap != nil && vol.ConfigMap.Name == componentProxyCAConfigMapName { foundVolume = true @@ -683,7 +683,6 @@ func matchTrustedCAVolume(deployment *appsv1.Deployment, expectPresent bool) boo } } - foundMount := false for _, container := range deployment.Spec.Template.Spec.Containers { for _, mount := range container.VolumeMounts { if mount.Name == componentProxyCAConfigMapName { @@ -693,10 +692,7 @@ func matchTrustedCAVolume(deployment *appsv1.Deployment, expectPresent bool) boo } } - if expectPresent { - return foundVolume && foundMount - } - return !foundVolume && !foundMount + return foundVolume, foundMount } func verifyTrustedCAConfigMapSynced(ctx context.Context, oc *exutil.CLI) error { diff --git a/test/extended/authentication/keycloak_client.go b/test/extended/authentication/keycloak_client.go index aff858b6be28..d852c2eec77d 100644 --- a/test/extended/authentication/keycloak_client.go +++ b/test/extended/authentication/keycloak_client.go @@ -241,6 +241,14 @@ func (kc *keycloakClient) UpdateClientRaw(id string, changes map[string]any) err return err } + // Deep-merge attributes so setting one attribute doesn't wipe the rest. + if changesAttrs, ok := changes["attributes"].(map[string]any); ok { + if existingAttrs, ok := existing["attributes"].(map[string]any); ok { + maps.Copy(existingAttrs, changesAttrs) + changes["attributes"] = existingAttrs + } + } + maps.Copy(existing, changes) var body bytes.Buffer From bdb030db23a7c7b35d9b62710e9c30b1aa0204e6 Mon Sep 17 00:00:00 2001 From: Ondra Kupka Date: Fri, 7 Aug 2026 09:13:48 +0200 Subject: [PATCH 4/9] Nitpick cleanups: logformat comment, reuse serviceHost, avoid mutating changes map Add a comment linking the squid logformat field order to its parser in waitForProxyTrafficFrom. Reuse the serviceHost variable for both the TLS SAN and proxy URLs instead of computing it twice. Rewrite UpdateClientRaw merge loop to avoid mutating the caller's changes map while preserving the deep-merge of attributes. --- .../authentication/component_proxy_helpers.go | 6 +++--- .../authentication/keycloak_client.go | 20 ++++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/test/extended/authentication/component_proxy_helpers.go b/test/extended/authentication/component_proxy_helpers.go index feb316a45e10..d2c256b17413 100644 --- a/test/extended/authentication/component_proxy_helpers.go +++ b/test/extended/authentication/component_proxy_helpers.go @@ -171,8 +171,8 @@ func deploySquidProxy(ctx context.Context, oc *exutil.CLI) (httpProxyURL, httpsP } ca := &libcrypto.CA{Config: caConfig, SerialGenerator: &libcrypto.RandomSerialGenerator{}} - serviceDNS := fmt.Sprintf("%s.%s.svc.cluster.local", squidServiceName, namespace) - serverCertConfig, err := ca.MakeServerCert(sets.New(serviceDNS), 2*time.Hour) + serviceHost := fmt.Sprintf("%s.%s.svc.cluster.local", squidServiceName, namespace) + serverCertConfig, err := ca.MakeServerCert(sets.New(serviceHost), 2*time.Hour) if err != nil { return "", "", nil, "", cleanup, fmt.Errorf("creating proxy server cert: %w", err) } @@ -186,6 +186,7 @@ func deploySquidProxy(ctx context.Context, oc *exutil.CLI) (httpProxyURL, httpsP return "", "", nil, "", cleanup, fmt.Errorf("encoding proxy server cert: %w", err) } + // logformat fields: method url sourceIP squidStatus httpCode — parsed by waitForProxyTrafficFrom squidConfig := fmt.Sprintf(`http_port %d https_port %d tls-cert=/etc/squid/tls/tls.crt tls-key=/etc/squid/tls/tls.key pid_filename /tmp/squid.pid @@ -325,7 +326,6 @@ buffered_logs off return "", "", nil, "", cleanup, fmt.Errorf("Squid proxy deployment did not become ready: %w", err) } - serviceHost := fmt.Sprintf("%s.%s.svc.cluster.local", squidServiceName, namespace) httpProxyURL = "http://" + net.JoinHostPort(serviceHost, strconv.Itoa(int(squidHTTPPort))) httpsProxyURL = "https://" + net.JoinHostPort(serviceHost, strconv.Itoa(int(squidHTTPSPort))) g.GinkgoWriter.Printf("Squid proxy deployed: http=%s https=%s\n", httpProxyURL, httpsProxyURL) diff --git a/test/extended/authentication/keycloak_client.go b/test/extended/authentication/keycloak_client.go index d852c2eec77d..c10bb78fcfd7 100644 --- a/test/extended/authentication/keycloak_client.go +++ b/test/extended/authentication/keycloak_client.go @@ -241,16 +241,22 @@ func (kc *keycloakClient) UpdateClientRaw(id string, changes map[string]any) err return err } - // Deep-merge attributes so setting one attribute doesn't wipe the rest. - if changesAttrs, ok := changes["attributes"].(map[string]any); ok { - if existingAttrs, ok := existing["attributes"].(map[string]any); ok { - maps.Copy(existingAttrs, changesAttrs) - changes["attributes"] = existingAttrs + // Shallow-merge top-level fields, deep-merge "attributes" to avoid clobbering. + for k, v := range changes { + if k == "attributes" { + if changesAttrs, ok := v.(map[string]any); ok { + existingAttrs, _ := existing["attributes"].(map[string]any) + if existingAttrs == nil { + existingAttrs = make(map[string]any) + } + maps.Copy(existingAttrs, changesAttrs) + existing["attributes"] = existingAttrs + continue + } } + existing[k] = v } - maps.Copy(existing, changes) - var body bytes.Buffer if err := json.NewEncoder(&body).Encode(existing); err != nil { return err From 1fa87c62605fda179026e7201bbc8e539159cf82 Mon Sep 17 00:00:00 2001 From: Ondra Kupka Date: Fri, 7 Aug 2026 12:58:25 +0200 Subject: [PATCH 5/9] Only set Authorization header on authenticated Keycloak requests DoRequest unconditionally set the Bearer token header even when authenticated=false, leaking an empty/stale token on unauthenticated calls like the initial token exchange. --- test/extended/authentication/component_proxy_helpers.go | 2 +- test/extended/authentication/keycloak_client.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/extended/authentication/component_proxy_helpers.go b/test/extended/authentication/component_proxy_helpers.go index d2c256b17413..ce3c215cdcb5 100644 --- a/test/extended/authentication/component_proxy_helpers.go +++ b/test/extended/authentication/component_proxy_helpers.go @@ -82,7 +82,7 @@ func saveAndRestoreAuthState(ctx context.Context, oc *exutil.CLI) (removalFunc, _, err = operatorClient.OperatorV1().Authentications().Update(ctx, fresh, metav1.UpdateOptions{}) return err }); err != nil { - g.GinkgoWriter.Printf("cleanup: failed to restore Authentication CR: %v\n", err) + g.GinkgoWriter.Printf("cleanup: failed to restore authentication/cluster: %v\n", err) } g.GinkgoWriter.Println("cleanup: restoring oauth/cluster") diff --git a/test/extended/authentication/keycloak_client.go b/test/extended/authentication/keycloak_client.go index c10bb78fcfd7..b2e885f6ff15 100644 --- a/test/extended/authentication/keycloak_client.go +++ b/test/extended/authentication/keycloak_client.go @@ -189,7 +189,9 @@ func (kc *keycloakClient) DoRequest(method, url, contentType string, authenticat return nil, fmt.Errorf("building request: %w", err) } - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", kc.accessToken)) + if authenticated { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", kc.accessToken)) + } req.Header.Set("Content-Type", contentType) req.Header.Set("Accept", runtime.ContentTypeJSON) From cbf9e9974ac4f25723c9d61575f442e03f40091c Mon Sep 17 00:00:00 2001 From: Ondra Kupka Date: Fri, 7 Aug 2026 13:03:41 +0200 Subject: [PATCH 6/9] Match trustedCA volume mount by volume name, not ConfigMap name The mount check compared mount.Name against the ConfigMap name constant, which only worked because the operator happened to use the same string for both. Use the actual volume name from the matched volume entry. --- test/extended/authentication/component_proxy_helpers.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/extended/authentication/component_proxy_helpers.go b/test/extended/authentication/component_proxy_helpers.go index ce3c215cdcb5..b76867ea5174 100644 --- a/test/extended/authentication/component_proxy_helpers.go +++ b/test/extended/authentication/component_proxy_helpers.go @@ -676,16 +676,21 @@ func verifyOAuthServerDeploymentProxyConfig(ctx context.Context, oc *exutil.CLI, } func trustedCAVolumeState(deployment *appsv1.Deployment) (foundVolume, foundMount bool) { + var volumeName string for _, vol := range deployment.Spec.Template.Spec.Volumes { if vol.ConfigMap != nil && vol.ConfigMap.Name == componentProxyCAConfigMapName { foundVolume = true + volumeName = vol.Name break } } + if !foundVolume { + return false, false + } for _, container := range deployment.Spec.Template.Spec.Containers { for _, mount := range container.VolumeMounts { - if mount.Name == componentProxyCAConfigMapName { + if mount.Name == volumeName { foundMount = true break } From 66b07f1fd17b5def5a7f71e0f9ef393d16d1e1dc Mon Sep 17 00:00:00 2001 From: Ondra Kupka Date: Fri, 7 Aug 2026 13:09:16 +0200 Subject: [PATCH 7/9] Reverse cleanup order to LIFO so resources are torn down before auth state is restored --- test/extended/authentication/component_proxy.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/extended/authentication/component_proxy.go b/test/extended/authentication/component_proxy.go index 0f2a73f03463..e066a0f6bded 100644 --- a/test/extended/authentication/component_proxy.go +++ b/test/extended/authentication/component_proxy.go @@ -2,6 +2,7 @@ package authentication import ( "context" + "slices" "time" g "github.com/onsi/ginkgo/v2" @@ -59,6 +60,8 @@ var _ = g.Describe("[sig-auth][Suite:openshift/conformance/serial][OCPFeatureGat }) g.AfterEach(func() { + // We are appending cleanups, but we actually want to do LIFO. + slices.Reverse(cleanups) _ = removeResources(ctx, cleanups...) g.By("Waiting for operators to be stable after test") From 5a56cf6d82cc63a2a131d5cbd261ec7e3c69a85a Mon Sep 17 00:00:00 2001 From: Ondra Kupka Date: Fri, 7 Aug 2026 14:48:31 +0200 Subject: [PATCH 8/9] Verify OAuth deployment has no stale proxy config before each test Poll the oauth-openshift deployment in BeforeEach to confirm proxy env vars and trustedCA volume are clean before proceeding. Prevents test-to-test interference when the previous test's cleanup is still propagating through the operator. --- test/extended/authentication/component_proxy.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/extended/authentication/component_proxy.go b/test/extended/authentication/component_proxy.go index e066a0f6bded..51c06430b966 100644 --- a/test/extended/authentication/component_proxy.go +++ b/test/extended/authentication/component_proxy.go @@ -54,6 +54,10 @@ var _ = g.Describe("[sig-auth][Suite:openshift/conformance/serial][OCPFeatureGat err = operator.WaitForOperatorsToSettle(ctx, oc.AdminConfigClient(), 10) o.Expect(err).NotTo(o.HaveOccurred()) + g.By("Waiting for OAuth server deployment to be stable before test") + err = verifyOAuthServerDeploymentProxyConfig(ctx, oc, "", "", "", false) + o.Expect(err).NotTo(o.HaveOccurred()) + g.GinkgoWriter.Printf("Squid proxy URL: http=%s https=%s\n", httpProxyURL, httpsProxyURL) g.GinkgoWriter.Printf("Keycloak issuer URL: %s\n", kcSetup.issuerURL) g.GinkgoWriter.Printf("Keycloak namespace: %s\n", kcSetup.namespace) From 2db333f06b572f5e6819a000bc4e154a339fa8ec Mon Sep 17 00:00:00 2001 From: Evan Hearne Date: Fri, 7 Aug 2026 18:55:31 +0100 Subject: [PATCH 9/9] add e2e tests for oauth-server functionality this change migrate tests over + add helper for to/from traffic check --- .../authentication/component_proxy.go | 420 +++++++++++++++++- .../authentication/component_proxy_helpers.go | 32 +- 2 files changed, 445 insertions(+), 7 deletions(-) diff --git a/test/extended/authentication/component_proxy.go b/test/extended/authentication/component_proxy.go index 51c06430b966..0b89370fc5ad 100644 --- a/test/extended/authentication/component_proxy.go +++ b/test/extended/authentication/component_proxy.go @@ -2,7 +2,11 @@ package authentication import ( "context" + "fmt" + "io" + "net/url" "slices" + "strings" "time" g "github.com/onsi/ginkgo/v2" @@ -11,9 +15,18 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" operatorv1 "github.com/openshift/api/operator/v1" - + libcrypto "github.com/openshift/library-go/pkg/crypto" + "github.com/openshift/library-go/pkg/oauth/tokenrequest" + "github.com/openshift/library-go/pkg/oauth/tokenrequest/challengehandlers" exutil "github.com/openshift/origin/test/extended/util" operator "github.com/openshift/origin/test/extended/util/operator" + + authnv1 "k8s.io/api/authentication/v1" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ) var _ = g.Describe("[sig-auth][Suite:openshift/conformance/serial][OCPFeatureGate:AuthenticationComponentProxy][Serial]", func() { @@ -82,8 +95,413 @@ var _ = g.Describe("[sig-auth][Suite:openshift/conformance/serial][OCPFeatureGat g.It("operator should fall back to original configuration on spec.proxy removal", func() { testFallbackOnProxyRemoval(ctx, oc, kcSetup, httpProxyURL, proxyNamespace) }) + g.It("should perform full OIDC login flow through the proxy when auth proxy config is applied", func() { + testProxyConfigPerformOIDCLogin(ctx, oc, kcSetup, httpProxyURL, proxyNamespace) + }) + g.It("should hot-reload mounted CA file on change when spec.proxy.trustedCA is set", func() { + testHotReloadCAFileChange(ctx, oc, caCertPEM, kcSetup, httpsProxyURL, proxyNamespace) + }) + g.It("should bypass proxy by directly connecting to idp to perform OIDC login flow when spec.proxy.noProxy contains idp", func() { + testBypassProxyNoProxyHost(ctx, oc, caCertPEM, kcSetup, httpProxyURL, httpsProxyURL, proxyNamespace) + }) }) +func createKeycloakUserPasswordGroup(kcSetup *keycloakProxySetup) (kcUser, kcPass, kcGroup string) { + testID := rand.String(8) + + kcGroup = fmt.Sprintf("e2e-proxy-kc-group-%s", testID) + kcUser = fmt.Sprintf("e2e-proxy-kc-user-%s", testID) + kcPass = fmt.Sprintf("e2e-proxy-kc-pass-%s", testID) + + err := kcSetup.client.CreateGroup(kcGroup) + o.Expect(err).NotTo(o.HaveOccurred()) + + err = kcSetup.client.CreateUser(kcUser, kcPass, kcGroup) + o.Expect(err).NotTo(o.HaveOccurred()) + + return kcUser, kcPass, kcGroup +} + +func assertOIDCLogin(ctx context.Context, oc *exutil.CLI, username, password, expectedGroup string) { + g.GinkgoHelper() + + kubeConfig := oc.AdminConfig() + + routeClient := oc.AdminRouteClient() + route, err := routeClient.RouteV1().Routes("openshift-authentication").Get(ctx, "oauth-openshift", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "should be able to get the OAuth server route") + oauthServerURL := fmt.Sprintf("https://%s", route.Spec.Host) + + err = wait.PollUntilContextTimeout(ctx, 10*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + tokenOpts := tokenrequest.NewRequestTokenOptions(rest.CopyConfig(kubeConfig), false) + tokenOpts, err := tokenOpts.WithChallengeHandlers( + challengehandlers.NewBasicChallengeHandler(oauthServerURL, "", nil, io.Discard, nil, username, password), + ) + if err != nil { + g.GinkgoWriter.Printf("failed to create challenge handler: %v", err) + return false, nil + } + + token, err := tokenOpts.RequestToken() + if err != nil { + g.GinkgoWriter.Printf("failed to request token: %v", err) + return false, nil + } + if token == "" { + g.GinkgoWriter.Print("received empty token") + return false, nil + } + + tokenConfig := rest.AnonymousClientConfig(kubeConfig) + tokenConfig.BearerToken = token + tokenKubeClient, err := kubernetes.NewForConfig(tokenConfig) + if err != nil { + g.GinkgoWriter.Printf("failed to create kube client with token: %v", err) + return false, nil + } + + ssr, err := tokenKubeClient.AuthenticationV1().SelfSubjectReviews().Create(ctx, &authnv1.SelfSubjectReview{}, metav1.CreateOptions{}) + if err != nil { + g.GinkgoWriter.Printf("failed to create SelfSubjectReview: %v", err) + return false, nil + } + + if ssr.Status.UserInfo.Username == "" { + g.GinkgoWriter.Print("SelfSubjectReview returned empty username") + return false, nil + } + + if slices.Contains(ssr.Status.UserInfo.Groups, expectedGroup) { + return true, nil + } + g.GinkgoWriter.Printf("expected group %q not found in groups: %v", expectedGroup, ssr.Status.UserInfo.Groups) + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "OIDC login flow should succeed") +} + +func deleteOIDCUserAndIdentities(ctx context.Context, oc *exutil.CLI, username string) { + g.GinkgoHelper() + userClient := oc.AdminUserClient().UserV1() + + user, err := userClient.Users().Get(ctx, username, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "should be able to get user %q", username) + + for _, identity := range user.Identities { + err = userClient.Identities().Delete(ctx, identity, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "should be able to delete identity %q", identity) + } + + err = userClient.Users().Delete(ctx, username, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "should be able to delete user %q", username) +} + +func getOAuthServerPodIPs(ctx context.Context, oc *exutil.CLI) []string { + g.GinkgoHelper() + oauthPods, err := oc.AdminKubeClient().CoreV1().Pods("openshift-authentication").List(ctx, metav1.ListOptions{LabelSelector: "app=oauth-openshift"}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(oauthPods.Items).NotTo(o.BeEmpty()) + var ips []string + for _, p := range oauthPods.Items { + ips = append(ips, p.Status.PodIP) + } + return ips +} + +func testProxyConfigPerformOIDCLogin(ctx context.Context, oc *exutil.CLI, kcSetup *keycloakProxySetup, httpProxyURL, proxyNamespace string) { + g.By("setting direct access grant for oauth flow") + kcClient, err := kcSetup.client.GetClientByClientID(kcSetup.clientID) + o.Expect(err).NotTo(o.HaveOccurred()) + err = kcSetup.client.UpdateClientRaw(kcClient.ID, map[string]any{ + "directAccessGrantsEnabled": true, + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + kcUser, kcPass, kcGroup := createKeycloakUserPasswordGroup(kcSetup) + + err = updateAuthenticationProxy(ctx, oc, operatorv1.AuthenticationProxyConfig{ + HTTPSProxy: httpProxyURL, + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verifying oauth-server has HTTPS_PROXY but not HTTP_PROXY") + err = verifyOAuthServerDeploymentProxyConfig( + ctx, oc, "", httpProxyURL, ".cluster.local,.svc,127.0.0.1,localhost", + false) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Registering Keycloak as OIDC IdP") + idpCleanups, err := addKeycloakOIDCIdPForProxy(ctx, oc, kcSetup) + g.DeferCleanup(func() { + _ = removeResources(ctx, idpCleanups...) + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for operator to pick up proxy and IdP changes and stabilize") + err = waitForOperatorToPickUpChanges(ctx, oc, "authentication") + o.Expect(err).NotTo(o.HaveOccurred()) + + logCutOff := time.Now() + + g.By("Performing full OIDC login flow through component proxy") + assertOIDCLogin(ctx, oc, kcUser, kcPass, kcGroup) + + g.By("Verifying Keycloak traffic from oauth-server went through the Squid proxy") + issuerURL, err := url.Parse(kcSetup.issuerURL) + o.Expect(err).NotTo(o.HaveOccurred()) + keycloakHost := issuerURL.Hostname() + + ips := getOAuthServerPodIPs(ctx, oc) + err = waitForProxyTrafficFromTo(ctx, oc, proxyNamespace, ips, keycloakHost, logCutOff, 5*time.Minute) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Logging user out for next login") + deleteOIDCUserAndIdentities(ctx, oc, kcUser) + + g.By("Removing component-scoped proxy config") + err = updateAuthenticationProxy(ctx, oc, operatorv1.AuthenticationProxyConfig{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Deleting Squid to prove the operator no longer routes through it") + err = deleteNamespaceSync(ctx, oc, proxyNamespace, 5*time.Minute) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for operator to reconcile proxy removal") + err = waitForOperatorToPickUpChanges(ctx, oc, "authentication") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Performing OIDC login flow via direct IdP connectivity after proxy removal") + assertOIDCLogin(ctx, oc, kcUser, kcPass, kcGroup) +} + +func testHotReloadCAFileChange(ctx context.Context, oc *exutil.CLI, caCertPEM []byte, kcSetup *keycloakProxySetup, httpsProxyURL, proxyNamespace string) { + g.By("setting direct access grant for oauth flow") + kcClient, err := kcSetup.client.GetClientByClientID(kcSetup.clientID) + o.Expect(err).NotTo(o.HaveOccurred()) + err = kcSetup.client.UpdateClientRaw(kcClient.ID, map[string]any{ + "directAccessGrantsEnabled": true, + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + kcUser, kcPass, kcGroup := createKeycloakUserPasswordGroup(kcSetup) + kubeClient := oc.AdminKubeClient() + g.By("Creating trustedCA ConfigMap in openshift-config") + configMapName, cmCleanup, err := createTrustedCAConfigMap(ctx, oc, caCertPEM) + g.DeferCleanup(cmCleanup) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Setting component-scoped proxy with trustedCA") + err = updateAuthenticationProxy(ctx, oc, operatorv1.AuthenticationProxyConfig{ + HTTPSProxy: httpsProxyURL, + TrustedCA: operatorv1.AuthenticationConfigMapReference{ + Name: configMapName, + }, + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Registering Keycloak as OIDC IdP") + idpCleanups, err := addKeycloakOIDCIdPForProxy(ctx, oc, kcSetup) + g.DeferCleanup(func() { + _ = removeResources(ctx, idpCleanups...) + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for operator to pick up proxy, trustedCA and IdP changes and stabilize") + err = waitForOperatorToPickUpChanges(ctx, oc, "authentication") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verifying trustedCA ConfigMap is synced to openshift-authentication namespace") + err = verifyTrustedCAConfigMapSynced(ctx, oc) + o.Expect(err).NotTo(o.HaveOccurred()) + + logCutOff := time.Now() + + g.By("Verifying OIDC login works after setting proxy with trustedCA") + assertOIDCLogin(ctx, oc, kcUser, kcPass, kcGroup) + + g.By("Verifying Keycloak traffic from oauth-server went through the Squid proxy") + issuerURL, err := url.Parse(kcSetup.issuerURL) + o.Expect(err).NotTo(o.HaveOccurred()) + keycloakHost := issuerURL.Hostname() + + ips := getOAuthServerPodIPs(ctx, oc) + err = waitForProxyTrafficFromTo(ctx, oc, proxyNamespace, ips, keycloakHost, logCutOff, 5*time.Minute) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Recording oauth-server pod names before CA rotation") + oauthServerPodList, err := kubeClient.CoreV1().Pods("openshift-authentication").List(ctx, metav1.ListOptions{LabelSelector: "app=oauth-openshift"}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(oauthServerPodList.Items).NotTo(o.BeEmpty()) + + podNamesBefore := sets.New[string]() + for _, pod := range oauthServerPodList.Items { + podNamesBefore.Insert(pod.Name) + } + + g.By("Rotating CA: generating new CA and server cert") + newCAConfig, err := libcrypto.MakeSelfSignedCAConfigForDuration("squid-proxy-ca", 2*time.Hour) + o.Expect(err).NotTo(o.HaveOccurred()) + newCA := &libcrypto.CA{Config: newCAConfig, SerialGenerator: &libcrypto.RandomSerialGenerator{}} + + serviceDNS := fmt.Sprintf("%s.%s.svc.cluster.local", squidServiceName, proxyNamespace) + newServerCertConfig, err := newCA.MakeServerCert(sets.New(serviceDNS), 2*time.Hour) + o.Expect(err).NotTo(o.HaveOccurred()) + + newCACertPEM, _, err := newCAConfig.GetPEMBytes() + o.Expect(err).NotTo(o.HaveOccurred()) + newServerCertPEM, newServerKeyPEM, err := newServerCertConfig.GetPEMBytes() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Updating squid-tls Secret with rotated cert") + tlsSecret, err := kubeClient.CoreV1().Secrets(proxyNamespace).Get(ctx, "squid-tls", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + tlsSecret.Data["tls.crt"] = newServerCertPEM + tlsSecret.Data["tls.key"] = newServerKeyPEM + _, err = kubeClient.CoreV1().Secrets(proxyNamespace).Update(ctx, tlsSecret, metav1.UpdateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + squidPods, err := kubeClient.CoreV1().Pods(proxyNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=squid-proxy", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(squidPods.Items).NotTo(o.BeEmpty()) + + g.By("Waiting for squid-tls Secret to propagate to pod volume") + err = wait.PollUntilContextTimeout(ctx, 10*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + output, execErr := oc.AsAdmin().Run("exec").Args( + "-n", proxyNamespace, + squidPods.Items[0].Name, + "-c", "squid", + "--", "cat", "/etc/squid/tls/tls.crt", + ).Output() + if execErr != nil { + g.GinkgoWriter.Printf("failed to read cert from squid pod: %v\n", execErr) + return false, nil + } + if !strings.Contains(strings.TrimSpace(output), strings.TrimSpace(string(newServerCertPEM))) { + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "squid-tls Secret should propagate to pod volume") + + g.By("Reconfiguring Squid to pick up new cert") + output, err := oc.AsAdmin().Run("exec").Args( + "-n", proxyNamespace, + squidPods.Items[0].Name, + "-c", "squid", + "--", "/usr/sbin/squid", "-k", "reconfigure", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred(), "squid reconfigure failed: %s", string(output)) + + g.By("Updating trustedCA ConfigMap with new CA") + cm, err := kubeClient.CoreV1().ConfigMaps("openshift-config").Get(ctx, configMapName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + cm.Data["ca-bundle.crt"] = string(newCACertPEM) + _, err = kubeClient.CoreV1().ConfigMaps("openshift-config").Update(ctx, cm, metav1.UpdateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verifying operator re-syncs promptly after trustedCA ConfigMap update") + err = operator.WaitForOperatorsToSettle(ctx, oc.AdminConfigClient(), 1) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for oauth-server pods to pick up the new CA file") + // https://github.com/openshift/cluster-authentication-operator/blob/master/ + // pkg/controllers/configobservation/oauth/observe_proxy_trusted_ca.go#L15 + caFilePath := "/var/config/system/configmaps/v4-0-config-system-auth-proxy-ca/ca-bundle.crt" + err = wait.PollUntilContextTimeout(ctx, 10*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + for _, pod := range podNamesBefore.UnsortedList() { + output, execErr := oc.AsAdmin().Run("exec").Args( + "-n", "openshift-authentication", + pod, + "-c", "oauth-openshift", + "--", "cat", caFilePath, + ).Output() + if execErr != nil { + g.GinkgoWriter.Printf("failed to read CA file from pod %s: %v\n", pod, execErr) + return false, nil + } + if !strings.Contains(strings.TrimSpace(output), strings.TrimSpace(string(newCACertPEM))) { + return false, nil + } + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "oauth-server pods should have picked up the new CA file") + + g.By("Logging user out for next login") + deleteOIDCUserAndIdentities(ctx, oc, kcUser) + + g.By("Verifying OIDC login works after CA rotation") + assertOIDCLogin(ctx, oc, kcUser, kcPass, kcGroup) + + g.By("Verifying oauth-server pods were NOT redeployed after CA rotation") + oauthServerPodListAfter, err := kubeClient.CoreV1().Pods("openshift-authentication").List(ctx, metav1.ListOptions{LabelSelector: "app=oauth-openshift"}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(oauthServerPodListAfter.Items).NotTo(o.BeEmpty()) + + podNamesAfter := sets.New[string]() + for _, pod := range oauthServerPodListAfter.Items { + podNamesAfter.Insert(pod.Name) + } + + o.Expect(podNamesAfter.Equal(podNamesBefore)).To(o.BeTrue(), "oauth-server pods should not have been redeployed after CA file change") +} + +func testBypassProxyNoProxyHost(ctx context.Context, oc *exutil.CLI, caCertPEM []byte, kcSetup *keycloakProxySetup, httpProxyURL, httpsProxyURL, proxyNamespace string) { + g.By("setting direct access grant for oauth flow") + kcClient, err := kcSetup.client.GetClientByClientID(kcSetup.clientID) + o.Expect(err).NotTo(o.HaveOccurred()) + err = kcSetup.client.UpdateClientRaw(kcClient.ID, map[string]any{ + "directAccessGrantsEnabled": true, + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + kcUser, kcPass, kcGroup := createKeycloakUserPasswordGroup(kcSetup) + + g.By("Setting component-scoped proxy with noProxy") + issuerURL, err := url.Parse(kcSetup.issuerURL) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating trustedCA ConfigMap in openshift-config") + configMapName, cmCleanup, err := createTrustedCAConfigMap(ctx, oc, caCertPEM) + g.DeferCleanup(cmCleanup) + o.Expect(err).NotTo(o.HaveOccurred()) + + keycloakHost := issuerURL.Hostname() + err = updateAuthenticationProxy(ctx, oc, operatorv1.AuthenticationProxyConfig{ + HTTPProxy: httpProxyURL, + HTTPSProxy: httpsProxyURL, + TrustedCA: operatorv1.AuthenticationConfigMapReference{ + Name: configMapName, + }, + NoProxy: []string{keycloakHost}, + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Registering Keycloak as OIDC IdP") + idpCleanups, err := addKeycloakOIDCIdPForProxy(ctx, oc, kcSetup) + g.DeferCleanup(func() { + _ = removeResources(ctx, idpCleanups...) + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for operator to pick up proxy and IdP changes and stabilize") + err = waitForOperatorToPickUpChanges(ctx, oc, "authentication") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verifying oauth-server has HTTP_PROXY, HTTPS_PROXY, and NO_PROXY with custom entry") + err = verifyOAuthServerDeploymentProxyConfig(ctx, oc, httpProxyURL, httpsProxyURL, ".cluster.local,.svc,127.0.0.1,localhost,"+keycloakHost, true) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Deleting Squid proxy namespace to prove noProxy bypasses it") + err = deleteNamespaceSync(ctx, oc, proxyNamespace, 5*time.Minute) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verifying OIDC login works after setting proxy with noProxy") + assertOIDCLogin(ctx, oc, kcUser, kcPass, kcGroup) +} + func testOIDCIdPThroughComponentProxy(ctx context.Context, oc *exutil.CLI, kcSetup *keycloakProxySetup, proxyURL string, trustedCACertPEM []byte, proxyNamespace string) { withTrustedCA := len(trustedCACertPEM) > 0 diff --git a/test/extended/authentication/component_proxy_helpers.go b/test/extended/authentication/component_proxy_helpers.go index b76867ea5174..36f385e87602 100644 --- a/test/extended/authentication/component_proxy_helpers.go +++ b/test/extended/authentication/component_proxy_helpers.go @@ -332,10 +332,6 @@ buffered_logs off return httpProxyURL, httpsProxyURL, caCertPEM, namespace, cleanup, nil } -func getSquidProxyLogs(ctx context.Context, oc *exutil.CLI, namespace string) (string, error) { - return getSquidProxyLogsSince(ctx, oc, namespace, time.Time{}) -} - func getSquidProxyLogsSince(ctx context.Context, oc *exutil.CLI, namespace string, since time.Time) (string, error) { kubeClient := oc.AdminKubeClient() @@ -385,6 +381,30 @@ func waitForProxyTrafficFrom(ctx context.Context, oc *exutil.CLI, proxyNamespace }) } +// waitForProxyTrafficFromTo polls the squid access log until a CONNECT entry +// from one of the given source IPs to the given destination host appears, +// confirming that the source routed traffic to the destination through the +// proxy. The log format is: "method url sourceIP status httpCode". +func waitForProxyTrafficFromTo(ctx context.Context, oc *exutil.CLI, proxyNamespace string, sourceIPs []string, destHost string, since time.Time, timeout time.Duration) error { + g.GinkgoWriter.Printf("waiting up to %s for proxy traffic from %v to %s\n", timeout, sourceIPs, destHost) + allowedIPs := sets.New(sourceIPs...) + return wait.PollUntilContextTimeout(ctx, 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + logs, err := getSquidProxyLogsSince(ctx, oc, proxyNamespace, since) + if err != nil { + g.GinkgoWriter.Printf("failed to read squid logs: %v\n", err) + return false, nil + } + for line := range strings.SplitSeq(logs, "\n") { + parts := strings.Fields(line) + if len(parts) >= 4 && parts[0] == "CONNECT" && strings.HasPrefix(parts[1], destHost) && allowedIPs.Has(parts[2]) && parts[3] == "TCP_TUNNEL" { + g.GinkgoWriter.Printf("confirmed proxy traffic from %s to %s: %s\n", parts[2], destHost, line) + return true, nil + } + } + return false, nil + }) +} + // keycloakProxySetup holds the results of deploying Keycloak for proxy tests, // before the IdP is registered in OpenShift. type keycloakProxySetup struct { @@ -658,8 +678,8 @@ func verifyOAuthServerDeploymentProxyConfig(ctx context.Context, oc *exutil.CLI, } else { // Use IsSuperset rather than exact match because the operator appends // the apiserver IP to NO_PROXY beyond the entries we configure. - actualNoProxy := sets.New[string](strings.Split(envVars["NO_PROXY"], ",")...) - expectedNoProxyEntries := sets.New[string](strings.Split(expectedNoProxy, ",")...) + actualNoProxy := sets.New(strings.Split(envVars["NO_PROXY"], ",")...) + expectedNoProxyEntries := sets.New(strings.Split(expectedNoProxy, ",")...) if !actualNoProxy.IsSuperset(expectedNoProxyEntries) { g.GinkgoWriter.Printf("proxy env mismatch: NO_PROXY=%q does not contain all of %q\n", envVars["NO_PROXY"], expectedNoProxy) return false, nil