From 80cc7c5fcdfd4378d0fc047a8d78befeb8a968e4 Mon Sep 17 00:00:00 2001 From: sauagarwa Date: Wed, 27 May 2026 20:04:37 -0400 Subject: [PATCH 1/6] feat(helm): add optional PostgreSQL backing store with Secret-based credentials - Add postgres.enabled and postgres.deploy values to control database backend (SQLite vs PostgreSQL) and subchart deployment independently. - Introduce db-secret.yaml template for Opaque Secret with assembled postgresql:// connection string injected via OPENSHELL_DB_URL env var. - Add Bitnami PostgreSQL as optional subchart dependency keyed on postgres.deploy to prevent subchart deployment in external mode. - Externalize JWT signing key file mode via sandboxJwt.secretDefaultMode with 0400 default matching upstream. - Add validation guard for postgres.deploy=true without postgres.enabled. - Add helm unit tests covering internal, external, URL-override, special character encoding, and misconfiguration error paths. - Update README with Kubernetes and OpenShift install examples for bundled and external PostgreSQL configurations. - Add helm dependency build to lint and unittest tasks. --- .gitignore | 3 + deploy/helm/openshell/Chart.lock | 6 + deploy/helm/openshell/Chart.yaml | 6 + deploy/helm/openshell/README.md | 70 +++++++++- deploy/helm/openshell/templates/_helpers.tpl | 26 ++++ .../helm/openshell/templates/db-secret.yaml | 14 ++ .../openshell/templates/gateway-config.yaml | 3 +- .../helm/openshell/templates/statefulset.yaml | 18 ++- .../openshell/tests/gateway_config_test.yaml | 122 ++++++++++++++++++ deploy/helm/openshell/values.yaml | 39 ++++++ tasks/helm.toml | 2 + 11 files changed, 304 insertions(+), 5 deletions(-) create mode 100644 deploy/helm/openshell/Chart.lock create mode 100644 deploy/helm/openshell/templates/db-secret.yaml diff --git a/.gitignore b/.gitignore index 24a77fce2..fb8679fa7 100644 --- a/.gitignore +++ b/.gitignore @@ -187,6 +187,9 @@ rootfs/ # Docker build artifacts (image tarballs, packaged helm charts) deploy/docker/.build/ +# Helm subchart tarballs (regenerated by `helm dependency build`) +deploy/helm/openshell/charts/ + # SBOM generated output (JSON, CSV) — release artifacts, not committed deploy/sbom/output/ diff --git a/deploy/helm/openshell/Chart.lock b/deploy/helm/openshell/Chart.lock new file mode 100644 index 000000000..f1a95f424 --- /dev/null +++ b/deploy/helm/openshell/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: postgresql + repository: oci://registry-1.docker.io/bitnamicharts + version: 18.6.7 +digest: sha256:ad78500c7c3a7ee365fd151890cf3368444d6b167c972052fc245024f5a25d9c +generated: "2026-05-27T17:48:47.648592-04:00" diff --git a/deploy/helm/openshell/Chart.yaml b/deploy/helm/openshell/Chart.yaml index 06608adb3..fbfba7b2c 100644 --- a/deploy/helm/openshell/Chart.yaml +++ b/deploy/helm/openshell/Chart.yaml @@ -11,3 +11,9 @@ type: application # empty), so a released chart automatically pulls the matching gateway and supervisor images. version: 0.0.0 appVersion: "0.0.0" +dependencies: + - name: postgresql + version: 18.6.7 + repository: oci://registry-1.docker.io/bitnamicharts + condition: postgres.deploy + alias: postgres diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 9df0b91a0..0d8e1bf02 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -32,9 +32,8 @@ oc create ns openshell # Sandboxes are deployed into the openshell namespace and use the openshell-sandbox service account oc adm policy add-scc-to-user privileged -z openshell-sandbox -n openshell -# Deploy openshell with overrides to allow SCC assignment of fsGroup and runAsUser for the gateway +# Deploy openshell with overrides for OpenShift SCC compatibility helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ - --set pkiInitJob.enabled=false \ --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null @@ -58,6 +57,73 @@ See [`values.yaml`](values.yaml) for source defaults. Selected overlays: - [`ci/values-cert-manager.yaml`](ci/values-cert-manager.yaml) - cert-manager integration - [`ci/values-keycloak.yaml`](ci/values-keycloak.yaml) - Keycloak OIDC integration +### Database backend + +By default, OpenShell uses SQLite: + +```yaml +server: + dbUrl: "sqlite:/var/openshell/openshell.db" +postgres: + enabled: false +``` + +Enable bundled PostgreSQL: + +```bash +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set postgres.enabled=true \ + --set postgres.deploy=true \ + --set postgres.auth.password=my-secret-password +``` + +Enable bundled PostgreSQL(OpenShift): + +```bash +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set postgres.enabled=true \ + --set postgres.deploy=true \ + --set postgres.auth.password=my-secret-password \ + --set server.disableTls=true \ + --set podSecurityContext.fsGroup=null \ + --set securityContext.runAsUser=null +``` + +Use external PostgreSQL: + +```bash +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set postgres.enabled=true \ + --set postgres.external.host=my-postgres.example.com \ + --set postgres.external.port=5432 \ + --set postgres.external.database=openshell \ + --set postgres.external.username=openshell \ + --set postgres.external.password=my-password +``` + +Use external PostgreSQL (OpenShift): + +```bash +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set postgres.enabled=true \ + --set postgres.external.host=my-postgres.example.com \ + --set postgres.external.port=5432 \ + --set postgres.external.database=openshell \ + --set postgres.external.username=openshell \ + --set postgres.external.password=my-password \ + --set server.disableTls=true \ + --set podSecurityContext.fsGroup=null \ + --set securityContext.runAsUser=null +``` + +Or provide a full connection URL directly: + +```bash +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set postgres.enabled=true \ + --set postgres.external.url="postgres://user:pass@host:5432/db?sslmode=require" +``` + ## PKI bootstrap By default, a pre-install/pre-upgrade hook Job runs `openshell-gateway generate-certs` diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 3e375a54a..5f1ca066d 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -102,6 +102,32 @@ Namespace where sandbox pods are created. An explicit {{- .Values.server.sandboxNamespace | default .Release.Namespace -}} {{- end }} +{{/* +Gateway database URL. +- postgres.enabled=false: use .Values.server.dbUrl (default sqlite) +- postgres.enabled=true + deploy=true: derive URL from bundled postgres subchart +- postgres.enabled=true + deploy=false: use external.url or compose external fields +*/}} +{{- define "openshell.dbUrl" -}} +{{- if .Values.postgres.enabled -}} +{{- if not .Values.postgres.deploy -}} +{{- if .Values.postgres.external.url -}} +{{- .Values.postgres.external.url -}} +{{- else -}} +{{- $host := required "postgres.external.host is required when postgres.deploy=false and no postgres.external.url is provided" .Values.postgres.external.host -}} +{{- $pw := required "postgres.external.password is required when postgres.deploy=false and no postgres.external.url is provided" .Values.postgres.external.password -}} +{{- printf "postgres://%s:%s@%s:%d/%s" (.Values.postgres.external.username | urlquery) ($pw | urlquery) $host (int (default 5432 .Values.postgres.external.port)) .Values.postgres.external.database -}} +{{- end -}} +{{- else -}} +{{- $pw := required "postgres.auth.password must be set when postgres.enabled=true" .Values.postgres.auth.password -}} +{{- $host := .Values.postgres.host | default (printf "%s-postgres.%s.svc.cluster.local" .Release.Name .Release.Namespace) -}} +{{- printf "postgres://%s:%s@%s:%d/%s" (.Values.postgres.auth.username | urlquery) ($pw | urlquery) $host (int .Values.postgres.port) .Values.postgres.auth.database -}} +{{- end -}} +{{- else -}} +{{- .Values.server.dbUrl -}} +{{- end -}} +{{- end }} + {{/* gRPC endpoint sandbox pods use to call back into the gateway. An explicit .Values.server.grpcEndpoint is used verbatim. Otherwise it is derived from diff --git a/deploy/helm/openshell/templates/db-secret.yaml b/deploy/helm/openshell/templates/db-secret.yaml new file mode 100644 index 000000000..2c1c21f1c --- /dev/null +++ b/deploy/helm/openshell/templates/db-secret.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if .Values.postgres.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "openshell.fullname" . }}-db + labels: + {{- include "openshell.labels" . | nindent 4 }} +type: Opaque +stringData: + db-url: {{ include "openshell.dbUrl" . | quote }} +{{- end }} diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index bd74664c5..6a8cbca83 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -8,7 +8,8 @@ at startup. CLI flags and OPENSHELL_* env vars on the StatefulSet container still override anything in this file. One value is intentionally NOT rendered here: - - server.dbUrl → passed via --db-url in the StatefulSet args + - server.dbUrl → passed via OPENSHELL_DB_URL env var (from Secret) + when postgres.enabled=true, or --db-url arg for SQLite */}} apiVersion: v1 kind: ConfigMap diff --git a/deploy/helm/openshell/templates/statefulset.yaml b/deploy/helm/openshell/templates/statefulset.yaml index 5dd4f1caf..f1a0e8a67 100644 --- a/deploy/helm/openshell/templates/statefulset.yaml +++ b/deploy/helm/openshell/templates/statefulset.yaml @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - +{{- if and .Values.postgres.deploy (not .Values.postgres.enabled) }} +{{- fail "postgres.deploy=true requires postgres.enabled=true" }} +{{- end }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -21,6 +23,9 @@ spec: # without this annotation a `helm upgrade` that only mutates the # ConfigMap would leave pods running with stale config. checksum/gateway-config: {{ include (print $.Template.BasePath "/gateway-config.yaml") . | sha256sum }} + {{- if .Values.postgres.enabled }} + checksum/db-secret: {{ include (print $.Template.BasePath "/db-secret.yaml") . | sha256sum }} + {{- end }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -54,9 +59,18 @@ spec: args: - --config - /etc/openshell/gateway.toml + {{- if not .Values.postgres.enabled }} - --db-url - {{ .Values.server.dbUrl | quote }} + {{- end }} env: + {{- if .Values.postgres.enabled }} + - name: OPENSHELL_DB_URL + valueFrom: + secretKeyRef: + name: {{ include "openshell.fullname" . }}-db + key: db-url + {{- end }} # All gateway settings live in the ConfigMap-backed TOML file # mounted at /etc/openshell/gateway.toml. The only env var below # is a process-level setting consumed by libraries outside @@ -137,7 +151,7 @@ spec: - name: sandbox-jwt secret: secretName: {{ .Values.server.sandboxJwt.signingSecretName | default (printf "%s-jwt-keys" (include "openshell.fullname" .)) }} - defaultMode: 0400 + defaultMode: {{ .Values.server.sandboxJwt.secretDefaultMode | default 0400 }} {{- if not .Values.server.disableTls }} - name: tls-cert secret: diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index f17203c6f..d282370ee 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -5,6 +5,7 @@ suite: gateway TOML config shape templates: - templates/gateway-config.yaml - templates/statefulset.yaml + - templates/db-secret.yaml release: name: openshell namespace: my-namespace @@ -125,3 +126,124 @@ tests: - matchRegex: path: data["gateway.toml"] pattern: 'server_sans\s*=\s*\["openshell", "\*\.dev\.openshell\.localhost"\]' + + - it: passes sqlite db-url via --db-url arg by default + template: templates/statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: "sqlite:/var/openshell/openshell.db" + + - it: does not create a db Secret when postgres is disabled + template: templates/db-secret.yaml + asserts: + - hasDocuments: + count: 0 + + - it: does not pass --db-url in args when postgres is enabled + template: templates/statefulset.yaml + set: + postgres.enabled: true + postgres.deploy: true + postgres.auth.password: test-pw + asserts: + - notContains: + path: spec.template.spec.containers[0].args + content: "--db-url" + + - it: injects OPENSHELL_DB_URL from Secret when postgres is enabled + template: templates/statefulset.yaml + set: + postgres.enabled: true + postgres.deploy: true + postgres.auth.password: test-pw + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_DB_URL + valueFrom: + secretKeyRef: + name: openshell-db + key: db-url + + - it: creates db Secret with internal postgres URL + template: templates/db-secret.yaml + set: + postgres.enabled: true + postgres.deploy: true + postgres.auth.password: test-pw + asserts: + - isKind: + of: Secret + - equal: + path: stringData["db-url"] + value: "postgres://openshell:test-pw@openshell-postgres.my-namespace.svc.cluster.local:5432/openshell" + + - it: creates db Secret with external postgres URL fields + template: templates/db-secret.yaml + set: + postgres.enabled: true + postgres.deploy: false + postgres.external.host: external-postgres.example.com + postgres.external.port: 5432 + postgres.external.database: openshell_ext + postgres.external.username: ext_user + postgres.external.password: ext_pass + asserts: + - isKind: + of: Secret + - equal: + path: stringData["db-url"] + value: "postgres://ext_user:ext_pass@external-postgres.example.com:5432/openshell_ext" + + - it: uses external.url verbatim when provided + template: templates/db-secret.yaml + set: + postgres.enabled: true + postgres.deploy: false + postgres.external.url: "postgres://custom:secret@my-host:5433/mydb?sslmode=require" + asserts: + - isKind: + of: Secret + - equal: + path: stringData["db-url"] + value: "postgres://custom:secret@my-host:5433/mydb?sslmode=require" + + - it: URL-encodes special characters in credentials + template: templates/db-secret.yaml + set: + postgres.enabled: true + postgres.deploy: true + postgres.auth.password: "p@ss:word" + asserts: + - equal: + path: stringData["db-url"] + value: "postgres://openshell:p%40ss%3Aword@openshell-postgres.my-namespace.svc.cluster.local:5432/openshell" + + - it: fails when postgres is enabled but no password is set + template: templates/statefulset.yaml + set: + postgres.enabled: true + postgres.deploy: true + asserts: + - failedTemplate: + errorMessage: "postgres.auth.password must be set when postgres.enabled=true" + + - it: fails when postgres.deploy=true but postgres.enabled=false + template: templates/statefulset.yaml + set: + postgres.deploy: true + asserts: + - failedTemplate: + errorMessage: "postgres.deploy=true requires postgres.enabled=true" + + - it: fails when external postgres has no password + template: templates/statefulset.yaml + set: + postgres.enabled: true + postgres.deploy: false + postgres.external.host: my-host.example.com + asserts: + - failedTemplate: + errorMessage: "postgres.external.password is required when postgres.deploy=false and no postgres.external.url is provided" diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 2d707168c..3e551a033 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -206,6 +206,10 @@ server: # values outside [600, 86400]. Default 3600 — generous, since the # supervisor consumes the token within seconds of pod start. k8sSaTokenTtlSecs: 3600 + # -- File mode for the mounted JWT signing key Secret. Default 0400 + # (owner-read only). Override to 0440 or 0444 if the container UID + # does not match the volume file owner. + secretDefaultMode: "" # OIDC (OpenID Connect) configuration for JWT-based authentication. # When issuer is set, the server validates Bearer tokens on gRPC requests. oidc: @@ -231,6 +235,41 @@ server: # issuer uses a non-public CA (e.g. OpenShift ingress, private PKI). caConfigMapName: "" +# Optional PostgreSQL backing store. +# - enabled=false (default): gateway uses server.dbUrl (SQLite by default). +# - enabled=true + deploy=true: deploy bundled PostgreSQL subchart and derive dbUrl. +# - enabled=true + deploy=false: derive dbUrl from postgres.external.* (or external.url). +postgres: + enabled: false + # -- Deploy the bundled Bitnami PostgreSQL subchart. Set to true to + # run PostgreSQL alongside the gateway. Leave false when using an + # external PostgreSQL instance. + deploy: false + # Internal host override. Leave empty to use: + # -postgres..svc.cluster.local + host: "" + port: 5432 + # External mode connection settings. + external: + # Full URL override, e.g. postgres://user:pass@host:5432/db + url: "" + host: "" + port: 5432 + database: openshell + username: openshell + password: "" + # Values below also configure the bundled Bitnami PostgreSQL subchart + # (aliased as "postgres" in Chart.yaml). The subchart uses these to + # initialise the PostgreSQL instance; the gateway uses them to compose + # the connection URL. They must stay in sync. + auth: + username: openshell + password: "" + database: openshell + primary: + persistence: + enabled: true + # NetworkPolicy restricting SSH ingress on sandbox pods to the gateway only. networkPolicy: # -- Create a NetworkPolicy restricting SSH ingress on sandbox pods to the gateway. diff --git a/tasks/helm.toml b/tasks/helm.toml index 31788088a..f25dadb09 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -26,6 +26,7 @@ hide = true description = "Lint the openshell Helm chart (defaults + all CI configuration variants)" run = """ set -e + helm dependency build deploy/helm/openshell echo "--- helm lint: defaults ---" echo "values files: deploy/helm/openshell/values.yaml" helm lint deploy/helm/openshell @@ -45,6 +46,7 @@ run = """ if ! helm plugin list | grep -q unittest; then helm plugin install https://github.com/helm-unittest/helm-unittest --verify=false fi + helm dependency build deploy/helm/openshell helm unittest deploy/helm/openshell """ From 9a5436021d68fb79c6149ea1ba99230a04fe1b80 Mon Sep 17 00:00:00 2001 From: sauagarwa Date: Wed, 27 May 2026 20:13:28 -0400 Subject: [PATCH 2/6] fix(helm): add database backend docs to README.md.gotmpl and regenerate The helm-docs CI check failed because the Database backend section was added directly to README.md instead of README.md.gotmpl. Move the content to the template and regenerate so the check passes. --- deploy/helm/openshell/README.md | 20 +++++++- deploy/helm/openshell/README.md.gotmpl | 67 ++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 0d8e1bf02..8b47eb9ac 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -32,8 +32,9 @@ oc create ns openshell # Sandboxes are deployed into the openshell namespace and use the openshell-sandbox service account oc adm policy add-scc-to-user privileged -z openshell-sandbox -n openshell -# Deploy openshell with overrides for OpenShift SCC compatibility +# Deploy openshell with overrides to allow SCC assignment of fsGroup and runAsUser for the gateway helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ + --set pkiInitJob.enabled=false \ --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null @@ -77,7 +78,7 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ @@ -177,6 +178,20 @@ cert-manager alternative. | podLabels | object | `{}` | Extra labels to add to the gateway pod. | | podLifecycle.terminationGracePeriodSeconds | int | `5` | Grace period, in seconds, before Kubernetes terminates the gateway pod. | | podSecurityContext.fsGroup | int | `1000` | fsGroup assigned to the gateway pod. | +| postgres.auth.database | string | `"openshell"` | | +| postgres.auth.password | string | `""` | | +| postgres.auth.username | string | `"openshell"` | | +| postgres.deploy | bool | `false` | Deploy the bundled Bitnami PostgreSQL subchart. Set to true to run PostgreSQL alongside the gateway. Leave false when using an external PostgreSQL instance. | +| postgres.enabled | bool | `false` | | +| postgres.external.database | string | `"openshell"` | | +| postgres.external.host | string | `""` | | +| postgres.external.password | string | `""` | | +| postgres.external.port | int | `5432` | | +| postgres.external.url | string | `""` | | +| postgres.external.username | string | `"openshell"` | | +| postgres.host | string | `""` | | +| postgres.port | int | `5432` | | +| postgres.primary.persistence.enabled | bool | `true` | | | probes.liveness.failureThreshold | int | `3` | Liveness probe failure threshold before the container is restarted. | | probes.liveness.initialDelaySeconds | int | `2` | Liveness probe initial delay, in seconds. | | probes.liveness.periodSeconds | int | `5` | Liveness probe period, in seconds. | @@ -217,6 +232,7 @@ cert-manager alternative. | server.sandboxImagePullPolicy | string | `""` | Kubernetes imagePullPolicy for sandbox pods. Empty = Kubernetes default (Always for :latest, IfNotPresent otherwise). Set to "Always" for dev clusters so new images are picked up without manual eviction. | | server.sandboxJwt.gatewayId | string | `""` | Stable gateway identity embedded in iss/aud of every minted token. Defaults to the release name so HA replicas share identity. | | server.sandboxJwt.k8sSaTokenTtlSecs | int | `3600` | Lifetime (seconds) of the projected ServiceAccount token kubelet writes into each sandbox pod for the IssueSandboxToken bootstrap exchange. Kubelet enforces a minimum of 600s; the driver clamps values outside [600, 86400]. Default 3600 — generous, since the supervisor consumes the token within seconds of pod start. | +| server.sandboxJwt.secretDefaultMode | string | `""` | File mode for the mounted JWT signing key Secret. Default 0400 (owner-read only). Override to 0440 or 0444 if the container UID does not match the volume file owner. | | server.sandboxJwt.signingSecretName | string | `""` | Name of the Opaque Secret holding the signing key material. Empty falls back to the chart fullname with "-jwt-keys" appended. | | server.sandboxJwt.ttlSecs | int | `3600` | Token TTL in seconds. Defaults to 3600 (1h). | | server.sandboxNamespace | string | `""` | Namespace where sandbox pods are created. Defaults to the Helm release namespace (.Release.Namespace) when left empty. | diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index 9e6a0ec65..b39a5ab71 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -58,6 +58,73 @@ See [`values.yaml`](values.yaml) for source defaults. Selected overlays: - [`ci/values-cert-manager.yaml`](ci/values-cert-manager.yaml) - cert-manager integration - [`ci/values-keycloak.yaml`](ci/values-keycloak.yaml) - Keycloak OIDC integration +### Database backend + +By default, OpenShell uses SQLite: + +```yaml +server: + dbUrl: "sqlite:/var/openshell/openshell.db" +postgres: + enabled: false +``` + +Enable bundled PostgreSQL: + +```bash +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set postgres.enabled=true \ + --set postgres.deploy=true \ + --set postgres.auth.password=my-secret-password +``` + +Enable bundled PostgreSQL (OpenShift): + +```bash +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set postgres.enabled=true \ + --set postgres.deploy=true \ + --set postgres.auth.password=my-secret-password \ + --set server.disableTls=true \ + --set podSecurityContext.fsGroup=null \ + --set securityContext.runAsUser=null +``` + +Use external PostgreSQL: + +```bash +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set postgres.enabled=true \ + --set postgres.external.host=my-postgres.example.com \ + --set postgres.external.port=5432 \ + --set postgres.external.database=openshell \ + --set postgres.external.username=openshell \ + --set postgres.external.password=my-password +``` + +Use external PostgreSQL (OpenShift): + +```bash +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set postgres.enabled=true \ + --set postgres.external.host=my-postgres.example.com \ + --set postgres.external.port=5432 \ + --set postgres.external.database=openshell \ + --set postgres.external.username=openshell \ + --set postgres.external.password=my-password \ + --set server.disableTls=true \ + --set podSecurityContext.fsGroup=null \ + --set securityContext.runAsUser=null +``` + +Or provide a full connection URL directly: + +```bash +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set postgres.enabled=true \ + --set postgres.external.url="postgres://user:pass@host:5432/db?sslmode=require" +``` + ## PKI bootstrap By default, a pre-install/pre-upgrade hook Job runs `openshell-gateway generate-certs` From 460ca48a7888c1d00f194b5346fbc39296aff075 Mon Sep 17 00:00:00 2001 From: sauagarwa Date: Thu, 28 May 2026 00:20:16 -0400 Subject: [PATCH 3/6] fix(helm): use Secret-based DB credentials and support existingSecret Replace the inline db-url stringData pattern with a proper Secret containing individual fields plus a uri key. When postgres.deploy=true the Bitnami service-binding secret is referenced directly; when deploy=false users can supply postgres.external.existingSecret to bring their own Secret, or let the chart generate one from the external field values. Also restructures the README database section for clarity, adds helm-unittest coverage for the new secret resolution paths, and fixes a markdown lint issue in the root README. --- deploy/helm/openshell/README.md | 74 +++++-- deploy/helm/openshell/README.md.gotmpl | 69 ++++-- .../openshell/ci/test-openshift-scenarios.sh | 199 ++++++++++++++++++ deploy/helm/openshell/templates/_helpers.tpl | 40 ++-- .../helm/openshell/templates/db-secret.yaml | 22 +- .../helm/openshell/templates/statefulset.yaml | 6 +- .../openshell/tests/gateway_config_test.yaml | 149 +++++++++---- deploy/helm/openshell/values.yaml | 21 +- 8 files changed, 474 insertions(+), 106 deletions(-) create mode 100755 deploy/helm/openshell/ci/test-openshift-scenarios.sh diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 8b47eb9ac..2c0db85fd 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -34,7 +34,6 @@ oc adm policy add-scc-to-user privileged -z openshell-sandbox -n openshell # Deploy openshell with overrides to allow SCC assignment of fsGroup and runAsUser for the gateway helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ - --set pkiInitJob.enabled=false \ --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null @@ -69,28 +68,52 @@ postgres: enabled: false ``` -Enable bundled PostgreSQL: +#### Use an existing Kubernetes Secret + +If you already have a Secret containing PostgreSQL credentials (e.g. managed +via GitOps or external-secrets-operator), point the chart at it directly: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ --set postgres.enabled=true \ - --set postgres.deploy=true \ - --set postgres.auth.password=my-secret-password + --set postgres.external.existingSecret=my-pg-credentials ``` -Enable bundled PostgreSQL (OpenShift): +On OpenShift, append the platform overrides: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ --set postgres.enabled=true \ - --set postgres.deploy=true \ - --set postgres.auth.password=my-secret-password \ + --set postgres.external.existingSecret=my-pg-credentials \ --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null ``` -Use external PostgreSQL: +The Secret must contain a `uri` key with the full connection string: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-pg-credentials +type: Opaque +data: + uri: # postgresql://user:pass@host:5432/dbname +``` + +#### Kubernetes + +Enable bundled PostgreSQL: + +```bash +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set postgres.enabled=true \ + --set postgres.deploy=true \ + --set postgres.auth.password=my-secret-password +``` + +Use external PostgreSQL (chart creates the Secret from fields): ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ @@ -102,27 +125,41 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ --set postgres.enabled=true \ - --set postgres.external.host=my-postgres.example.com \ - --set postgres.external.port=5432 \ - --set postgres.external.database=openshell \ - --set postgres.external.username=openshell \ - --set postgres.external.password=my-password \ + --set postgres.deploy=true \ + --set postgres.auth.password=my-secret-password \ --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null ``` -Or provide a full connection URL directly: +Use external PostgreSQL on OpenShift: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ --set postgres.enabled=true \ - --set postgres.external.url="postgres://user:pass@host:5432/db?sslmode=require" + --set postgres.external.host=my-postgres.example.com \ + --set postgres.external.port=5432 \ + --set postgres.external.database=openshell \ + --set postgres.external.username=openshell \ + --set postgres.external.password=my-password \ + --set server.disableTls=true \ + --set podSecurityContext.fsGroup=null \ + --set securityContext.runAsUser=null ``` ## PKI bootstrap @@ -184,14 +221,13 @@ cert-manager alternative. | postgres.deploy | bool | `false` | Deploy the bundled Bitnami PostgreSQL subchart. Set to true to run PostgreSQL alongside the gateway. Leave false when using an external PostgreSQL instance. | | postgres.enabled | bool | `false` | | | postgres.external.database | string | `"openshell"` | | +| postgres.external.existingSecret | string | `""` | Name of a pre-existing Opaque Secret containing PostgreSQL credentials. When set, the chart does not create its own db Secret and reads directly from this one. The Secret must contain a `uri` key with the full connection string, e.g. postgresql://user:pass@host:5432/dbname. | | postgres.external.host | string | `""` | | | postgres.external.password | string | `""` | | | postgres.external.port | int | `5432` | | -| postgres.external.url | string | `""` | | | postgres.external.username | string | `"openshell"` | | -| postgres.host | string | `""` | | -| postgres.port | int | `5432` | | | postgres.primary.persistence.enabled | bool | `true` | | +| postgres.serviceBindings.enabled | bool | `true` | | | probes.liveness.failureThreshold | int | `3` | Liveness probe failure threshold before the container is restarted. | | probes.liveness.initialDelaySeconds | int | `2` | Liveness probe initial delay, in seconds. | | probes.liveness.periodSeconds | int | `5` | Liveness probe period, in seconds. | diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index b39a5ab71..6c9733fbc 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -34,7 +34,6 @@ oc adm policy add-scc-to-user privileged -z openshell-sandbox -n openshell # Deploy openshell with overrides to allow SCC assignment of fsGroup and runAsUser for the gateway helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ - --set pkiInitJob.enabled=false \ --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null @@ -69,28 +68,52 @@ postgres: enabled: false ``` -Enable bundled PostgreSQL: +#### Use an existing Kubernetes Secret + +If you already have a Secret containing PostgreSQL credentials (e.g. managed +via GitOps or external-secrets-operator), point the chart at it directly: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ --set postgres.enabled=true \ - --set postgres.deploy=true \ - --set postgres.auth.password=my-secret-password + --set postgres.external.existingSecret=my-pg-credentials ``` -Enable bundled PostgreSQL (OpenShift): +On OpenShift, append the platform overrides: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ --set postgres.enabled=true \ - --set postgres.deploy=true \ - --set postgres.auth.password=my-secret-password \ + --set postgres.external.existingSecret=my-pg-credentials \ --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null ``` -Use external PostgreSQL: +The Secret must contain a `uri` key with the full connection string: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-pg-credentials +type: Opaque +data: + uri: # postgresql://user:pass@host:5432/dbname +``` + +#### Kubernetes + +Enable bundled PostgreSQL: + +```bash +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set postgres.enabled=true \ + --set postgres.deploy=true \ + --set postgres.auth.password=my-secret-password +``` + +Use external PostgreSQL (chart creates the Secret from fields): ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ @@ -102,27 +125,41 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ --set postgres.enabled=true \ - --set postgres.external.host=my-postgres.example.com \ - --set postgres.external.port=5432 \ - --set postgres.external.database=openshell \ - --set postgres.external.username=openshell \ - --set postgres.external.password=my-password \ + --set postgres.deploy=true \ + --set postgres.auth.password=my-secret-password \ --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null ``` -Or provide a full connection URL directly: +Use external PostgreSQL on OpenShift: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ --set postgres.enabled=true \ - --set postgres.external.url="postgres://user:pass@host:5432/db?sslmode=require" + --set postgres.external.host=my-postgres.example.com \ + --set postgres.external.port=5432 \ + --set postgres.external.database=openshell \ + --set postgres.external.username=openshell \ + --set postgres.external.password=my-password \ + --set server.disableTls=true \ + --set podSecurityContext.fsGroup=null \ + --set securityContext.runAsUser=null ``` ## PKI bootstrap diff --git a/deploy/helm/openshell/ci/test-openshift-scenarios.sh b/deploy/helm/openshell/ci/test-openshift-scenarios.sh new file mode 100755 index 000000000..5ff54fbd7 --- /dev/null +++ b/deploy/helm/openshell/ci/test-openshift-scenarios.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Validates all OpenShift database-backend scenarios against a live cluster. +# +# Prerequisites: +# - oc CLI authenticated to an OpenShift cluster +# - helm 3.x installed +# - Chart dependencies built (helm dependency build deploy/helm/openshell) +# +# Usage: +# ./deploy/helm/openshell/ci/test-openshift-scenarios.sh [--chart-path ./deploy/helm/openshell] [--image-tag dev] + +set -euo pipefail + +CHART_PATH="${CHART_PATH:-./deploy/helm/openshell}" +NAMESPACE="openshell" +RELEASE="openshell" +IMAGE_TAG="${IMAGE_TAG:-dev}" +WAIT_TIMEOUT="120s" +PASSED=0 +FAILED=0 +SCENARIOS=() + +while [[ $# -gt 0 ]]; do + case $1 in + --chart-path) CHART_PATH="$2"; shift 2 ;; + --image-tag) IMAGE_TAG="$2"; shift 2 ;; + --namespace) NAMESPACE="$2"; shift 2 ;; + *) echo "Unknown option: $1" >&2; exit 1 ;; + esac +done + +# --- helpers ---------------------------------------------------------------- + +log() { echo "==> $*"; } +pass() { log "PASS: $1"; PASSED=$((PASSED + 1)); SCENARIOS+=("PASS $1"); } +fail() { log "FAIL: $1 — $2"; FAILED=$((FAILED + 1)); SCENARIOS+=("FAIL $1: $2"); } + +wait_for_ready() { + local label="$1" timeout="$2" + if oc wait pod -n "$NAMESPACE" -l "$label" --for=condition=Ready --timeout="$timeout" 2>/dev/null; then + return 0 + fi + return 1 +} + +cleanup_release() { + log "Cleaning up release $RELEASE" + helm uninstall "$RELEASE" -n "$NAMESPACE" --wait 2>/dev/null || true + # Wait for pods to terminate + for i in $(seq 1 30); do + if [ -z "$(oc get pods -n "$NAMESPACE" -l "app.kubernetes.io/instance=$RELEASE" --no-headers 2>/dev/null)" ]; then + break + fi + sleep 2 + done + # Clean up PVCs left by StatefulSets + oc delete pvc -n "$NAMESPACE" -l "app.kubernetes.io/instance=$RELEASE" --wait=false 2>/dev/null || true +} + +verify_gateway() { + local scenario="$1" + if wait_for_ready "app.kubernetes.io/name=openshell,app.kubernetes.io/instance=$RELEASE" "$WAIT_TIMEOUT"; then + # Check the pod is actually running (not CrashLoopBackOff) + local phase + phase=$(oc get pod -n "$NAMESPACE" -l "app.kubernetes.io/name=openshell,app.kubernetes.io/instance=$RELEASE" \ + -o jsonpath='{.items[0].status.phase}' 2>/dev/null) + if [ "$phase" = "Running" ]; then + pass "$scenario" + else + fail "$scenario" "pod phase is $phase, expected Running" + fi + else + local status + status=$(oc get pods -n "$NAMESPACE" -l "app.kubernetes.io/name=openshell" --no-headers 2>/dev/null || echo "no pods found") + fail "$scenario" "gateway pod not ready within $WAIT_TIMEOUT ($status)" + fi +} + +# --- setup ------------------------------------------------------------------ + +log "Setting up namespace $NAMESPACE" +oc create ns "$NAMESPACE" 2>/dev/null || true +oc adm policy add-scc-to-user privileged -z "${RELEASE}-sandbox" -n "$NAMESPACE" + +OPENSHIFT_FLAGS=( + --set server.disableTls=true + --set podSecurityContext.fsGroup=null + --set securityContext.runAsUser=null + --set image.tag="$IMAGE_TAG" +) + +# --- scenario 1: SQLite (default, no postgres) ----------------------------- + +SCENARIO="SQLite (default)" +log "Testing: $SCENARIO" +cleanup_release + +helm install "$RELEASE" "$CHART_PATH" -n "$NAMESPACE" \ + "${OPENSHIFT_FLAGS[@]}" + +verify_gateway "$SCENARIO" +cleanup_release + +# --- scenario 2: Bundled PostgreSQL (deploy=true) --------------------------- + +SCENARIO="Bundled PostgreSQL (deploy=true)" +log "Testing: $SCENARIO" +cleanup_release + +helm install "$RELEASE" "$CHART_PATH" -n "$NAMESPACE" \ + "${OPENSHIFT_FLAGS[@]}" \ + --set postgres.enabled=true \ + --set postgres.deploy=true \ + --set postgres.auth.password=test-password + +# Wait for postgres to be ready first +log "Waiting for bundled PostgreSQL..." +wait_for_ready "app.kubernetes.io/name=postgres,app.kubernetes.io/instance=$RELEASE" "$WAIT_TIMEOUT" || true + +verify_gateway "$SCENARIO" +cleanup_release + +# --- scenario 3: External PostgreSQL with existing Secret ------------------- + +SCENARIO="External PostgreSQL (existingSecret)" +log "Testing: $SCENARIO" +cleanup_release + +# Deploy a standalone Bitnami PostgreSQL as the "external" database +EXTERNAL_PG_RELEASE="pg-external" +EXTERNAL_PG_PASSWORD="ext-test-password" +EXTERNAL_PG_DATABASE="openshell" +EXTERNAL_PG_USERNAME="openshell" + +log "Deploying standalone PostgreSQL as external database..." +helm install "$EXTERNAL_PG_RELEASE" oci://registry-1.docker.io/bitnamicharts/postgresql \ + -n "$NAMESPACE" \ + --set auth.username="$EXTERNAL_PG_USERNAME" \ + --set auth.password="$EXTERNAL_PG_PASSWORD" \ + --set auth.database="$EXTERNAL_PG_DATABASE" \ + --set primary.podSecurityContext.fsGroup=null \ + --set primary.containerSecurityContext.runAsUser=null \ + --wait --timeout "$WAIT_TIMEOUT" 2>/dev/null || true + +wait_for_ready "app.kubernetes.io/name=postgresql,app.kubernetes.io/instance=$EXTERNAL_PG_RELEASE" "$WAIT_TIMEOUT" || true + +EXTERNAL_PG_HOST="${EXTERNAL_PG_RELEASE}-postgresql.${NAMESPACE}.svc.cluster.local" +EXTERNAL_PG_URI="postgresql://${EXTERNAL_PG_USERNAME}:${EXTERNAL_PG_PASSWORD}@${EXTERNAL_PG_HOST}:5432/${EXTERNAL_PG_DATABASE}" + +# Create the existing Secret with the expected keys +log "Creating existing Secret with PostgreSQL credentials..." +oc create secret generic my-pg-credentials -n "$NAMESPACE" \ + --from-literal=host="$EXTERNAL_PG_HOST" \ + --from-literal=port="5432" \ + --from-literal=username="$EXTERNAL_PG_USERNAME" \ + --from-literal=password="$EXTERNAL_PG_PASSWORD" \ + --from-literal=database="$EXTERNAL_PG_DATABASE" \ + --from-literal=uri="$EXTERNAL_PG_URI" \ + 2>/dev/null || true + +# Install OpenShell pointing at the existing Secret +helm install "$RELEASE" "$CHART_PATH" -n "$NAMESPACE" \ + "${OPENSHIFT_FLAGS[@]}" \ + --set postgres.enabled=true \ + --set postgres.external.existingSecret=my-pg-credentials + +verify_gateway "$SCENARIO" + +# Cleanup external postgres and secret +cleanup_release +helm uninstall "$EXTERNAL_PG_RELEASE" -n "$NAMESPACE" --wait 2>/dev/null || true +oc delete secret my-pg-credentials -n "$NAMESPACE" 2>/dev/null || true +oc delete pvc -n "$NAMESPACE" -l "app.kubernetes.io/instance=$EXTERNAL_PG_RELEASE" --wait=false 2>/dev/null || true + +# --- teardown --------------------------------------------------------------- + +log "Removing SCC binding and namespace" +oc adm policy remove-scc-from-user privileged -z "${RELEASE}-sandbox" -n "$NAMESPACE" 2>/dev/null || true +oc delete ns "$NAMESPACE" --wait=false 2>/dev/null || true + +# --- summary ---------------------------------------------------------------- + +echo "" +echo "========================================" +echo " Test Summary" +echo "========================================" +for s in "${SCENARIOS[@]}"; do + echo " $s" +done +echo "----------------------------------------" +echo " Passed: $PASSED Failed: $FAILED" +echo "========================================" + +if [ "$FAILED" -gt 0 ]; then + exit 1 +fi diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 5f1ca066d..5d2b2803c 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -103,28 +103,34 @@ Namespace where sandbox pods are created. An explicit {{- end }} {{/* -Gateway database URL. -- postgres.enabled=false: use .Values.server.dbUrl (default sqlite) -- postgres.enabled=true + deploy=true: derive URL from bundled postgres subchart -- postgres.enabled=true + deploy=false: use external.url or compose external fields +Fully qualified name of the PostgreSQL subchart, mirroring the Bitnami +common.names.fullname template so we stay in sync when users set +postgres.fullnameOverride or postgres.nameOverride. */}} -{{- define "openshell.dbUrl" -}} -{{- if .Values.postgres.enabled -}} -{{- if not .Values.postgres.deploy -}} -{{- if .Values.postgres.external.url -}} -{{- .Values.postgres.external.url -}} +{{- define "openshell.postgresFullname" -}} +{{- if .Values.postgres.fullnameOverride -}} +{{- .Values.postgres.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} -{{- $host := required "postgres.external.host is required when postgres.deploy=false and no postgres.external.url is provided" .Values.postgres.external.host -}} -{{- $pw := required "postgres.external.password is required when postgres.deploy=false and no postgres.external.url is provided" .Values.postgres.external.password -}} -{{- printf "postgres://%s:%s@%s:%d/%s" (.Values.postgres.external.username | urlquery) ($pw | urlquery) $host (int (default 5432 .Values.postgres.external.port)) .Values.postgres.external.database -}} -{{- end -}} +{{- $name := default "postgres" .Values.postgres.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} {{- else -}} -{{- $pw := required "postgres.auth.password must be set when postgres.enabled=true" .Values.postgres.auth.password -}} -{{- $host := .Values.postgres.host | default (printf "%s-postgres.%s.svc.cluster.local" .Release.Name .Release.Namespace) -}} -{{- printf "postgres://%s:%s@%s:%d/%s" (.Values.postgres.auth.username | urlquery) ($pw | urlquery) $host (int .Values.postgres.port) .Values.postgres.auth.database -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} {{- end -}} +{{- end }} + +{{/* +Name of the Secret holding the PostgreSQL connection URI. +- deploy=true: derive from Bitnami service-binding naming convention +- deploy=false + existingSecret set: use it verbatim +- deploy=false + no existingSecret: use chart-generated "-db" +*/}} +{{- define "openshell.dbSecretName" -}} +{{- if .Values.postgres.deploy -}} +{{- printf "%s-svcbind-custom-user" (include "openshell.postgresFullname" .) -}} {{- else -}} -{{- .Values.server.dbUrl -}} +{{- .Values.postgres.external.existingSecret | default (printf "%s-db" (include "openshell.fullname" .)) -}} {{- end -}} {{- end }} diff --git a/deploy/helm/openshell/templates/db-secret.yaml b/deploy/helm/openshell/templates/db-secret.yaml index 2c1c21f1c..b7e5bb0e3 100644 --- a/deploy/helm/openshell/templates/db-secret.yaml +++ b/deploy/helm/openshell/templates/db-secret.yaml @@ -1,14 +1,28 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if .Values.postgres.enabled }} +{{/* +Chart-managed db Secret for external PostgreSQL only. +When postgres.deploy=true the Bitnami subchart creates the service-binding +secret (with uri key) via postgres.serviceBindings.enabled=true. +When postgres.external.existingSecret is set the user brings their own Secret. +*/}} +{{- if and .Values.postgres.enabled (not .Values.postgres.deploy) (not .Values.postgres.external.existingSecret) }} +{{- $host := required "postgres.external.host is required when postgres.deploy=false and no postgres.external.existingSecret is provided" .Values.postgres.external.host }} +{{- $pw := required "postgres.external.password is required when postgres.deploy=false and no postgres.external.existingSecret is provided" .Values.postgres.external.password }} +{{- $port := toString (int (default 5432 .Values.postgres.external.port)) }} apiVersion: v1 kind: Secret metadata: - name: {{ include "openshell.fullname" . }}-db + name: {{ include "openshell.dbSecretName" . }} labels: {{- include "openshell.labels" . | nindent 4 }} type: Opaque -stringData: - db-url: {{ include "openshell.dbUrl" . | quote }} +data: + host: {{ $host | b64enc | quote }} + port: {{ $port | b64enc | quote }} + username: {{ .Values.postgres.external.username | b64enc | quote }} + password: {{ $pw | b64enc | quote }} + database: {{ .Values.postgres.external.database | b64enc | quote }} + uri: {{ printf "postgresql://%s:%s@%s:%s/%s" (.Values.postgres.external.username | urlquery) ($pw | urlquery) $host $port .Values.postgres.external.database | b64enc | quote }} {{- end }} diff --git a/deploy/helm/openshell/templates/statefulset.yaml b/deploy/helm/openshell/templates/statefulset.yaml index f1a0e8a67..53c3c4320 100644 --- a/deploy/helm/openshell/templates/statefulset.yaml +++ b/deploy/helm/openshell/templates/statefulset.yaml @@ -23,7 +23,7 @@ spec: # without this annotation a `helm upgrade` that only mutates the # ConfigMap would leave pods running with stale config. checksum/gateway-config: {{ include (print $.Template.BasePath "/gateway-config.yaml") . | sha256sum }} - {{- if .Values.postgres.enabled }} + {{- if and .Values.postgres.enabled (not .Values.postgres.deploy) (not .Values.postgres.external.existingSecret) }} checksum/db-secret: {{ include (print $.Template.BasePath "/db-secret.yaml") . | sha256sum }} {{- end }} {{- with .Values.podAnnotations }} @@ -68,8 +68,8 @@ spec: - name: OPENSHELL_DB_URL valueFrom: secretKeyRef: - name: {{ include "openshell.fullname" . }}-db - key: db-url + name: {{ include "openshell.dbSecretName" . }} + key: uri {{- end }} # All gateway settings live in the ConfigMap-backed TOML file # mounted at /etc/openshell/gateway.toml. The only env var below diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index d282370ee..8a820c218 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -151,12 +151,63 @@ tests: path: spec.template.spec.containers[0].args content: "--db-url" - - it: injects OPENSHELL_DB_URL from Secret when postgres is enabled + - it: references Bitnami service-binding Secret when postgres.deploy is true template: templates/statefulset.yaml set: postgres.enabled: true postgres.deploy: true postgres.auth.password: test-pw + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_DB_URL + valueFrom: + secretKeyRef: + name: openshell-postgres-svcbind-custom-user + key: uri + + - it: respects postgres.fullnameOverride for bundled service-binding Secret + template: templates/statefulset.yaml + set: + postgres.enabled: true + postgres.deploy: true + postgres.auth.password: test-pw + postgres.fullnameOverride: my-pg + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_DB_URL + valueFrom: + secretKeyRef: + name: my-pg-svcbind-custom-user + key: uri + + - it: respects postgres.nameOverride for bundled service-binding Secret + template: templates/statefulset.yaml + set: + postgres.enabled: true + postgres.deploy: true + postgres.auth.password: test-pw + postgres.nameOverride: pgdb + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_DB_URL + valueFrom: + secretKeyRef: + name: openshell-pgdb-svcbind-custom-user + key: uri + + - it: references chart-created Secret for external postgres + template: templates/statefulset.yaml + set: + postgres.enabled: true + postgres.deploy: false + postgres.external.host: pg.example.com + postgres.external.password: ext-pw asserts: - contains: path: spec.template.spec.containers[0].env @@ -165,22 +216,43 @@ tests: valueFrom: secretKeyRef: name: openshell-db - key: db-url + key: uri - - it: creates db Secret with internal postgres URL + - it: references existing Secret when existingSecret is set + template: templates/statefulset.yaml + set: + postgres.enabled: true + postgres.external.existingSecret: my-pg-secret + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_DB_URL + valueFrom: + secretKeyRef: + name: my-pg-secret + key: uri + + - it: does not create db Secret when postgres.deploy is true template: templates/db-secret.yaml set: postgres.enabled: true postgres.deploy: true postgres.auth.password: test-pw asserts: - - isKind: - of: Secret - - equal: - path: stringData["db-url"] - value: "postgres://openshell:test-pw@openshell-postgres.my-namespace.svc.cluster.local:5432/openshell" + - hasDocuments: + count: 0 + + - it: does not create db Secret when existingSecret is set + template: templates/db-secret.yaml + set: + postgres.enabled: true + postgres.external.existingSecret: my-pg-secret + asserts: + - hasDocuments: + count: 0 - - it: creates db Secret with external postgres URL fields + - it: creates db Secret with individual keys for external postgres template: templates/db-secret.yaml set: postgres.enabled: true @@ -194,41 +266,40 @@ tests: - isKind: of: Secret - equal: - path: stringData["db-url"] - value: "postgres://ext_user:ext_pass@external-postgres.example.com:5432/openshell_ext" - - - it: uses external.url verbatim when provided - template: templates/db-secret.yaml - set: - postgres.enabled: true - postgres.deploy: false - postgres.external.url: "postgres://custom:secret@my-host:5433/mydb?sslmode=require" - asserts: - - isKind: - of: Secret + path: data.host + decodeBase64: true + value: "external-postgres.example.com" + - equal: + path: data.username + decodeBase64: true + value: "ext_user" + - equal: + path: data.password + decodeBase64: true + value: "ext_pass" - equal: - path: stringData["db-url"] - value: "postgres://custom:secret@my-host:5433/mydb?sslmode=require" + path: data.database + decodeBase64: true + value: "openshell_ext" + - equal: + path: data.uri + decodeBase64: true + value: "postgresql://ext_user:ext_pass@external-postgres.example.com:5432/openshell_ext" - - it: URL-encodes special characters in credentials + - it: URL-encodes special characters in external credentials template: templates/db-secret.yaml set: postgres.enabled: true - postgres.deploy: true - postgres.auth.password: "p@ss:word" + postgres.deploy: false + postgres.external.host: pg.example.com + postgres.external.username: "user@corp" + postgres.external.password: "p@ss:word/secret" + postgres.external.database: mydb asserts: - equal: - path: stringData["db-url"] - value: "postgres://openshell:p%40ss%3Aword@openshell-postgres.my-namespace.svc.cluster.local:5432/openshell" - - - it: fails when postgres is enabled but no password is set - template: templates/statefulset.yaml - set: - postgres.enabled: true - postgres.deploy: true - asserts: - - failedTemplate: - errorMessage: "postgres.auth.password must be set when postgres.enabled=true" + path: data.uri + decodeBase64: true + value: "postgresql://user%40corp:p%40ss%3Aword%2Fsecret@pg.example.com:5432/mydb" - it: fails when postgres.deploy=true but postgres.enabled=false template: templates/statefulset.yaml @@ -238,7 +309,7 @@ tests: - failedTemplate: errorMessage: "postgres.deploy=true requires postgres.enabled=true" - - it: fails when external postgres has no password + - it: fails when external postgres has no password and no existingSecret template: templates/statefulset.yaml set: postgres.enabled: true @@ -246,4 +317,4 @@ tests: postgres.external.host: my-host.example.com asserts: - failedTemplate: - errorMessage: "postgres.external.password is required when postgres.deploy=false and no postgres.external.url is provided" + errorMessage: "postgres.external.password is required when postgres.deploy=false and no postgres.external.existingSecret is provided" diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 3e551a033..d4d45389a 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -238,21 +238,22 @@ server: # Optional PostgreSQL backing store. # - enabled=false (default): gateway uses server.dbUrl (SQLite by default). # - enabled=true + deploy=true: deploy bundled PostgreSQL subchart and derive dbUrl. -# - enabled=true + deploy=false: derive dbUrl from postgres.external.* (or external.url). +# - enabled=true + deploy=false: use postgres.external.existingSecret or compose +# from postgres.external.* fields. postgres: enabled: false # -- Deploy the bundled Bitnami PostgreSQL subchart. Set to true to # run PostgreSQL alongside the gateway. Leave false when using an # external PostgreSQL instance. deploy: false - # Internal host override. Leave empty to use: - # -postgres..svc.cluster.local - host: "" - port: 5432 - # External mode connection settings. + # External mode connection settings (used when deploy=false). external: - # Full URL override, e.g. postgres://user:pass@host:5432/db - url: "" + # -- Name of a pre-existing Opaque Secret containing PostgreSQL + # credentials. When set, the chart does not create its own db Secret + # and reads directly from this one. The Secret must contain a `uri` + # key with the full connection string, e.g. + # postgresql://user:pass@host:5432/dbname. + existingSecret: "" host: "" port: 5432 database: openshell @@ -266,6 +267,10 @@ postgres: username: openshell password: "" database: openshell + # Enable Bitnami service-binding Secrets so the gateway can read the + # connection URI directly from the subchart-managed Secret. + serviceBindings: + enabled: true primary: persistence: enabled: true From bbe06af9b6f09e185e316d84f9f193a5ba79b5bd Mon Sep 17 00:00:00 2001 From: sauagarwa Date: Thu, 28 May 2026 08:34:52 -0400 Subject: [PATCH 4/6] refactor(helm): move OpenShift e2e script to e2e/rust/ and add mise task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move test-openshift-scenarios.sh from deploy/helm/openshell/ci/ to e2e/rust/e2e-openshift.sh, matching the existing e2e script naming convention. Register it as `e2e:openshift` in tasks/test.toml — not wired into the `test` or `e2e` aggregates so it only runs on explicit invocation against a live OpenShift cluster. --- .../test-openshift-scenarios.sh => e2e/rust/e2e-openshift.sh | 3 ++- tasks/test.toml | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) rename deploy/helm/openshell/ci/test-openshift-scenarios.sh => e2e/rust/e2e-openshift.sh (98%) diff --git a/deploy/helm/openshell/ci/test-openshift-scenarios.sh b/e2e/rust/e2e-openshift.sh similarity index 98% rename from deploy/helm/openshell/ci/test-openshift-scenarios.sh rename to e2e/rust/e2e-openshift.sh index 5ff54fbd7..60ef886fe 100755 --- a/deploy/helm/openshell/ci/test-openshift-scenarios.sh +++ b/e2e/rust/e2e-openshift.sh @@ -10,7 +10,8 @@ # - Chart dependencies built (helm dependency build deploy/helm/openshell) # # Usage: -# ./deploy/helm/openshell/ci/test-openshift-scenarios.sh [--chart-path ./deploy/helm/openshell] [--image-tag dev] +# mise run e2e:openshift +# e2e/rust/e2e-openshift.sh [--chart-path ./deploy/helm/openshell] [--image-tag dev] set -euo pipefail diff --git a/tasks/test.toml b/tasks/test.toml index 5ccd38e97..6b5c19cbf 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -99,3 +99,7 @@ run = [ description = "Run GPU e2e against a standalone gateway with the Docker compute driver" env = { OPENSHELL_E2E_DOCKER_GPU = "1", OPENSHELL_E2E_DOCKER_TEST = "gpu_device_selection", OPENSHELL_E2E_DOCKER_FEATURES = "e2e-docker-gpu" } run = "e2e/rust/e2e-docker.sh" + +["e2e:openshift"] +description = "Run OpenShift database-backend integration scenarios against a live cluster (requires oc CLI authenticated to an OpenShift cluster)" +run = "e2e/rust/e2e-openshift.sh" From 25ed9f66647ec33b287e017217a8ad3a94cc1a2c Mon Sep 17 00:00:00 2001 From: sauagarwa Date: Thu, 28 May 2026 14:10:40 -0400 Subject: [PATCH 5/6] feat(e2e): add database backend scenarios to Kubernetes e2e MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend with-kube-gateway.sh with an optional multi-scenario loop gated by OPENSHELL_E2E_KUBE_DB_SCENARIOS=1. When enabled, the script installs the Helm chart three times — SQLite (default), bundled PostgreSQL, and external PostgreSQL with existingSecret — running the full test suite against each backend. When unset, existing single-install behavior is unchanged. Also adds helm dependency build before helm install, fixing CI failures caused by the missing PostgreSQL subchart dependency. --- e2e/with-kube-gateway.sh | 409 +++++++++++++++++++++++++++++++-------- tasks/test.toml | 5 + 2 files changed, 337 insertions(+), 77 deletions(-) diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 10d46982c..717f9351d 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -23,6 +23,12 @@ # - Existing-context mode pulls from ${OPENSHELL_REGISTRY}/{gateway,supervisor}:${IMAGE_TAG} # (defaults: ghcr.io/nvidia/openshell, latest). CI sets IMAGE_TAG to the # commit SHA and preloads or publishes the images before running this script. +# +# Database backend scenarios: +# Set OPENSHELL_E2E_KUBE_DB_SCENARIOS=1 to run the test command against +# three database configurations (SQLite, bundled PostgreSQL, external +# PostgreSQL with existingSecret). When unset, the default single-install +# behavior is unchanged. set -euo pipefail @@ -52,6 +58,7 @@ PORTFORWARD_LOG="${WORKDIR}/portforward.log" PORTFORWARD_HEALTH_PID="" PORTFORWARD_HEALTH_LOG="${WORKDIR}/portforward-health.log" HELM_INSTALLED=0 +EXTERNAL_PG_DEPLOYED=0 # Isolate CLI/SDK gateway metadata from the developer's real config. export XDG_CONFIG_HOME="${WORKDIR}/config" @@ -104,6 +111,15 @@ cleanup() { fi fi + if [ "${EXTERNAL_PG_DEPLOYED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ] && [ -n "${NAMESPACE}" ]; then + helmctl uninstall pg-external --namespace "${NAMESPACE}" --wait \ + --timeout 60s >/dev/null 2>&1 || true + kctl -n "${NAMESPACE}" delete secret my-pg-credentials \ + --ignore-not-found >/dev/null 2>&1 || true + kctl delete pvc -n "${NAMESPACE}" \ + -l "app.kubernetes.io/instance=pg-external" --wait=false >/dev/null 2>&1 || true + fi + if [ "${HELM_INSTALLED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ] && [ -n "${NAMESPACE}" ]; then if command -v helm >/dev/null 2>&1; then helmctl uninstall "${RELEASE_NAME}" --namespace "${NAMESPACE}" --wait \ @@ -129,6 +145,209 @@ cleanup() { } trap cleanup EXIT +# --- DB-scenario helpers (used only when OPENSHELL_E2E_KUBE_DB_SCENARIOS=1) --- + +scenario_stop_portforward() { + if [ -n "${PORTFORWARD_PID}" ]; then + kill "${PORTFORWARD_PID}" >/dev/null 2>&1 || true + wait "${PORTFORWARD_PID}" >/dev/null 2>&1 || true + PORTFORWARD_PID="" + fi + if [ -n "${PORTFORWARD_HEALTH_PID}" ]; then + kill "${PORTFORWARD_HEALTH_PID}" >/dev/null 2>&1 || true + wait "${PORTFORWARD_HEALTH_PID}" >/dev/null 2>&1 || true + PORTFORWARD_HEALTH_PID="" + fi +} + +scenario_cleanup_release() { + helmctl uninstall "${RELEASE_NAME}" --namespace "${NAMESPACE}" --wait \ + --timeout 120s 2>/dev/null || true + HELM_INSTALLED=0 + for _ in $(seq 1 30); do + remaining="$(kctl get pods -n "${NAMESPACE}" \ + -l "app.kubernetes.io/instance=${RELEASE_NAME}" --no-headers 2>/dev/null || true)" + if [ -z "${remaining}" ]; then + break + fi + sleep 2 + done + kctl delete pvc -n "${NAMESPACE}" \ + -l "app.kubernetes.io/instance=${RELEASE_NAME}" --wait=false 2>/dev/null || true +} + +scenario_deploy_external_pg() { + local pg_host pg_uri + echo "==> Deploying standalone PostgreSQL as external database..." + helmctl install pg-external oci://registry-1.docker.io/bitnamicharts/postgresql \ + --namespace "${NAMESPACE}" \ + --set auth.username=openshell \ + --set auth.password=ext-test-password \ + --set auth.database=openshell \ + --wait --timeout 120s 2>/dev/null || true + EXTERNAL_PG_DEPLOYED=1 + + kctl -n "${NAMESPACE}" wait pod \ + -l "app.kubernetes.io/name=postgresql,app.kubernetes.io/instance=pg-external" \ + --for=condition=Ready --timeout=120s || true + + pg_host="pg-external-postgresql.${NAMESPACE}.svc.cluster.local" + pg_uri="postgresql://openshell:ext-test-password@${pg_host}:5432/openshell" + + echo "==> Creating Secret with PostgreSQL credentials..." + kctl -n "${NAMESPACE}" create secret generic my-pg-credentials \ + --from-literal=host="${pg_host}" \ + --from-literal=port="5432" \ + --from-literal=username="openshell" \ + --from-literal=password="ext-test-password" \ + --from-literal=database="openshell" \ + --from-literal=uri="${pg_uri}" \ + 2>/dev/null || true +} + +scenario_cleanup_external_pg() { + echo "==> Cleaning up external PostgreSQL..." + helmctl uninstall pg-external --namespace "${NAMESPACE}" --wait \ + --timeout 60s 2>/dev/null || true + kctl -n "${NAMESPACE}" delete secret my-pg-credentials \ + --ignore-not-found >/dev/null 2>&1 || true + kctl delete pvc -n "${NAMESPACE}" \ + -l "app.kubernetes.io/instance=pg-external" --wait=false 2>/dev/null || true + EXTERNAL_PG_DEPLOYED=0 +} + +# Run a single DB-backend scenario: install chart → port-forward → run tests → cleanup. +# Usage: run_scenario "label" "type" [extra --set flags...] +# type: sqlite | bundled-pg | external-pg +run_scenario() { + local scenario_label="$1" scenario_type="$2" + shift 2 + local scenario_exit=0 + + echo "" + echo "========================================" + echo "==> Scenario: ${scenario_label}" + echo "========================================" + + helmctl install "${RELEASE_NAME}" "${ROOT}/deploy/helm/openshell" \ + --namespace "${NAMESPACE}" --create-namespace \ + --values "${ROOT}/deploy/helm/openshell/ci/values-skaffold.yaml" \ + --set "fullnameOverride=openshell" \ + --set "image.repository=${REGISTRY_VALUE}/gateway" \ + --set "image.tag=${IMAGE_TAG_VALUE}" \ + --set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \ + --set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \ + "$@" \ + --wait --timeout 5m + HELM_INSTALLED=1 + + if [ "${scenario_type}" = "bundled-pg" ]; then + echo "Waiting for bundled PostgreSQL to become ready..." + kctl -n "${NAMESPACE}" wait pod \ + -l "app.kubernetes.io/name=postgres,app.kubernetes.io/instance=${RELEASE_NAME}" \ + --for=condition=Ready --timeout=120s || true + fi + + LOCAL_PORT="$(e2e_pick_port)" + echo "Starting kubectl port-forward svc/openshell ${LOCAL_PORT}:8080..." + kctl -n "${NAMESPACE}" port-forward "svc/openshell" \ + "${LOCAL_PORT}:8080" >"${PORTFORWARD_LOG}" 2>&1 & + PORTFORWARD_PID=$! + + local elapsed=0 pf_timeout=30 + while [ "${elapsed}" -lt "${pf_timeout}" ]; do + if ! kill -0 "${PORTFORWARD_PID}" 2>/dev/null; then + echo "ERROR: kubectl port-forward exited before becoming reachable" >&2 + cat "${PORTFORWARD_LOG}" >&2 || true + DB_FAILED=$((DB_FAILED + 1)) + DB_SCENARIOS_SUMMARY+=("FAIL ${scenario_label}: port-forward died") + scenario_stop_portforward + scenario_cleanup_release + return + fi + if curl -s -o /dev/null --connect-timeout 1 "http://127.0.0.1:${LOCAL_PORT}"; then + break + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + if [ "${elapsed}" -ge "${pf_timeout}" ]; then + echo "ERROR: port-forward did not accept TCP within ${pf_timeout}s" >&2 + cat "${PORTFORWARD_LOG}" >&2 || true + DB_FAILED=$((DB_FAILED + 1)) + DB_SCENARIOS_SUMMARY+=("FAIL ${scenario_label}: port-forward timeout") + scenario_stop_portforward + scenario_cleanup_release + return + fi + + HEALTH_LOCAL_PORT="$(e2e_pick_port)" + echo "Starting kubectl port-forward sts/${RELEASE_NAME} ${HEALTH_LOCAL_PORT}:health..." + kctl -n "${NAMESPACE}" port-forward "sts/${RELEASE_NAME}" \ + "${HEALTH_LOCAL_PORT}:health" >"${PORTFORWARD_HEALTH_LOG}" 2>&1 & + PORTFORWARD_HEALTH_PID=$! + + elapsed=0 + while [ "${elapsed}" -lt "${pf_timeout}" ]; do + if ! kill -0 "${PORTFORWARD_HEALTH_PID}" 2>/dev/null; then + echo "ERROR: kubectl health port-forward exited before becoming reachable" >&2 + cat "${PORTFORWARD_HEALTH_LOG}" >&2 || true + DB_FAILED=$((DB_FAILED + 1)) + DB_SCENARIOS_SUMMARY+=("FAIL ${scenario_label}: health port-forward died") + scenario_stop_portforward + scenario_cleanup_release + return + fi + if curl -s -o /dev/null --connect-timeout 1 "http://127.0.0.1:${HEALTH_LOCAL_PORT}/healthz"; then + break + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + if [ "${elapsed}" -ge "${pf_timeout}" ]; then + echo "ERROR: health port-forward did not accept TCP within ${pf_timeout}s" >&2 + cat "${PORTFORWARD_HEALTH_LOG}" >&2 || true + DB_FAILED=$((DB_FAILED + 1)) + DB_SCENARIOS_SUMMARY+=("FAIL ${scenario_label}: health port-forward timeout") + scenario_stop_portforward + scenario_cleanup_release + return + fi + + export OPENSHELL_E2E_HEALTH_PORT="${HEALTH_LOCAL_PORT}" + + GATEWAY_NAME="openshell-e2e-kube-${LOCAL_PORT}" + GATEWAY_ENDPOINT="http://127.0.0.1:${LOCAL_PORT}" + e2e_register_plaintext_gateway \ + "${XDG_CONFIG_HOME}" \ + "${GATEWAY_NAME}" \ + "${GATEWAY_ENDPOINT}" \ + "${LOCAL_PORT}" + + export OPENSHELL_GATEWAY="${GATEWAY_NAME}" + export OPENSHELL_E2E_DRIVER="kubernetes" + export OPENSHELL_E2E_SANDBOX_NAMESPACE="${NAMESPACE}" + export OPENSHELL_PROVISION_TIMEOUT="${OPENSHELL_PROVISION_TIMEOUT:-300}" + + echo "Running e2e command against ${GATEWAY_ENDPOINT}: ${E2E_CMD[*]}" + "${E2E_CMD[@]}" || scenario_exit=$? + + scenario_stop_portforward + scenario_cleanup_release + + if [ "${scenario_exit}" -eq 0 ]; then + echo "==> PASS: ${scenario_label}" + DB_PASSED=$((DB_PASSED + 1)) + DB_SCENARIOS_SUMMARY+=("PASS ${scenario_label}") + else + echo "==> FAIL: ${scenario_label} (exit code ${scenario_exit})" + DB_FAILED=$((DB_FAILED + 1)) + DB_SCENARIOS_SUMMARY+=("FAIL ${scenario_label}: exit code ${scenario_exit}") + fi +} + +# --- end DB-scenario helpers --- + require_cmd() { if ! command -v "$1" >/dev/null 2>&1; then echo "ERROR: $1 is required to run Helm-backed e2e tests" >&2 @@ -306,91 +525,127 @@ if [ -n "${HOST_GATEWAY_IP}" ]; then helm_extra_args+=(--set "server.hostGatewayIP=${HOST_GATEWAY_IP}") fi -echo "Installing Helm chart (release=${RELEASE_NAME}, namespace=${NAMESPACE}, tag=${IMAGE_TAG_VALUE})..." -helmctl install "${RELEASE_NAME}" "${ROOT}/deploy/helm/openshell" \ - --namespace "${NAMESPACE}" --create-namespace \ - --values "${ROOT}/deploy/helm/openshell/ci/values-skaffold.yaml" \ - --set "fullnameOverride=openshell" \ - --set "image.repository=${REGISTRY_VALUE}/gateway" \ - --set "image.tag=${IMAGE_TAG_VALUE}" \ - --set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \ - --set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \ - "${helm_extra_args[@]}" \ - --wait --timeout 5m -HELM_INSTALLED=1 - -LOCAL_PORT="$(e2e_pick_port)" -echo "Starting kubectl port-forward svc/openshell ${LOCAL_PORT}:8080..." -kctl -n "${NAMESPACE}" port-forward "svc/openshell" \ - "${LOCAL_PORT}:8080" >"${PORTFORWARD_LOG}" 2>&1 & -PORTFORWARD_PID=$! - -elapsed=0 -timeout=30 -while [ "${elapsed}" -lt "${timeout}" ]; do - if ! kill -0 "${PORTFORWARD_PID}" 2>/dev/null; then - echo "ERROR: kubectl port-forward exited before becoming reachable" >&2 - cat "${PORTFORWARD_LOG}" >&2 || true +helm dependency build "${ROOT}/deploy/helm/openshell" + +if [ "${OPENSHELL_E2E_KUBE_DB_SCENARIOS:-0}" = "1" ]; then + # --- Multi-scenario mode: test all database backends --- + DB_PASSED=0 + DB_FAILED=0 + DB_SCENARIOS_SUMMARY=() + E2E_CMD=("$@") + + run_scenario "SQLite (default)" sqlite \ + "${helm_extra_args[@]}" + + run_scenario "Bundled PostgreSQL (deploy=true)" bundled-pg \ + "${helm_extra_args[@]}" \ + --set postgres.enabled=true \ + --set postgres.deploy=true \ + --set postgres.auth.password=test-password + + scenario_deploy_external_pg + run_scenario "External PostgreSQL (existingSecret)" external-pg \ + "${helm_extra_args[@]}" \ + --set postgres.enabled=true \ + --set postgres.external.existingSecret=my-pg-credentials + scenario_cleanup_external_pg + + echo "" + echo "========================================" + echo " DB Scenario Test Summary" + echo "========================================" + for s in "${DB_SCENARIOS_SUMMARY[@]}"; do + echo " $s" + done + echo "----------------------------------------" + echo " Passed: $DB_PASSED Failed: $DB_FAILED" + echo "========================================" + + if [ "$DB_FAILED" -gt 0 ]; then exit 1 fi - if curl -s -o /dev/null --connect-timeout 1 "http://127.0.0.1:${LOCAL_PORT}"; then - break +else + # --- Single-install mode (default, existing behavior) --- + echo "Installing Helm chart (release=${RELEASE_NAME}, namespace=${NAMESPACE}, tag=${IMAGE_TAG_VALUE})..." + helmctl install "${RELEASE_NAME}" "${ROOT}/deploy/helm/openshell" \ + --namespace "${NAMESPACE}" --create-namespace \ + --values "${ROOT}/deploy/helm/openshell/ci/values-skaffold.yaml" \ + --set "fullnameOverride=openshell" \ + --set "image.repository=${REGISTRY_VALUE}/gateway" \ + --set "image.tag=${IMAGE_TAG_VALUE}" \ + --set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \ + --set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \ + "${helm_extra_args[@]}" \ + --wait --timeout 5m + HELM_INSTALLED=1 + + LOCAL_PORT="$(e2e_pick_port)" + echo "Starting kubectl port-forward svc/openshell ${LOCAL_PORT}:8080..." + kctl -n "${NAMESPACE}" port-forward "svc/openshell" \ + "${LOCAL_PORT}:8080" >"${PORTFORWARD_LOG}" 2>&1 & + PORTFORWARD_PID=$! + + elapsed=0 + timeout=30 + while [ "${elapsed}" -lt "${timeout}" ]; do + if ! kill -0 "${PORTFORWARD_PID}" 2>/dev/null; then + echo "ERROR: kubectl port-forward exited before becoming reachable" >&2 + cat "${PORTFORWARD_LOG}" >&2 || true + exit 1 + fi + if curl -s -o /dev/null --connect-timeout 1 "http://127.0.0.1:${LOCAL_PORT}"; then + break + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + if [ "${elapsed}" -ge "${timeout}" ]; then + echo "ERROR: port-forward did not accept TCP within ${timeout}s" >&2 + cat "${PORTFORWARD_LOG}" >&2 || true + exit 1 fi - sleep 1 - elapsed=$((elapsed + 1)) -done -if [ "${elapsed}" -ge "${timeout}" ]; then - echo "ERROR: port-forward did not accept TCP within ${timeout}s" >&2 - cat "${PORTFORWARD_LOG}" >&2 || true - exit 1 -fi -# Dedicated port-forward to the gateway pod's health listener. The chart's -# Service intentionally exposes only the gRPC and metrics ports — kubelet -# probes the health endpoint directly on the pod IP — so the /readyz e2e -# test reaches it through this separate forward. Target the named `health` -# containerPort declared on the StatefulSet so a future override of -# `service.healthPort` stays compatible without touching this script. -HEALTH_LOCAL_PORT="$(e2e_pick_port)" -echo "Starting kubectl port-forward sts/${RELEASE_NAME} ${HEALTH_LOCAL_PORT}:health..." -kctl -n "${NAMESPACE}" port-forward "sts/${RELEASE_NAME}" \ - "${HEALTH_LOCAL_PORT}:health" >"${PORTFORWARD_HEALTH_LOG}" 2>&1 & -PORTFORWARD_HEALTH_PID=$! - -elapsed=0 -timeout=30 -while [ "${elapsed}" -lt "${timeout}" ]; do - if ! kill -0 "${PORTFORWARD_HEALTH_PID}" 2>/dev/null; then - echo "ERROR: kubectl health port-forward exited before becoming reachable" >&2 + HEALTH_LOCAL_PORT="$(e2e_pick_port)" + echo "Starting kubectl port-forward sts/${RELEASE_NAME} ${HEALTH_LOCAL_PORT}:health..." + kctl -n "${NAMESPACE}" port-forward "sts/${RELEASE_NAME}" \ + "${HEALTH_LOCAL_PORT}:health" >"${PORTFORWARD_HEALTH_LOG}" 2>&1 & + PORTFORWARD_HEALTH_PID=$! + + elapsed=0 + timeout=30 + while [ "${elapsed}" -lt "${timeout}" ]; do + if ! kill -0 "${PORTFORWARD_HEALTH_PID}" 2>/dev/null; then + echo "ERROR: kubectl health port-forward exited before becoming reachable" >&2 + cat "${PORTFORWARD_HEALTH_LOG}" >&2 || true + exit 1 + fi + if curl -s -o /dev/null --connect-timeout 1 "http://127.0.0.1:${HEALTH_LOCAL_PORT}/healthz"; then + break + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + if [ "${elapsed}" -ge "${timeout}" ]; then + echo "ERROR: health port-forward did not accept TCP within ${timeout}s" >&2 cat "${PORTFORWARD_HEALTH_LOG}" >&2 || true exit 1 fi - if curl -s -o /dev/null --connect-timeout 1 "http://127.0.0.1:${HEALTH_LOCAL_PORT}/healthz"; then - break - fi - sleep 1 - elapsed=$((elapsed + 1)) -done -if [ "${elapsed}" -ge "${timeout}" ]; then - echo "ERROR: health port-forward did not accept TCP within ${timeout}s" >&2 - cat "${PORTFORWARD_HEALTH_LOG}" >&2 || true - exit 1 -fi -export OPENSHELL_E2E_HEALTH_PORT="${HEALTH_LOCAL_PORT}" + export OPENSHELL_E2E_HEALTH_PORT="${HEALTH_LOCAL_PORT}" -GATEWAY_NAME="openshell-e2e-kube-${LOCAL_PORT}" -GATEWAY_ENDPOINT="http://127.0.0.1:${LOCAL_PORT}" -e2e_register_plaintext_gateway \ - "${XDG_CONFIG_HOME}" \ - "${GATEWAY_NAME}" \ - "${GATEWAY_ENDPOINT}" \ - "${LOCAL_PORT}" + GATEWAY_NAME="openshell-e2e-kube-${LOCAL_PORT}" + GATEWAY_ENDPOINT="http://127.0.0.1:${LOCAL_PORT}" + e2e_register_plaintext_gateway \ + "${XDG_CONFIG_HOME}" \ + "${GATEWAY_NAME}" \ + "${GATEWAY_ENDPOINT}" \ + "${LOCAL_PORT}" -export OPENSHELL_GATEWAY="${GATEWAY_NAME}" -export OPENSHELL_E2E_DRIVER="kubernetes" -export OPENSHELL_E2E_SANDBOX_NAMESPACE="${NAMESPACE}" -export OPENSHELL_PROVISION_TIMEOUT="${OPENSHELL_PROVISION_TIMEOUT:-300}" + export OPENSHELL_GATEWAY="${GATEWAY_NAME}" + export OPENSHELL_E2E_DRIVER="kubernetes" + export OPENSHELL_E2E_SANDBOX_NAMESPACE="${NAMESPACE}" + export OPENSHELL_PROVISION_TIMEOUT="${OPENSHELL_PROVISION_TIMEOUT:-300}" -echo "Running e2e command against ${GATEWAY_ENDPOINT}: $*" -"$@" + echo "Running e2e command against ${GATEWAY_ENDPOINT}: $*" + "$@" +fi diff --git a/tasks/test.toml b/tasks/test.toml index 6b5c19cbf..ce8977656 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -80,6 +80,11 @@ run = "e2e/rust/e2e-podman.sh" description = "Run Rust CLI e2e tests against an OpenShell gateway deployed on Kubernetes via Helm (set OPENSHELL_E2E_KUBE_CONTEXT to reuse a cluster; otherwise creates a local k3d cluster when k3d is installed; set OPENSHELL_E2E_KUBE_TEST= to scope to one test)" run = "e2e/rust/e2e-kubernetes.sh" +["e2e:kubernetes:db"] +description = "Run Kubernetes e2e with all database backend scenarios (SQLite, bundled PostgreSQL, external PostgreSQL with existingSecret)" +env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" } +run = "e2e/rust/e2e-kubernetes.sh" + ["e2e:vm"] description = "Start openshell-gateway with the VM compute driver and run the cluster-agnostic smoke e2e" run = "e2e/rust/e2e-vm.sh" From 84e88f705a5db65ff0e36578f13b4392e8d373a3 Mon Sep 17 00:00:00 2001 From: sauagarwa Date: Thu, 28 May 2026 18:35:33 -0400 Subject: [PATCH 6/6] refactor(helm): simplify PostgreSQL config to two orthogonal controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace postgres.deploy and postgres.external.* with two simple controls: - postgres.enabled: deploy the bundled Bitnami PostgreSQL subchart - server.externalDbSecret: name of a pre-existing Secret with a uri key Delete db-secret.yaml — the chart no longer generates Secrets from individual credential fields. Users either get the Bitnami service-binding secret (bundled) or bring their own via server.externalDbSecret. Add validation that postgres.serviceBindings.enabled must stay true when using bundled PostgreSQL, preventing a confusing runtime failure. --- deploy/helm/openshell/Chart.lock | 4 +- deploy/helm/openshell/Chart.yaml | 2 +- deploy/helm/openshell/README.md | 92 ++---------- deploy/helm/openshell/README.md.gotmpl | 80 ++-------- deploy/helm/openshell/templates/_helpers.tpl | 11 +- .../helm/openshell/templates/db-secret.yaml | 28 ---- .../openshell/templates/gateway-config.yaml | 3 +- .../helm/openshell/templates/statefulset.yaml | 11 +- .../openshell/tests/gateway_config_test.yaml | 139 +++++------------- deploy/helm/openshell/values.yaml | 43 ++---- e2e/rust/e2e-openshift.sh | 20 +-- e2e/with-kube-gateway.sh | 18 +-- 12 files changed, 106 insertions(+), 345 deletions(-) delete mode 100644 deploy/helm/openshell/templates/db-secret.yaml diff --git a/deploy/helm/openshell/Chart.lock b/deploy/helm/openshell/Chart.lock index f1a95f424..b079e8e6e 100644 --- a/deploy/helm/openshell/Chart.lock +++ b/deploy/helm/openshell/Chart.lock @@ -2,5 +2,5 @@ dependencies: - name: postgresql repository: oci://registry-1.docker.io/bitnamicharts version: 18.6.7 -digest: sha256:ad78500c7c3a7ee365fd151890cf3368444d6b167c972052fc245024f5a25d9c -generated: "2026-05-27T17:48:47.648592-04:00" +digest: sha256:e4df764483edb0695ac56dd4e27eb3a225a9c0b0ef52a8b60e3e0b51e36153ab +generated: "2026-05-28T18:05:12.507876-04:00" diff --git a/deploy/helm/openshell/Chart.yaml b/deploy/helm/openshell/Chart.yaml index fbfba7b2c..69693662b 100644 --- a/deploy/helm/openshell/Chart.yaml +++ b/deploy/helm/openshell/Chart.yaml @@ -15,5 +15,5 @@ dependencies: - name: postgresql version: 18.6.7 repository: oci://registry-1.docker.io/bitnamicharts - condition: postgres.deploy + condition: postgres.enabled alias: postgres diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 2c0db85fd..1bccd772b 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -68,62 +68,35 @@ postgres: enabled: false ``` -#### Use an existing Kubernetes Secret +#### External PostgreSQL -If you already have a Secret containing PostgreSQL credentials (e.g. managed -via GitOps or external-secrets-operator), point the chart at it directly: +Create a Secret containing the PostgreSQL connection URI if one does not +already exist: ```bash -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set postgres.enabled=true \ - --set postgres.external.existingSecret=my-pg-credentials +kubectl create secret generic my-pg-credentials -n openshell \ + --from-literal=uri="postgresql://user:pass@host:5432/dbname" ``` -On OpenShift, append the platform overrides: +Then install the chart pointing at that Secret: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set postgres.enabled=true \ - --set postgres.external.existingSecret=my-pg-credentials \ - --set server.disableTls=true \ - --set podSecurityContext.fsGroup=null \ - --set securityContext.runAsUser=null -``` - -The Secret must contain a `uri` key with the full connection string: - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: my-pg-credentials -type: Opaque -data: - uri: # postgresql://user:pass@host:5432/dbname + -n openshell \ + --set server.externalDbSecret=my-pg-credentials ``` -#### Kubernetes +#### Bundled PostgreSQL -Enable bundled PostgreSQL: +Deploy a PostgreSQL instance alongside the gateway using the bundled +Bitnami subchart. A random password is generated automatically: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set postgres.enabled=true \ - --set postgres.deploy=true \ - --set postgres.auth.password=my-secret-password + --set postgres.enabled=true ``` -Use external PostgreSQL (chart creates the Secret from fields): - -```bash -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set postgres.enabled=true \ - --set postgres.external.host=my-postgres.example.com \ - --set postgres.external.port=5432 \ - --set postgres.external.database=openshell \ - --set postgres.external.username=openshell \ - --set postgres.external.password=my-password -``` +To set an explicit password, add `--set postgres.auth.password=my-secret-password`. #### OpenShift @@ -135,33 +108,6 @@ Append these flags to any of the PostgreSQL commands above for OpenShift: --set securityContext.runAsUser=null ``` -Enable bundled PostgreSQL on OpenShift: - -```bash -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set postgres.enabled=true \ - --set postgres.deploy=true \ - --set postgres.auth.password=my-secret-password \ - --set server.disableTls=true \ - --set podSecurityContext.fsGroup=null \ - --set securityContext.runAsUser=null -``` - -Use external PostgreSQL on OpenShift: - -```bash -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set postgres.enabled=true \ - --set postgres.external.host=my-postgres.example.com \ - --set postgres.external.port=5432 \ - --set postgres.external.database=openshell \ - --set postgres.external.username=openshell \ - --set postgres.external.password=my-password \ - --set server.disableTls=true \ - --set podSecurityContext.fsGroup=null \ - --set securityContext.runAsUser=null -``` - ## PKI bootstrap By default, a pre-install/pre-upgrade hook Job runs `openshell-gateway generate-certs` @@ -218,14 +164,7 @@ cert-manager alternative. | postgres.auth.database | string | `"openshell"` | | | postgres.auth.password | string | `""` | | | postgres.auth.username | string | `"openshell"` | | -| postgres.deploy | bool | `false` | Deploy the bundled Bitnami PostgreSQL subchart. Set to true to run PostgreSQL alongside the gateway. Leave false when using an external PostgreSQL instance. | -| postgres.enabled | bool | `false` | | -| postgres.external.database | string | `"openshell"` | | -| postgres.external.existingSecret | string | `""` | Name of a pre-existing Opaque Secret containing PostgreSQL credentials. When set, the chart does not create its own db Secret and reads directly from this one. The Secret must contain a `uri` key with the full connection string, e.g. postgresql://user:pass@host:5432/dbname. | -| postgres.external.host | string | `""` | | -| postgres.external.password | string | `""` | | -| postgres.external.port | int | `5432` | | -| postgres.external.username | string | `"openshell"` | | +| postgres.enabled | bool | `false` | Deploy the bundled Bitnami PostgreSQL subchart. | | postgres.primary.persistence.enabled | bool | `true` | | | postgres.serviceBindings.enabled | bool | `true` | | | probes.liveness.failureThreshold | int | `3` | Liveness probe failure threshold before the container is restarted. | @@ -249,10 +188,11 @@ cert-manager alternative. | securityContext.runAsNonRoot | bool | `true` | Require the gateway container to run as a non-root user. | | securityContext.runAsUser | int | `1000` | UID assigned to the gateway container. | | server.auth.allowUnauthenticatedUsers | bool | `false` | UNSAFE: accept unauthenticated CLI/user requests as a local developer principal. Intended only for trusted local Skaffold/k3d development or a fully trusted fronting proxy. Leave false for shared or production clusters. | -| server.dbUrl | string | `"sqlite:/var/openshell/openshell.db"` | Gateway database URL. | +| server.dbUrl | string | `"sqlite:/var/openshell/openshell.db"` | Gateway database URL (used for the default SQLite backend). | | server.disableTls | bool | `false` | Disable TLS entirely - the server listens on plaintext HTTP. Set to true when a reverse proxy / tunnel terminates TLS at the edge. | | server.enableLoopbackServiceHttp | bool | `true` | Enable plaintext HTTP routing for loopback sandbox service URLs on TLS-enabled gateways. | | server.enableUserNamespaces | bool | `false` | Enable Kubernetes user namespace isolation (hostUsers: false) for sandbox pods. Requires Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), plus a supporting container runtime and Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host UID and capabilities become namespaced. | +| server.externalDbSecret | string | `""` | Name of a pre-existing Opaque Secret containing a PostgreSQL connection URI (key: uri). When set, the gateway reads OPENSHELL_DB_URL from this Secret instead of using dbUrl. The Secret must contain a `uri` key, e.g. postgresql://user:pass@host:5432/dbname. | | server.grpcEndpoint | string | `""` | gRPC endpoint sandboxes call back into the gateway. Leave empty to derive it from the chart fullname, release namespace, service port, and disableTls flag, for example https://openshell.openshell.svc.cluster.local:8080. Override only when sandboxes must reach the gateway via a different hostname (e.g. an external ingress or a host alias). | | server.hostGatewayIP | string | `""` | Host gateway IP for sandbox pod hostAliases. When set, sandbox pods get hostAliases entries mapping host.docker.internal and host.openshell.internal to this IP, allowing them to reach services running on the Docker host. Auto-detected by the cluster entrypoint script. | | server.logLevel | string | `"info"` | Gateway log level. | diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index 6c9733fbc..2444e18bc 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -68,62 +68,35 @@ postgres: enabled: false ``` -#### Use an existing Kubernetes Secret +#### External PostgreSQL -If you already have a Secret containing PostgreSQL credentials (e.g. managed -via GitOps or external-secrets-operator), point the chart at it directly: +Create a Secret containing the PostgreSQL connection URI if one does not +already exist: ```bash -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set postgres.enabled=true \ - --set postgres.external.existingSecret=my-pg-credentials +kubectl create secret generic my-pg-credentials -n openshell \ + --from-literal=uri="postgresql://user:pass@host:5432/dbname" ``` -On OpenShift, append the platform overrides: +Then install the chart pointing at that Secret: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set postgres.enabled=true \ - --set postgres.external.existingSecret=my-pg-credentials \ - --set server.disableTls=true \ - --set podSecurityContext.fsGroup=null \ - --set securityContext.runAsUser=null -``` - -The Secret must contain a `uri` key with the full connection string: - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: my-pg-credentials -type: Opaque -data: - uri: # postgresql://user:pass@host:5432/dbname + -n openshell \ + --set server.externalDbSecret=my-pg-credentials ``` -#### Kubernetes +#### Bundled PostgreSQL -Enable bundled PostgreSQL: +Deploy a PostgreSQL instance alongside the gateway using the bundled +Bitnami subchart. A random password is generated automatically: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set postgres.enabled=true \ - --set postgres.deploy=true \ - --set postgres.auth.password=my-secret-password + --set postgres.enabled=true ``` -Use external PostgreSQL (chart creates the Secret from fields): - -```bash -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set postgres.enabled=true \ - --set postgres.external.host=my-postgres.example.com \ - --set postgres.external.port=5432 \ - --set postgres.external.database=openshell \ - --set postgres.external.username=openshell \ - --set postgres.external.password=my-password -``` +To set an explicit password, add `--set postgres.auth.password=my-secret-password`. #### OpenShift @@ -135,33 +108,6 @@ Append these flags to any of the PostgreSQL commands above for OpenShift: --set securityContext.runAsUser=null ``` -Enable bundled PostgreSQL on OpenShift: - -```bash -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set postgres.enabled=true \ - --set postgres.deploy=true \ - --set postgres.auth.password=my-secret-password \ - --set server.disableTls=true \ - --set podSecurityContext.fsGroup=null \ - --set securityContext.runAsUser=null -``` - -Use external PostgreSQL on OpenShift: - -```bash -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ - --set postgres.enabled=true \ - --set postgres.external.host=my-postgres.example.com \ - --set postgres.external.port=5432 \ - --set postgres.external.database=openshell \ - --set postgres.external.username=openshell \ - --set postgres.external.password=my-password \ - --set server.disableTls=true \ - --set podSecurityContext.fsGroup=null \ - --set securityContext.runAsUser=null -``` - ## PKI bootstrap By default, a pre-install/pre-upgrade hook Job runs `openshell-gateway generate-certs` diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 5d2b2803c..c40be9b80 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -122,15 +122,14 @@ postgres.fullnameOverride or postgres.nameOverride. {{/* Name of the Secret holding the PostgreSQL connection URI. -- deploy=true: derive from Bitnami service-binding naming convention -- deploy=false + existingSecret set: use it verbatim -- deploy=false + no existingSecret: use chart-generated "-db" +- server.externalDbSecret set: use it verbatim (always wins) +- postgres.enabled=true: derive from Bitnami service-binding naming convention */}} {{- define "openshell.dbSecretName" -}} -{{- if .Values.postgres.deploy -}} -{{- printf "%s-svcbind-custom-user" (include "openshell.postgresFullname" .) -}} +{{- if .Values.server.externalDbSecret -}} +{{- .Values.server.externalDbSecret -}} {{- else -}} -{{- .Values.postgres.external.existingSecret | default (printf "%s-db" (include "openshell.fullname" .)) -}} +{{- printf "%s-svcbind-custom-user" (include "openshell.postgresFullname" .) -}} {{- end -}} {{- end }} diff --git a/deploy/helm/openshell/templates/db-secret.yaml b/deploy/helm/openshell/templates/db-secret.yaml deleted file mode 100644 index b7e5bb0e3..000000000 --- a/deploy/helm/openshell/templates/db-secret.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -{{/* -Chart-managed db Secret for external PostgreSQL only. -When postgres.deploy=true the Bitnami subchart creates the service-binding -secret (with uri key) via postgres.serviceBindings.enabled=true. -When postgres.external.existingSecret is set the user brings their own Secret. -*/}} -{{- if and .Values.postgres.enabled (not .Values.postgres.deploy) (not .Values.postgres.external.existingSecret) }} -{{- $host := required "postgres.external.host is required when postgres.deploy=false and no postgres.external.existingSecret is provided" .Values.postgres.external.host }} -{{- $pw := required "postgres.external.password is required when postgres.deploy=false and no postgres.external.existingSecret is provided" .Values.postgres.external.password }} -{{- $port := toString (int (default 5432 .Values.postgres.external.port)) }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "openshell.dbSecretName" . }} - labels: - {{- include "openshell.labels" . | nindent 4 }} -type: Opaque -data: - host: {{ $host | b64enc | quote }} - port: {{ $port | b64enc | quote }} - username: {{ .Values.postgres.external.username | b64enc | quote }} - password: {{ $pw | b64enc | quote }} - database: {{ .Values.postgres.external.database | b64enc | quote }} - uri: {{ printf "postgresql://%s:%s@%s:%s/%s" (.Values.postgres.external.username | urlquery) ($pw | urlquery) $host $port .Values.postgres.external.database | b64enc | quote }} -{{- end }} diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 6a8cbca83..52b5a03ac 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -9,7 +9,8 @@ still override anything in this file. One value is intentionally NOT rendered here: - server.dbUrl → passed via OPENSHELL_DB_URL env var (from Secret) - when postgres.enabled=true, or --db-url arg for SQLite + when postgres.enabled=true or server.externalDbSecret + is set, otherwise --db-url arg for SQLite */}} apiVersion: v1 kind: ConfigMap diff --git a/deploy/helm/openshell/templates/statefulset.yaml b/deploy/helm/openshell/templates/statefulset.yaml index 53c3c4320..087748d38 100644 --- a/deploy/helm/openshell/templates/statefulset.yaml +++ b/deploy/helm/openshell/templates/statefulset.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if and .Values.postgres.deploy (not .Values.postgres.enabled) }} -{{- fail "postgres.deploy=true requires postgres.enabled=true" }} +{{- if and .Values.postgres.enabled (not .Values.postgres.serviceBindings.enabled) (not .Values.server.externalDbSecret) }} +{{- fail "postgres.serviceBindings.enabled must be true when using bundled PostgreSQL" }} {{- end }} apiVersion: apps/v1 kind: StatefulSet @@ -23,9 +23,6 @@ spec: # without this annotation a `helm upgrade` that only mutates the # ConfigMap would leave pods running with stale config. checksum/gateway-config: {{ include (print $.Template.BasePath "/gateway-config.yaml") . | sha256sum }} - {{- if and .Values.postgres.enabled (not .Values.postgres.deploy) (not .Values.postgres.external.existingSecret) }} - checksum/db-secret: {{ include (print $.Template.BasePath "/db-secret.yaml") . | sha256sum }} - {{- end }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -59,12 +56,12 @@ spec: args: - --config - /etc/openshell/gateway.toml - {{- if not .Values.postgres.enabled }} + {{- if not (or .Values.postgres.enabled .Values.server.externalDbSecret) }} - --db-url - {{ .Values.server.dbUrl | quote }} {{- end }} env: - {{- if .Values.postgres.enabled }} + {{- if or .Values.postgres.enabled .Values.server.externalDbSecret }} - name: OPENSHELL_DB_URL valueFrom: secretKeyRef: diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 8a820c218..2788bd6b9 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -5,7 +5,6 @@ suite: gateway TOML config shape templates: - templates/gateway-config.yaml - templates/statefulset.yaml - - templates/db-secret.yaml release: name: openshell namespace: my-namespace @@ -134,28 +133,29 @@ tests: path: spec.template.spec.containers[0].args content: "sqlite:/var/openshell/openshell.db" - - it: does not create a db Secret when postgres is disabled - template: templates/db-secret.yaml - asserts: - - hasDocuments: - count: 0 - - - it: does not pass --db-url in args when postgres is enabled + - it: does not pass --db-url in args when postgres.enabled is true template: templates/statefulset.yaml set: postgres.enabled: true - postgres.deploy: true postgres.auth.password: test-pw asserts: - notContains: path: spec.template.spec.containers[0].args content: "--db-url" - - it: references Bitnami service-binding Secret when postgres.deploy is true + - it: does not pass --db-url in args when externalDbSecret is set + template: templates/statefulset.yaml + set: + server.externalDbSecret: my-pg-secret + asserts: + - notContains: + path: spec.template.spec.containers[0].args + content: "--db-url" + + - it: references Bitnami service-binding Secret when postgres.enabled is true template: templates/statefulset.yaml set: postgres.enabled: true - postgres.deploy: true postgres.auth.password: test-pw asserts: - contains: @@ -171,7 +171,6 @@ tests: template: templates/statefulset.yaml set: postgres.enabled: true - postgres.deploy: true postgres.auth.password: test-pw postgres.fullnameOverride: my-pg asserts: @@ -188,7 +187,6 @@ tests: template: templates/statefulset.yaml set: postgres.enabled: true - postgres.deploy: true postgres.auth.password: test-pw postgres.nameOverride: pgdb asserts: @@ -201,13 +199,21 @@ tests: name: openshell-pgdb-svcbind-custom-user key: uri - - it: references chart-created Secret for external postgres + - it: fails when serviceBindings is disabled with bundled postgres template: templates/statefulset.yaml set: postgres.enabled: true - postgres.deploy: false - postgres.external.host: pg.example.com - postgres.external.password: ext-pw + postgres.serviceBindings.enabled: false + asserts: + - failedTemplate: + errorMessage: "postgres.serviceBindings.enabled must be true when using bundled PostgreSQL" + + - it: allows serviceBindings disabled when externalDbSecret is set + template: templates/statefulset.yaml + set: + postgres.enabled: true + postgres.serviceBindings.enabled: false + server.externalDbSecret: my-external-secret asserts: - contains: path: spec.template.spec.containers[0].env @@ -215,14 +221,13 @@ tests: name: OPENSHELL_DB_URL valueFrom: secretKeyRef: - name: openshell-db + name: my-external-secret key: uri - - it: references existing Secret when existingSecret is set + - it: references externalDbSecret when set without postgres.enabled template: templates/statefulset.yaml set: - postgres.enabled: true - postgres.external.existingSecret: my-pg-secret + server.externalDbSecret: my-pg-secret asserts: - contains: path: spec.template.spec.containers[0].env @@ -233,88 +238,18 @@ tests: name: my-pg-secret key: uri - - it: does not create db Secret when postgres.deploy is true - template: templates/db-secret.yaml - set: - postgres.enabled: true - postgres.deploy: true - postgres.auth.password: test-pw - asserts: - - hasDocuments: - count: 0 - - - it: does not create db Secret when existingSecret is set - template: templates/db-secret.yaml - set: - postgres.enabled: true - postgres.external.existingSecret: my-pg-secret - asserts: - - hasDocuments: - count: 0 - - - it: creates db Secret with individual keys for external postgres - template: templates/db-secret.yaml - set: - postgres.enabled: true - postgres.deploy: false - postgres.external.host: external-postgres.example.com - postgres.external.port: 5432 - postgres.external.database: openshell_ext - postgres.external.username: ext_user - postgres.external.password: ext_pass - asserts: - - isKind: - of: Secret - - equal: - path: data.host - decodeBase64: true - value: "external-postgres.example.com" - - equal: - path: data.username - decodeBase64: true - value: "ext_user" - - equal: - path: data.password - decodeBase64: true - value: "ext_pass" - - equal: - path: data.database - decodeBase64: true - value: "openshell_ext" - - equal: - path: data.uri - decodeBase64: true - value: "postgresql://ext_user:ext_pass@external-postgres.example.com:5432/openshell_ext" - - - it: URL-encodes special characters in external credentials - template: templates/db-secret.yaml - set: - postgres.enabled: true - postgres.deploy: false - postgres.external.host: pg.example.com - postgres.external.username: "user@corp" - postgres.external.password: "p@ss:word/secret" - postgres.external.database: mydb - asserts: - - equal: - path: data.uri - decodeBase64: true - value: "postgresql://user%40corp:p%40ss%3Aword%2Fsecret@pg.example.com:5432/mydb" - - - it: fails when postgres.deploy=true but postgres.enabled=false - template: templates/statefulset.yaml - set: - postgres.deploy: true - asserts: - - failedTemplate: - errorMessage: "postgres.deploy=true requires postgres.enabled=true" - - - it: fails when external postgres has no password and no existingSecret + - it: externalDbSecret overrides bundled secret when both are set template: templates/statefulset.yaml set: postgres.enabled: true - postgres.deploy: false - postgres.external.host: my-host.example.com + postgres.auth.password: test-pw + server.externalDbSecret: my-external-secret asserts: - - failedTemplate: - errorMessage: "postgres.external.password is required when postgres.deploy=false and no postgres.external.existingSecret is provided" + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_DB_URL + valueFrom: + secretKeyRef: + name: my-external-secret + key: uri diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index d4d45389a..bfcb68a48 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -138,8 +138,13 @@ server: # -- Namespace where sandbox pods are created. Defaults to the Helm release # namespace (.Release.Namespace) when left empty. sandboxNamespace: "" - # -- Gateway database URL. + # -- Gateway database URL (used for the default SQLite backend). dbUrl: "sqlite:/var/openshell/openshell.db" + # -- Name of a pre-existing Opaque Secret containing a PostgreSQL + # connection URI (key: uri). When set, the gateway reads OPENSHELL_DB_URL + # from this Secret instead of using dbUrl. The Secret must contain a + # `uri` key, e.g. postgresql://user:pass@host:5432/dbname. + externalDbSecret: "" # -- Default sandbox image used when requests do not specify one. sandboxImage: "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" # -- Kubernetes imagePullPolicy for sandbox pods. Empty = Kubernetes default @@ -235,40 +240,22 @@ server: # issuer uses a non-public CA (e.g. OpenShift ingress, private PKI). caConfigMapName: "" -# Optional PostgreSQL backing store. -# - enabled=false (default): gateway uses server.dbUrl (SQLite by default). -# - enabled=true + deploy=true: deploy bundled PostgreSQL subchart and derive dbUrl. -# - enabled=true + deploy=false: use postgres.external.existingSecret or compose -# from postgres.external.* fields. +# Bundled PostgreSQL backing store (Bitnami subchart). +# Set enabled=true to deploy a PostgreSQL instance alongside the gateway. +# For external PostgreSQL, leave enabled=false and set server.externalDbSecret +# to a pre-existing Secret containing a `uri` key. postgres: + # -- Deploy the bundled Bitnami PostgreSQL subchart. enabled: false - # -- Deploy the bundled Bitnami PostgreSQL subchart. Set to true to - # run PostgreSQL alongside the gateway. Leave false when using an - # external PostgreSQL instance. - deploy: false - # External mode connection settings (used when deploy=false). - external: - # -- Name of a pre-existing Opaque Secret containing PostgreSQL - # credentials. When set, the chart does not create its own db Secret - # and reads directly from this one. The Secret must contain a `uri` - # key with the full connection string, e.g. - # postgresql://user:pass@host:5432/dbname. - existingSecret: "" - host: "" - port: 5432 - database: openshell - username: openshell - password: "" - # Values below also configure the bundled Bitnami PostgreSQL subchart + # Values below configure the bundled Bitnami PostgreSQL subchart # (aliased as "postgres" in Chart.yaml). The subchart uses these to - # initialise the PostgreSQL instance; the gateway uses them to compose - # the connection URL. They must stay in sync. + # initialise the PostgreSQL instance. auth: username: openshell password: "" database: openshell - # Enable Bitnami service-binding Secrets so the gateway can read the - # connection URI directly from the subchart-managed Secret. + # Bitnami service-binding Secrets — must stay true. The gateway reads + # the connection URI from the subchart-managed Secret. serviceBindings: enabled: true primary: diff --git a/e2e/rust/e2e-openshift.sh b/e2e/rust/e2e-openshift.sh index 60ef886fe..588a5572e 100755 --- a/e2e/rust/e2e-openshift.sh +++ b/e2e/rust/e2e-openshift.sh @@ -105,17 +105,15 @@ helm install "$RELEASE" "$CHART_PATH" -n "$NAMESPACE" \ verify_gateway "$SCENARIO" cleanup_release -# --- scenario 2: Bundled PostgreSQL (deploy=true) --------------------------- +# --- scenario 2: Bundled PostgreSQL ------------------------------------------- -SCENARIO="Bundled PostgreSQL (deploy=true)" +SCENARIO="Bundled PostgreSQL" log "Testing: $SCENARIO" cleanup_release helm install "$RELEASE" "$CHART_PATH" -n "$NAMESPACE" \ "${OPENSHIFT_FLAGS[@]}" \ - --set postgres.enabled=true \ - --set postgres.deploy=true \ - --set postgres.auth.password=test-password + --set postgres.enabled=true # Wait for postgres to be ready first log "Waiting for bundled PostgreSQL..." @@ -126,7 +124,7 @@ cleanup_release # --- scenario 3: External PostgreSQL with existing Secret ------------------- -SCENARIO="External PostgreSQL (existingSecret)" +SCENARIO="External PostgreSQL (externalDbSecret)" log "Testing: $SCENARIO" cleanup_release @@ -151,22 +149,16 @@ wait_for_ready "app.kubernetes.io/name=postgresql,app.kubernetes.io/instance=$EX EXTERNAL_PG_HOST="${EXTERNAL_PG_RELEASE}-postgresql.${NAMESPACE}.svc.cluster.local" EXTERNAL_PG_URI="postgresql://${EXTERNAL_PG_USERNAME}:${EXTERNAL_PG_PASSWORD}@${EXTERNAL_PG_HOST}:5432/${EXTERNAL_PG_DATABASE}" -# Create the existing Secret with the expected keys +# Create the existing Secret with the uri key log "Creating existing Secret with PostgreSQL credentials..." oc create secret generic my-pg-credentials -n "$NAMESPACE" \ - --from-literal=host="$EXTERNAL_PG_HOST" \ - --from-literal=port="5432" \ - --from-literal=username="$EXTERNAL_PG_USERNAME" \ - --from-literal=password="$EXTERNAL_PG_PASSWORD" \ - --from-literal=database="$EXTERNAL_PG_DATABASE" \ --from-literal=uri="$EXTERNAL_PG_URI" \ 2>/dev/null || true # Install OpenShell pointing at the existing Secret helm install "$RELEASE" "$CHART_PATH" -n "$NAMESPACE" \ "${OPENSHIFT_FLAGS[@]}" \ - --set postgres.enabled=true \ - --set postgres.external.existingSecret=my-pg-credentials + --set server.externalDbSecret=my-pg-credentials verify_gateway "$SCENARIO" diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 717f9351d..53d291ffa 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -194,13 +194,8 @@ scenario_deploy_external_pg() { pg_host="pg-external-postgresql.${NAMESPACE}.svc.cluster.local" pg_uri="postgresql://openshell:ext-test-password@${pg_host}:5432/openshell" - echo "==> Creating Secret with PostgreSQL credentials..." + echo "==> Creating Secret with PostgreSQL URI..." kctl -n "${NAMESPACE}" create secret generic my-pg-credentials \ - --from-literal=host="${pg_host}" \ - --from-literal=port="5432" \ - --from-literal=username="openshell" \ - --from-literal=password="ext-test-password" \ - --from-literal=database="openshell" \ --from-literal=uri="${pg_uri}" \ 2>/dev/null || true } @@ -537,17 +532,14 @@ if [ "${OPENSHELL_E2E_KUBE_DB_SCENARIOS:-0}" = "1" ]; then run_scenario "SQLite (default)" sqlite \ "${helm_extra_args[@]}" - run_scenario "Bundled PostgreSQL (deploy=true)" bundled-pg \ + run_scenario "Bundled PostgreSQL" bundled-pg \ "${helm_extra_args[@]}" \ - --set postgres.enabled=true \ - --set postgres.deploy=true \ - --set postgres.auth.password=test-password + --set postgres.enabled=true scenario_deploy_external_pg - run_scenario "External PostgreSQL (existingSecret)" external-pg \ + run_scenario "External PostgreSQL (externalDbSecret)" external-pg \ "${helm_extra_args[@]}" \ - --set postgres.enabled=true \ - --set postgres.external.existingSecret=my-pg-credentials + --set server.externalDbSecret=my-pg-credentials scenario_cleanup_external_pg echo ""