From ad21b6eb8767ecb93c631791242c0b24f6141b5e Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Mon, 3 Aug 2026 15:53:49 +0100
Subject: [PATCH 1/5] feat(supervisor): configurable tolerations for run pods

Kubernetes clusters that taint their worker nodes had no way to get run pods
scheduled without patching the supervisor. KUBERNETES_RUNNER_TOLERATIONS
applies tolerations to every run pod, in the same format as the existing
scheduled-run tolerations and merged with them for scheduled runs. The Helm
chart takes it as a list at supervisor.config.kubernetes.runnerTolerations.

Toleration keys and values are now checked against the Kubernetes naming
rules at startup, so a typo fails immediately instead of rejecting every pod
create with the cause buried in an API server error. The node label env var
is trimmed and validated for the same reason.

Also documents that an empty KUBERNETES_WORKER_NODETYPE_LABEL disables the
node selector, which the Helm chart has always relied on.
---
 .../supervisor-run-pod-tolerations.md         |   6 +
 apps/supervisor/src/env.ts                    |  65 +--------
 apps/supervisor/src/envUtil.test.ts           | 117 ++++++++++++++++-
 apps/supervisor/src/envUtil.ts                | 123 ++++++++++++++++++
 .../src/workloadManager/kubernetes.test.ts    |  42 ++++++
 .../src/workloadManager/kubernetes.ts         |  28 ++--
 .../src/workloadManager/kubernetesPodSpec.ts  |  30 +++++
 docs/self-hosting/env/supervisor.mdx          |   3 +-
 hosting/k8s/helm/templates/supervisor.yaml    |   4 +
 hosting/k8s/helm/values.yaml                  |   3 +-
 10 files changed, 342 insertions(+), 79 deletions(-)
 create mode 100644 .server-changes/supervisor-run-pod-tolerations.md

diff --git a/.server-changes/supervisor-run-pod-tolerations.md b/.server-changes/supervisor-run-pod-tolerations.md
new file mode 100644
index 00000000000..14f37271be5
--- /dev/null
+++ b/.server-changes/supervisor-run-pod-tolerations.md
@@ -0,0 +1,6 @@
+---
+area: supervisor
+type: feature
+---
+
+Self-hosted Kubernetes deployments can now add tolerations to run pods, so runs are allowed onto tainted nodes.
diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts
index 2d96cb1e407..8184004d9f1 100644
--- a/apps/supervisor/src/env.ts
+++ b/apps/supervisor/src/env.ts
@@ -1,7 +1,7 @@
 import { randomUUID } from "crypto";
 import { env as stdEnv } from "std-env";
 import { z } from "zod";
-import { AdditionalEnvVars, BoolEnv } from "./envUtil.js";
+import { AdditionalEnvVars, BoolEnv, NodeLabelValue, Tolerations } from "./envUtil.js";
 
 export const Env = z
   .object({
@@ -173,7 +173,7 @@ export const Env = z
     // Kubernetes settings
     KUBERNETES_FORCE_ENABLED: BoolEnv.default(false),
     KUBERNETES_NAMESPACE: z.string().default("default"),
-    KUBERNETES_WORKER_NODETYPE_LABEL: z.string().default("v4-worker"),
+    KUBERNETES_WORKER_NODETYPE_LABEL: NodeLabelValue.default("v4-worker"),
     KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv
     KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"),
     KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"),
@@ -256,65 +256,8 @@ export const Env = z
       .max(100)
       .default(20),
 
-    // Schedule toleration settings - scheduled runs tolerate taints on the dedicated pool
-    // Comma-separated list of tolerations in the format: key=value:effect
-    // For Exists operator (no value): key:effect
-    KUBERNETES_SCHEDULED_RUN_TOLERATIONS: z
-      .string()
-      .transform((val, ctx) => {
-        const tolerations = val
-          .split(",")
-          .map((entry) => entry.trim())
-          .filter((entry) => entry.length > 0)
-          .map((entry) => {
-            const colonIdx = entry.lastIndexOf(":");
-            if (colonIdx === -1) {
-              ctx.addIssue({
-                code: z.ZodIssueCode.custom,
-                message: `Invalid toleration format (missing effect): "${entry}"`,
-              });
-              return z.NEVER;
-            }
-
-            const effect = entry.slice(colonIdx + 1);
-            const validEffects = ["NoSchedule", "NoExecute", "PreferNoSchedule"];
-            if (!validEffects.includes(effect)) {
-              ctx.addIssue({
-                code: z.ZodIssueCode.custom,
-                message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join(
-                  ", "
-                )}`,
-              });
-              return z.NEVER;
-            }
-
-            const keyValue = entry.slice(0, colonIdx);
-            const eqIdx = keyValue.indexOf("=");
-            const key = eqIdx === -1 ? keyValue : keyValue.slice(0, eqIdx);
-
-            if (!key) {
-              ctx.addIssue({
-                code: z.ZodIssueCode.custom,
-                message: `Invalid toleration format (empty key): "${entry}"`,
-              });
-              return z.NEVER;
-            }
-
-            if (eqIdx === -1) {
-              return { key, operator: "Exists" as const, effect };
-            }
-
-            return {
-              key,
-              operator: "Equal" as const,
-              value: keyValue.slice(eqIdx + 1),
-              effect,
-            };
-          });
-
-        return tolerations;
-      })
-      .optional(),
+    KUBERNETES_RUNNER_TOLERATIONS: Tolerations.optional(), // every run pod
+    KUBERNETES_SCHEDULED_RUN_TOLERATIONS: Tolerations.optional(), // schedule-tree runs only
 
     // Placement tags settings
     PLACEMENT_TAGS_ENABLED: BoolEnv.default(false),
diff --git a/apps/supervisor/src/envUtil.test.ts b/apps/supervisor/src/envUtil.test.ts
index c3d35758f16..16a93eccea4 100644
--- a/apps/supervisor/src/envUtil.test.ts
+++ b/apps/supervisor/src/envUtil.test.ts
@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest";
-import { BoolEnv, AdditionalEnvVars } from "./envUtil.js";
+import { BoolEnv, AdditionalEnvVars, NodeLabelValue, Tolerations } from "./envUtil.js";
 
 describe("BoolEnv", () => {
   it("should parse string 'true' as true", () => {
@@ -78,3 +78,118 @@ describe("AdditionalEnvVars", () => {
     });
   });
 });
+
+describe("NodeLabelValue", () => {
+  it("should keep a clean value untouched", () => {
+    expect(NodeLabelValue.parse("v4-worker")).toBe("v4-worker");
+  });
+
+  it("should trim surrounding whitespace, which Kubernetes would reject", () => {
+    expect(NodeLabelValue.parse(" v4-worker ")).toBe("v4-worker");
+    expect(NodeLabelValue.parse("\tv4-worker\n")).toBe("v4-worker");
+  });
+
+  it("should treat a whitespace-only value as the empty off-switch", () => {
+    expect(NodeLabelValue.parse("")).toBe("");
+    expect(NodeLabelValue.parse("   ")).toBe("");
+  });
+
+  it("should still apply a default only when unset", () => {
+    const withDefault = NodeLabelValue.default("v4-worker");
+    expect(withDefault.parse(undefined)).toBe("v4-worker");
+    expect(withDefault.parse("")).toBe("");
+  });
+
+  it("should reject a value Kubernetes would reject, rather than 422 every pod create", () => {
+    for (const invalid of ["my worker", "-bad-", "bad.", "a".repeat(64)]) {
+      expect(NodeLabelValue.safeParse(invalid).success).toBe(false);
+    }
+  });
+});
+
+describe("Tolerations", () => {
+  it("should parse key=value entries as Equal", () => {
+    expect(Tolerations.parse("dedicated=runs:NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should parse entries without a value as Exists", () => {
+    expect(Tolerations.parse("scheduled-runs:NoExecute")).toEqual([
+      { key: "scheduled-runs", operator: "Exists", effect: "NoExecute" },
+    ]);
+  });
+
+  it("should parse an empty string as no tolerations", () => {
+    expect(Tolerations.parse("")).toEqual([]);
+    expect(Tolerations.parse("  ")).toEqual([]);
+  });
+
+  it("should skip blank entries and trim whitespace", () => {
+    expect(Tolerations.parse(" a=b:NoSchedule , ,")).toEqual([
+      { key: "a", operator: "Equal", value: "b", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should reject a missing effect, an unknown effect, and an empty key", () => {
+    for (const invalid of ["dedicated=runs", "dedicated=runs:Nope", "=runs:NoSchedule"]) {
+      expect(Tolerations.safeParse(invalid).success).toBe(false);
+    }
+  });
+
+  it("should accept a hyphenated key, a digit-suffixed key, and every effect", () => {
+    expect(
+      Tolerations.parse("capacity-1=true:PreferNoSchedule,spot:NoExecute,gpu=a10:NoSchedule")
+    ).toEqual([
+      { key: "capacity-1", operator: "Equal", value: "true", effect: "PreferNoSchedule" },
+      { key: "spot", operator: "Exists", effect: "NoExecute" },
+      { key: "gpu", operator: "Equal", value: "a10", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should accept a DNS-subdomain prefixed key", () => {
+    expect(
+      Tolerations.parse("node.cluster.x-k8s.io/machinepool=scheduled-runs:NoSchedule")
+    ).toEqual([
+      {
+        key: "node.cluster.x-k8s.io/machinepool",
+        operator: "Equal",
+        value: "scheduled-runs",
+        effect: "NoSchedule",
+      },
+    ]);
+  });
+
+  it("should reject a key or value that Kubernetes would reject at pod create", () => {
+    for (const invalid of [
+      "dedicated=prod runs:NoSchedule",
+      "ded icated=runs:NoSchedule",
+      "dedicated=-runs:NoSchedule",
+      `dedicated=${"r".repeat(64)}:NoSchedule`,
+      `${"a".repeat(64)}=runs:NoSchedule`,
+      `example.com/${"a".repeat(64)}=runs:NoSchedule`,
+      "a/b/c=runs:NoSchedule",
+      "Example.com/pool=runs:NoSchedule",
+    ]) {
+      expect(Tolerations.safeParse(invalid).success).toBe(false);
+    }
+  });
+
+  it("should bound the prefix and the name separately, as Kubernetes does", () => {
+    const longestPrefix = `${"a".repeat(63)}.${"b".repeat(63)}.${"c".repeat(63)}.${"d".repeat(61)}`;
+    expect(longestPrefix.length).toBe(253);
+
+    expect(Tolerations.parse(`${longestPrefix}/${"n".repeat(63)}=runs:NoSchedule`)).toHaveLength(1);
+    expect(Tolerations.safeParse(`${longestPrefix}a/pool=runs:NoSchedule`).success).toBe(false);
+  });
+
+  it("should tolerate whitespace around the separators", () => {
+    expect(Tolerations.parse("dedicated = runs : NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should reject a stray extra effect instead of folding it into the value", () => {
+    expect(Tolerations.safeParse("dedicated=runs:NoSchedule:NoExecute").success).toBe(false);
+  });
+});
diff --git a/apps/supervisor/src/envUtil.ts b/apps/supervisor/src/envUtil.ts
index 917f984cc37..944c14848cf 100644
--- a/apps/supervisor/src/envUtil.ts
+++ b/apps/supervisor/src/envUtil.ts
@@ -16,6 +16,129 @@ export const BoolEnv = baseBoolEnv as Omit<typeof baseBoolEnv, "default"> & {
   default: (value: boolean) => z.ZodDefault<typeof baseBoolEnv>;
 };
 
+const QUALIFIED_NAME = /^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$/;
+const DNS_SUBDOMAIN = /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/;
+const LABEL_VALUE = /^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$/;
+const QUALIFIED_NAME_MAX = 63;
+const DNS_SUBDOMAIN_MAX = 253;
+const LABEL_VALUE_MAX = 63;
+
+/**
+ * isLabelValue mirrors the Kubernetes label value rules. Empty is valid upstream.
+ */
+function isLabelValue(value: string): boolean {
+  return value.length <= LABEL_VALUE_MAX && LABEL_VALUE.test(value);
+}
+
+/**
+ * isQualifiedName mirrors the Kubernetes qualified name rules used for taint and
+ * label keys: an optional DNS subdomain prefix before the slash, then the name.
+ * The two halves have different length limits and different case rules, so a
+ * single pattern with one overall bound gets both ends wrong.
+ */
+function isQualifiedName(key: string): boolean {
+  const slashIdx = key.indexOf("/");
+
+  if (slashIdx === -1) {
+    return key.length <= QUALIFIED_NAME_MAX && QUALIFIED_NAME.test(key);
+  }
+
+  const prefix = key.slice(0, slashIdx);
+  const name = key.slice(slashIdx + 1);
+
+  return (
+    prefix.length <= DNS_SUBDOMAIN_MAX &&
+    DNS_SUBDOMAIN.test(prefix) &&
+    name.length <= QUALIFIED_NAME_MAX &&
+    QUALIFIED_NAME.test(name)
+  );
+}
+
+/**
+ * A node label value. Trimmed because Kubernetes rejects surrounding whitespace
+ * outright, so a padded value fails every pod create. Deliberately no `min(1)`:
+ * empty is the off-switch, and the Helm chart ships empty by default.
+ */
+export const NodeLabelValue = z.string().trim().refine(isLabelValue, {
+  message:
+    "Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside, at most 63 characters",
+});
+
+/**
+ * Comma-separated pod tolerations in the format `key=value:effect`, or `key:effect`
+ * for the Exists operator. Keys and values are checked against the Kubernetes
+ * naming rules here so a typo fails at startup, rather than 422ing every single
+ * pod create with the cause buried in an API server message.
+ */
+export const Tolerations = z.string().transform((val, ctx) => {
+  return val
+    .split(",")
+    .map((entry) => entry.trim())
+    .filter((entry) => entry.length > 0)
+    .map((entry) => {
+      const colonIdx = entry.lastIndexOf(":");
+      if (colonIdx === -1) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration format (missing effect): "${entry}"`,
+        });
+        return z.NEVER;
+      }
+
+      const effect = entry.slice(colonIdx + 1).trim();
+      const validEffects = ["NoSchedule", "NoExecute", "PreferNoSchedule"];
+      if (!validEffects.includes(effect)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join(
+            ", "
+          )}`,
+        });
+        return z.NEVER;
+      }
+
+      const keyValue = entry.slice(0, colonIdx);
+      const eqIdx = keyValue.indexOf("=");
+      const key = (eqIdx === -1 ? keyValue : keyValue.slice(0, eqIdx)).trim();
+
+      if (!key) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration format (empty key): "${entry}"`,
+        });
+        return z.NEVER;
+      }
+
+      if (!isQualifiedName(key)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration key "${key}" in "${entry}". Must be a Kubernetes taint key, optionally prefixed with a DNS subdomain.`,
+        });
+        return z.NEVER;
+      }
+
+      if (eqIdx === -1) {
+        return { key, operator: "Exists" as const, effect };
+      }
+
+      const value = keyValue.slice(eqIdx + 1).trim();
+      if (!isLabelValue(value)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration value "${value}" in "${entry}". Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside.`,
+        });
+        return z.NEVER;
+      }
+
+      return {
+        key,
+        operator: "Equal" as const,
+        value,
+        effect,
+      };
+    });
+});
+
 export const AdditionalEnvVars = z.preprocess((val) => {
   if (typeof val !== "string") {
     return val;
diff --git a/apps/supervisor/src/workloadManager/kubernetes.test.ts b/apps/supervisor/src/workloadManager/kubernetes.test.ts
index 85ad3cbebff..bb15c23e9f4 100644
--- a/apps/supervisor/src/workloadManager/kubernetes.test.ts
+++ b/apps/supervisor/src/workloadManager/kubernetes.test.ts
@@ -1,6 +1,8 @@
 import { describe, expect, it } from "vitest";
 import {
   BLOCK_IO_URING_SECCOMP_PROFILE,
+  nodetypeNodeSelector,
+  runPodTolerations,
   withBlockIoUringSeccompProfile,
 } from "./kubernetesPodSpec.js";
 
@@ -14,6 +16,46 @@ const basePodSpec = {
   },
 };
 
+describe("nodetypeNodeSelector", () => {
+  it("omits the nodeSelector entirely when the label is empty or unset", () => {
+    for (const label of ["", undefined]) {
+      expect(nodetypeNodeSelector(label)).toEqual({});
+    }
+  });
+
+  it("pins to nodetype=<label> when set", () => {
+    expect(nodetypeNodeSelector("v4-worker")).toEqual({ nodeSelector: { nodetype: "v4-worker" } });
+  });
+});
+
+describe("runPodTolerations", () => {
+  const worker = [{ key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" }];
+  const scheduled = [{ key: "scheduled-runs", operator: "Exists", effect: "NoSchedule" }];
+
+  it("leaves tolerations unset when neither is configured", () => {
+    expect(runPodTolerations(undefined, undefined, false)).toBeUndefined();
+    expect(runPodTolerations(undefined, undefined, true)).toBeUndefined();
+    expect(runPodTolerations([], [], true)).toBeUndefined();
+  });
+
+  it("applies the worker tolerations to every run", () => {
+    expect(runPodTolerations(worker, undefined, false)).toEqual(worker);
+    expect(runPodTolerations(worker, undefined, true)).toEqual(worker);
+  });
+
+  it("applies the scheduled-run tolerations on their own, as before this option existed", () => {
+    expect(runPodTolerations(undefined, scheduled, true)).toEqual(scheduled);
+    expect(runPodTolerations(undefined, scheduled, false)).toBeUndefined();
+    expect(runPodTolerations([], scheduled, true)).toEqual(scheduled);
+  });
+
+  it("adds the scheduled-run tolerations only for scheduled runs", () => {
+    expect(runPodTolerations(worker, scheduled, false)).toEqual(worker);
+    expect(runPodTolerations(worker, [], true)).toEqual(worker);
+    expect(runPodTolerations(worker, scheduled, true)).toEqual([...worker, ...scheduled]);
+  });
+});
+
 describe("withBlockIoUringSeccompProfile", () => {
   it("adds the Localhost io_uring profile for node-24 and above, preserving pod security defaults", () => {
     for (const runtime of ["node-24", "node-26", "node-30", "experimental-node-24"]) {
diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts
index 0263b229e13..1b88bafbc28 100644
--- a/apps/supervisor/src/workloadManager/kubernetes.ts
+++ b/apps/supervisor/src/workloadManager/kubernetes.ts
@@ -14,7 +14,11 @@ import { PlacementTagProcessor } from "@trigger.dev/core/v3/serverOnly";
 import { env } from "../env.js";
 import { type K8sApi, createK8sApi, type k8s } from "../clients/kubernetes.js";
 import { getRunnerId } from "../util.js";
-import { withBlockIoUringSeccompProfile } from "./kubernetesPodSpec.js";
+import {
+  nodetypeNodeSelector,
+  runPodTolerations,
+  withBlockIoUringSeccompProfile,
+} from "./kubernetesPodSpec.js";
 
 type ResourceQuantities = {
   [K in "cpu" | "memory" | "ephemeral-storage"]?: string;
@@ -127,7 +131,7 @@ export class KubernetesWorkloadManager implements WorkloadManager {
           spec: {
             ...podSpec,
             affinity: this.#getAffinity(opts),
-            tolerations: this.#getScheduleTolerations(this.#isScheduledRun(opts)),
+            tolerations: this.#getTolerations(this.#isScheduledRun(opts)),
             terminationGracePeriodSeconds: 60 * 60,
             containers: [
               {
@@ -329,13 +333,7 @@ export class KubernetesWorkloadManager implements WorkloadManager {
             schedulerName: env.KUBERNETES_SCHEDULER_NAME,
           }
         : {}),
-      ...(env.KUBERNETES_WORKER_NODETYPE_LABEL
-        ? {
-            nodeSelector: {
-              nodetype: env.KUBERNETES_WORKER_NODETYPE_LABEL,
-            },
-          }
-        : {}),
+      ...nodetypeNodeSelector(env.KUBERNETES_WORKER_NODETYPE_LABEL),
       ...(env.KUBERNETES_POD_DNS_NDOTS_OVERRIDE_ENABLED
         ? {
             dnsConfig: {
@@ -557,12 +555,12 @@ export class KubernetesWorkloadManager implements WorkloadManager {
     };
   }
 
-  #getScheduleTolerations(isScheduledRun: boolean): k8s.V1Toleration[] | undefined {
-    if (!isScheduledRun || !env.KUBERNETES_SCHEDULED_RUN_TOLERATIONS?.length) {
-      return undefined;
-    }
-
-    return env.KUBERNETES_SCHEDULED_RUN_TOLERATIONS;
+  #getTolerations(isScheduledRun: boolean): k8s.V1Toleration[] | undefined {
+    return runPodTolerations(
+      env.KUBERNETES_RUNNER_TOLERATIONS,
+      env.KUBERNETES_SCHEDULED_RUN_TOLERATIONS,
+      isScheduledRun
+    );
   }
 
   #getProjectPodAffinity(projectId: string): k8s.V1PodAffinity | undefined {
diff --git a/apps/supervisor/src/workloadManager/kubernetesPodSpec.ts b/apps/supervisor/src/workloadManager/kubernetesPodSpec.ts
index 32c4410ca43..ba15e563f6d 100644
--- a/apps/supervisor/src/workloadManager/kubernetesPodSpec.ts
+++ b/apps/supervisor/src/workloadManager/kubernetesPodSpec.ts
@@ -6,6 +6,36 @@ import type { k8s } from "../clients/kubernetes.js";
  */
 export const BLOCK_IO_URING_SECCOMP_PROFILE = "profiles/block-io-uring.json";
 
+/**
+ * An empty label is the documented off-switch, leaving the pod unpinned. The Helm
+ * chart ships an empty value, so don't collapse this into a fallback default -
+ * that would pin every chart install to a label its nodes don't carry.
+ */
+export function nodetypeNodeSelector(
+  label: string | undefined
+): Pick<k8s.V1PodSpec, "nodeSelector"> {
+  return label ? { nodeSelector: { nodetype: label } } : {};
+}
+
+/**
+ * Tolerations for a run pod: the cluster-wide set, plus the scheduled-run set when the
+ * run came from a schedule tree. Not reconciled - Kubernetes matches tolerations as an
+ * any-match set, so a broad entry in one set can subsume a narrower one in the other.
+ * Returns undefined rather than an empty array to leave the field unset.
+ */
+export function runPodTolerations(
+  runnerTolerations: k8s.V1Toleration[] | undefined,
+  scheduledRunTolerations: k8s.V1Toleration[] | undefined,
+  isScheduledRun: boolean
+): k8s.V1Toleration[] | undefined {
+  const tolerations = [
+    ...(runnerTolerations ?? []),
+    ...(isScheduledRun ? (scheduledRunTolerations ?? []) : []),
+  ];
+
+  return tolerations.length > 0 ? tolerations : undefined;
+}
+
 /**
  * Node >= 24 always creates io_uring fds, which can't be checkpointed. Blocking
  * io_uring_setup makes libuv fall back to epoll. Other runtimes don't need this,
diff --git a/docs/self-hosting/env/supervisor.mdx b/docs/self-hosting/env/supervisor.mdx
index ffa057747fe..a7e4ef96692 100644
--- a/docs/self-hosting/env/supervisor.mdx
+++ b/docs/self-hosting/env/supervisor.mdx
@@ -46,7 +46,8 @@ mode: "wide"
 | **Kubernetes settings**                     |          |             |                                                             |
 | `KUBERNETES_FORCE_ENABLED`                  | No       | false       | Force Kubernetes mode.                                      |
 | `KUBERNETES_NAMESPACE`                      | No       | default     | The namespace that runs should be in.                       |
-| `KUBERNETES_WORKER_NODETYPE_LABEL`          | No       | v4-worker   | Nodes for runs need this label, e.g. `nodetype=v4-worker`.  |
+| `KUBERNETES_WORKER_NODETYPE_LABEL`          | No       | v4-worker   | Nodes for runs need `nodetype=<this>`. Empty: any node.     |
+| `KUBERNETES_RUNNER_TOLERATIONS`             | No       | —           | Run pod tolerations. CSV: `key=value:effect`/`key:effect`.  |
 | `KUBERNETES_IMAGE_PULL_SECRETS`             | No       | —           | Image pull secrets (CSV).                                   |
 | `KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT`   | No       | 10Gi        | Ephemeral storage size limit. Applies to all runs.          |
 | `KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST` | No       | 2Gi         | Ephemeral storage size request. Applies to all runs.        |
diff --git a/hosting/k8s/helm/templates/supervisor.yaml b/hosting/k8s/helm/templates/supervisor.yaml
index 11fd7a7f6d9..84a6350f035 100644
--- a/hosting/k8s/helm/templates/supervisor.yaml
+++ b/hosting/k8s/helm/templates/supervisor.yaml
@@ -170,6 +170,10 @@ spec:
               value: {{ .Values.supervisor.config.kubernetes.forceEnabled | quote }}
             - name: KUBERNETES_WORKER_NODETYPE_LABEL
               value: {{ .Values.supervisor.config.kubernetes.workerNodetypeLabel | quote }}
+            {{- with .Values.supervisor.config.kubernetes.runnerTolerations }}
+            - name: KUBERNETES_RUNNER_TOLERATIONS
+              value: {{ join "," . | quote }}
+            {{- end }}
             {{- $registryAuthEnabled := false }}
             {{- if .Values.registry.deploy }}
               {{- $registryAuthEnabled = .Values.registry.auth.enabled }}
diff --git a/hosting/k8s/helm/values.yaml b/hosting/k8s/helm/values.yaml
index e6ea51864f1..354d8e55ba1 100644
--- a/hosting/k8s/helm/values.yaml
+++ b/hosting/k8s/helm/values.yaml
@@ -296,6 +296,7 @@ supervisor:
       forceEnabled: true
       namespace: "" # Default: uses release namespace
       workerNodetypeLabel: "" # When set, runs will only be scheduled on nodes with "nodetype=<label>"
+      runnerTolerations: [] # Run pod tolerations, e.g. ["dedicated=runs:NoSchedule"]
       ephemeralStorageSizeLimit: "" # Default: 10Gi
       ephemeralStorageSizeRequest: "" # Default: 2Gi´
     podCleaner:
@@ -386,7 +387,7 @@ supervisor:
         key: ""
 
   nodeSelector: {}
-  tolerations: []
+  tolerations: [] # For the supervisor pod itself, not run pods
   affinity: {}
 
 # PostgreSQL configuration

From af73c538a232379085fe5a47f1fdf6b6d3976823 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Mon, 3 Aug 2026 16:15:22 +0100
Subject: [PATCH 2/5] test(supervisor): pin the empty toleration value as an
 exact valueless-taint match

---
 apps/supervisor/src/envUtil.test.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/apps/supervisor/src/envUtil.test.ts b/apps/supervisor/src/envUtil.test.ts
index 16a93eccea4..acafcf703a1 100644
--- a/apps/supervisor/src/envUtil.test.ts
+++ b/apps/supervisor/src/envUtil.test.ts
@@ -120,6 +120,12 @@ describe("Tolerations", () => {
     ]);
   });
 
+  it("should keep an empty value as an exact match for a valueless taint", () => {
+    expect(Tolerations.parse("dedicated=:NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Equal", value: "", effect: "NoSchedule" },
+    ]);
+  });
+
   it("should parse an empty string as no tolerations", () => {
     expect(Tolerations.parse("")).toEqual([]);
     expect(Tolerations.parse("  ")).toEqual([]);

From 0b28b7f78089b052d4c6c708c8b6b8aafdce64fd Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Mon, 3 Aug 2026 16:28:59 +0100
Subject: [PATCH 3/5] fix(supervisor): reject an empty toleration value

An empty value after "=" parses as an exact match on a valueless taint,
which is legal but almost never what was meant. A templated value that
resolved empty produced a toleration that matched nothing, so the
supervisor started and every run pod stayed pending. The error names the
Exists form for anyone who did mean it.
---
 apps/supervisor/src/envUtil.test.ts | 12 +++++++++---
 apps/supervisor/src/envUtil.ts      |  8 ++++++++
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/apps/supervisor/src/envUtil.test.ts b/apps/supervisor/src/envUtil.test.ts
index acafcf703a1..33095676d29 100644
--- a/apps/supervisor/src/envUtil.test.ts
+++ b/apps/supervisor/src/envUtil.test.ts
@@ -120,9 +120,15 @@ describe("Tolerations", () => {
     ]);
   });
 
-  it("should keep an empty value as an exact match for a valueless taint", () => {
-    expect(Tolerations.parse("dedicated=:NoSchedule")).toEqual([
-      { key: "dedicated", operator: "Equal", value: "", effect: "NoSchedule" },
+  it("should reject an empty value, and point at the Exists form instead", () => {
+    const result = Tolerations.safeParse("dedicated=:NoSchedule");
+    expect(result.success).toBe(false);
+    expect(result.error?.issues[0]?.message).toBe(
+      'Invalid toleration format (empty value): "dedicated=:NoSchedule". Drop the "=" to tolerate any value of "dedicated".'
+    );
+
+    expect(Tolerations.parse("dedicated:NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Exists", effect: "NoSchedule" },
     ]);
   });
 
diff --git a/apps/supervisor/src/envUtil.ts b/apps/supervisor/src/envUtil.ts
index 944c14848cf..abf317e6a52 100644
--- a/apps/supervisor/src/envUtil.ts
+++ b/apps/supervisor/src/envUtil.ts
@@ -122,6 +122,14 @@ export const Tolerations = z.string().transform((val, ctx) => {
       }
 
       const value = keyValue.slice(eqIdx + 1).trim();
+      if (!value) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration format (empty value): "${entry}". Drop the "=" to tolerate any value of "${key}".`,
+        });
+        return z.NEVER;
+      }
+
       if (!isLabelValue(value)) {
         ctx.addIssue({
           code: z.ZodIssueCode.custom,

From 470971b40e8dd48ee4d31efada1d10c119afd851 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Mon, 3 Aug 2026 16:34:44 +0100
Subject: [PATCH 4/5] fix(supervisor): warn on an empty toleration value
 instead of rejecting it

The previous parser accepted an empty value, so rejecting it would
crash-loop a supervisor on upgrade for a config that worked. Keep parsing
it as an exact match on a valueless taint, and warn at startup since it is
usually a value that resolved empty by accident.
---
 apps/supervisor/src/envUtil.test.ts | 10 ++++------
 apps/supervisor/src/envUtil.ts      |  9 ++++-----
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/apps/supervisor/src/envUtil.test.ts b/apps/supervisor/src/envUtil.test.ts
index 33095676d29..378830f8ab0 100644
--- a/apps/supervisor/src/envUtil.test.ts
+++ b/apps/supervisor/src/envUtil.test.ts
@@ -120,12 +120,10 @@ describe("Tolerations", () => {
     ]);
   });
 
-  it("should reject an empty value, and point at the Exists form instead", () => {
-    const result = Tolerations.safeParse("dedicated=:NoSchedule");
-    expect(result.success).toBe(false);
-    expect(result.error?.issues[0]?.message).toBe(
-      'Invalid toleration format (empty value): "dedicated=:NoSchedule". Drop the "=" to tolerate any value of "dedicated".'
-    );
+  it("should keep an empty value as an exact match for a valueless taint", () => {
+    expect(Tolerations.parse("dedicated=:NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Equal", value: "", effect: "NoSchedule" },
+    ]);
 
     expect(Tolerations.parse("dedicated:NoSchedule")).toEqual([
       { key: "dedicated", operator: "Exists", effect: "NoSchedule" },
diff --git a/apps/supervisor/src/envUtil.ts b/apps/supervisor/src/envUtil.ts
index abf317e6a52..67811f76fcb 100644
--- a/apps/supervisor/src/envUtil.ts
+++ b/apps/supervisor/src/envUtil.ts
@@ -123,11 +123,10 @@ export const Tolerations = z.string().transform((val, ctx) => {
 
       const value = keyValue.slice(eqIdx + 1).trim();
       if (!value) {
-        ctx.addIssue({
-          code: z.ZodIssueCode.custom,
-          message: `Invalid toleration format (empty value): "${entry}". Drop the "=" to tolerate any value of "${key}".`,
-        });
-        return z.NEVER;
+        logger.warn(
+          'Toleration has an empty value, so it matches only a taint whose value is also empty. Drop the "=" to tolerate any value of this key.',
+          { entry, key }
+        );
       }
 
       if (!isLabelValue(value)) {

From 4447ebc9baf4a14766f824c5bc5169679ab58638 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Mon, 3 Aug 2026 16:36:50 +0100
Subject: [PATCH 5/5] docs(supervisor): note that an invalid toleration now
 blocks startup

---
 .server-changes/supervisor-run-pod-tolerations.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.server-changes/supervisor-run-pod-tolerations.md b/.server-changes/supervisor-run-pod-tolerations.md
index 14f37271be5..885b2b596da 100644
--- a/.server-changes/supervisor-run-pod-tolerations.md
+++ b/.server-changes/supervisor-run-pod-tolerations.md
@@ -3,4 +3,4 @@ area: supervisor
 type: feature
 ---
 
-Self-hosted Kubernetes deployments can now add tolerations to run pods, so runs are allowed onto tainted nodes.
+Self-hosted Kubernetes deployments can now add tolerations to run pods, so runs are allowed onto tainted nodes. An invalid toleration now stops the supervisor at startup instead of failing every run pod, so check existing values before upgrading.