Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions cmd/thv-operator/api/v1beta1/mcpserver_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,22 @@ type ProxyDeploymentOverrides struct {
// +listType=atomic
// +optional
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`

// NodeSelector constrains the proxy pod to nodes with matching labels.
// Mirrors the scheduling control podTemplateSpec gives the MCP server pod, so
// the proxy can be steered onto the same nodes (e.g. a pre-warmed pool).
// +optional
NodeSelector map[string]string `json:"nodeSelector,omitempty"`

// Tolerations allow the proxy pod to schedule onto tainted nodes, such as a
// dedicated pre-warmed pool.
// +listType=atomic
// +optional
Tolerations []corev1.Toleration `json:"tolerations,omitempty"`

// Affinity sets node/pod affinity and anti-affinity for the proxy pod.
// +optional
Affinity *corev1.Affinity `json:"affinity,omitempty"`
}

// ResourceMetadataOverrides defines metadata overrides for a resource
Expand Down
19 changes: 19 additions & 0 deletions cmd/thv-operator/api/v1beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

19 changes: 19 additions & 0 deletions cmd/thv-operator/controllers/mcpremoteproxy_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -1795,13 +1795,32 @@ func (r *MCPRemoteProxyReconciler) podSpecNeedsUpdate(
!equality.Semantic.DeepEqual(
deployment.Spec.Template.Spec.ImagePullSecrets,
expectedDeployment.Spec.Template.Spec.ImagePullSecrets,
) ||
// Scheduling can come from podTemplateSpec as well as from
// resourceOverrides, so compare against the rebuilt Deployment rather
// than the overrides alone.
!equality.Semantic.DeepEqual(
deployment.Spec.Template.Spec.NodeSelector,
expectedDeployment.Spec.Template.Spec.NodeSelector,
) ||
!equality.Semantic.DeepEqual(
deployment.Spec.Template.Spec.Tolerations,
expectedDeployment.Spec.Template.Spec.Tolerations,
) ||
!equality.Semantic.DeepEqual(
deployment.Spec.Template.Spec.Affinity,
expectedDeployment.Spec.Template.Spec.Affinity,
)
}

if deployment.Spec.Template.Spec.ServiceAccountName != serviceAccountNameForRemoteProxy(proxy) {
return true
}

if proxySchedulingNeedsUpdate(deployment, proxy.Spec.ResourceOverrides) {
return true
}

expected := r.imagePullSecretsForRemoteProxy(proxy)
return !equality.Semantic.DeepEqual(deployment.Spec.Template.Spec.ImagePullSecrets, expected)
}
Expand Down
4 changes: 4 additions & 0 deletions cmd/thv-operator/controllers/mcpremoteproxy_deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ func (r *MCPRemoteProxyReconciler) deploymentForMCPRemoteProxy(
deploymentLabels, deploymentAnnotations := r.buildDeploymentMetadata(ls, proxy)
deploymentTemplateLabels, deploymentTemplateAnnotations := r.buildPodTemplateMetadata(ls, proxy, runConfigChecksum)
podSecurityContext, containerSecurityContext := r.buildSecurityContexts(ctx, proxy)
proxyNodeSelector, proxyTolerations, proxyAffinity := proxyDeploymentScheduling(proxy.Spec.ResourceOverrides)

dep := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -86,6 +87,9 @@ func (r *MCPRemoteProxyReconciler) deploymentForMCPRemoteProxy(
Spec: corev1.PodSpec{
ServiceAccountName: serviceAccountNameForRemoteProxy(proxy),
ImagePullSecrets: r.imagePullSecretsForRemoteProxy(proxy),
NodeSelector: proxyNodeSelector,
Tolerations: proxyTolerations,
Affinity: proxyAffinity,
Containers: []corev1.Container{{
Image: getToolhiveRunnerImage(),
Name: mcpRemoteProxyContainerName,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -236,3 +236,69 @@ func TestMCPRemoteProxyPodTemplateSpecDriftDetection(t *testing.T) {
proxy.Spec.PodTemplateSpec = nil
assert.True(t, reconciler.deploymentNeedsUpdate(t.Context(), deployment, proxy, "test-checksum"))
}

// Scheduling can arrive from either resourceOverrides.proxyDeployment or
// podTemplateSpec. Drift detection must account for both, so an override-only
// comparison would report false drift on a podTemplateSpec-supplied value.
func TestMCPRemoteProxySchedulingOverridesAndPodTemplateSpec(t *testing.T) {
t.Parallel()

scheme := testutil.NewScheme(t)
reconciler := &MCPRemoteProxyReconciler{
Scheme: scheme,
PlatformDetector: ctrlutil.NewSharedPlatformDetector(),
}

t.Run("overrides applied and stable", func(t *testing.T) {
t.Parallel()

proxy := &mcpv1beta1.MCPRemoteProxy{
ObjectMeta: metav1.ObjectMeta{Name: "sched-proxy", Namespace: "default"},
Spec: mcpv1beta1.MCPRemoteProxySpec{
RemoteURL: "https://mcp.example.com",
ResourceOverrides: &mcpv1beta1.ResourceOverrides{
ProxyDeployment: &mcpv1beta1.ProxyDeploymentOverrides{
NodeSelector: map[string]string{"workload-class": "mcp-warm"},
Tolerations: []corev1.Toleration{{
Key: "workload-class",
Operator: corev1.TolerationOpEqual,
Value: "mcp-warm",
Effect: corev1.TaintEffectNoSchedule,
}},
},
},
},
}

deployment := reconciler.deploymentForMCPRemoteProxy(t.Context(), proxy, "test-checksum")
require.NotNil(t, deployment)
assert.Equal(t, map[string]string{"workload-class": "mcp-warm"},
deployment.Spec.Template.Spec.NodeSelector)
require.Len(t, deployment.Spec.Template.Spec.Tolerations, 1)
assert.False(t, reconciler.deploymentNeedsUpdate(t.Context(), deployment, proxy, "test-checksum"),
"freshly built deployment must not report drift")

// Clearing the overrides must be detected.
proxy.Spec.ResourceOverrides = nil
assert.True(t, reconciler.deploymentNeedsUpdate(t.Context(), deployment, proxy, "test-checksum"),
"clearing the overrides must be detected as drift")
})

t.Run("podTemplateSpec-supplied scheduling is not false drift", func(t *testing.T) {
t.Parallel()

proxy := &mcpv1beta1.MCPRemoteProxy{
ObjectMeta: metav1.ObjectMeta{Name: "tmpl-sched-proxy", Namespace: "default"},
Spec: mcpv1beta1.MCPRemoteProxySpec{
RemoteURL: "https://mcp.example.com",
PodTemplateSpec: rawPodTemplateSpecJSON(t, `{"spec":{"nodeSelector":{"disk":"ssd"}}}`),
},
}

deployment := reconciler.deploymentForMCPRemoteProxy(t.Context(), proxy, "test-checksum")
require.NotNil(t, deployment)
assert.Equal(t, map[string]string{"disk": "ssd"}, deployment.Spec.Template.Spec.NodeSelector)
assert.False(t, reconciler.deploymentNeedsUpdate(t.Context(), deployment, proxy, "test-checksum"),
"scheduling from podTemplateSpec must not be mistaken for drift against empty overrides")
})
}
43 changes: 43 additions & 0 deletions cmd/thv-operator/controllers/mcpserver_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -1039,6 +1039,40 @@ func logOBOSecretEnvVarError(ctx context.Context, err error) {
"see the referenced MCPExternalAuthConfig status for details")
}

// proxyDeploymentScheduling returns the scheduling constraints for the proxy pod
// from resourceOverrides.proxyDeployment. Takes ResourceOverrides directly because
// MCPServer and MCPRemoteProxy share that type, so both get identical behaviour
// from one implementation.
//
// The operator sets no proxy scheduling of its own, so these are authoritative
// rather than merged. Used by both the deployment builders and their
// deploymentNeedsUpdate drift checks so construction and comparison cannot
// diverge.
func proxyDeploymentScheduling(
overrides *mcpv1beta1.ResourceOverrides,
) (map[string]string, []corev1.Toleration, *corev1.Affinity) {
if overrides == nil || overrides.ProxyDeployment == nil {
return nil, nil, nil
}
o := overrides.ProxyDeployment
return o.NodeSelector, o.Tolerations, o.Affinity
}

// proxySchedulingNeedsUpdate reports whether the Deployment's proxy pod scheduling
// has drifted from what resourceOverrides.proxyDeployment asks for. Shared by both
// controllers' drift checks; equality.Semantic treats nil and empty as equal so an
// unset override does not read as perpetual drift.
func proxySchedulingNeedsUpdate(
deployment *appsv1.Deployment,
overrides *mcpv1beta1.ResourceOverrides,
) bool {
wantNodeSelector, wantTolerations, wantAffinity := proxyDeploymentScheduling(overrides)
podSpec := deployment.Spec.Template.Spec
return !equality.Semantic.DeepEqual(podSpec.NodeSelector, wantNodeSelector) ||
!equality.Semantic.DeepEqual(podSpec.Tolerations, wantTolerations) ||
!equality.Semantic.DeepEqual(podSpec.Affinity, wantAffinity)
}

// deploymentForMCPServer returns a MCPServer Deployment object
//
//nolint:gocyclo
Expand Down Expand Up @@ -1381,6 +1415,7 @@ func (r *MCPServerReconciler) deploymentForMCPServer(
env = ctrlutil.EnsureRequiredEnvVars(ctx, env)

imagePullSecrets := r.imagePullSecretsForMCPServer(m)
proxyNodeSelector, proxyTolerations, proxyAffinity := proxyDeploymentScheduling(m.Spec.ResourceOverrides)

dep := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -1402,6 +1437,9 @@ func (r *MCPServerReconciler) deploymentForMCPServer(
Spec: corev1.PodSpec{
ServiceAccountName: ctrlutil.ProxyRunnerServiceAccountName(m.Name),
ImagePullSecrets: imagePullSecrets,
NodeSelector: proxyNodeSelector,
Tolerations: proxyTolerations,
Affinity: proxyAffinity,
TerminationGracePeriodSeconds: int64Ptr(defaultTerminationGracePeriodSeconds),
Containers: []corev1.Container{{
Image: getToolhiveRunnerImage(),
Expand Down Expand Up @@ -1955,6 +1993,11 @@ func (r *MCPServerReconciler) deploymentNeedsUpdate(
return true
}

// Check if the proxy pod scheduling overrides have changed.
if proxySchedulingNeedsUpdate(deployment, mcpServer.Spec.ResourceOverrides) {
return true
}

// Check if the resource requirements have changed
if !equality.Semantic.DeepEqual(container.Resources, resourceRequirementsForMCPServer(mcpServer)) {
return true
Expand Down
Loading