bundle: add per-resource deploy_targets to scope resources to targets (draft for #2878)

acceptance/bundle/deploy_targets/databricks.yml

@@ -0,0 +1,23 @@
+bundle:
+  name: deploy-targets
+
+# Each job is defined once at the top level. deploy_targets restricts a job to
+# a subset of targets. A job with no deploy_targets is deployed to every target.
+resources:
+  jobs:
+    shared_job:
+      name: shared_job
+
+    dev_and_staging_job:
+      name: dev_and_staging_job
+      deploy_targets: [dev, staging]
+
+    prod_only_job:
+      name: prod_only_job
+      deploy_targets: [prod]
+
+targets:
+  dev:
+    default: true
+  staging:
+  prod:

acceptance/bundle/deploy_targets/output.txt

@@ -0,0 +1,18 @@
+
+>>> [CLI] bundle validate -o json -t dev
+[
+  "dev_and_staging_job",
+  "shared_job"
+]
+
+>>> [CLI] bundle validate -o json -t staging
+[
+  "dev_and_staging_job",
+  "shared_job"
+]
+
+>>> [CLI] bundle validate -o json -t prod
+[
+  "prod_only_job",
+  "shared_job"
+]

acceptance/bundle/deploy_targets/script

@@ -0,0 +1,3 @@
+trace $CLI bundle validate -o json -t dev | jq '.resources.jobs | keys'
+trace $CLI bundle validate -o json -t staging | jq '.resources.jobs | keys'
+trace $CLI bundle validate -o json -t prod | jq '.resources.jobs | keys'

acceptance/bundle/deploy_targets/test.toml

@@ -0,0 +1,3 @@
+Local = true
+Cloud = false
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]

bundle/config/mutator/resourcemutator/filter_deploy_targets.go

@@ -0,0 +1,87 @@
+package resourcemutator
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/merge"
+)
+
+type filterDeployTargets struct{}
+
+// FilterDeployTargets drops resources that declare a deploy_targets list which
+// does not include the selected target. A resource with an empty (or absent)
+// deploy_targets list is left untouched and deploys to every target, matching
+// the historical behavior of a top-level resource.
+//
+// This lets a single top-level resource definition be scoped to a subset of
+// targets without nesting its body under targets.<name>.resources. It runs
+// after resources are fully materialized (after PythonMutator) so that
+// dynamically added resources are also filtered, and before validation so that
+// required-field checks only see resources that will actually be deployed.
+func FilterDeployTargets() bundle.Mutator {
+	return &filterDeployTargets{}
+}
+
+func (m *filterDeployTargets) Name() string {
+	return "FilterDeployTargets"
+}
+
+func (m *filterDeployTargets) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
+	target := b.Config.Bundle.Target
+
+	// Compute, per resource type, the names to drop for the selected target.
+	// Reading the typed structs is sufficient here because deploy_targets is a
+	// plain string slice; the actual removal happens on the dyn tree below so it
+	// deletes from the real resource maps rather than a copy.
+	dropByType := map[string][]string{}
+	for _, group := range b.Config.Resources.AllResources() {
+		typeName := group.Description.PluralName
+		for name, resource := range group.Resources {
+			deployTargets := resource.GetDeployTargets()
+			if len(deployTargets) == 0 {
+				continue
+			}
+			if !slicesContains(deployTargets, target) {
+				dropByType[typeName] = append(dropByType[typeName], name)
+			}
+		}
+	}
+
+	if len(dropByType) == 0 {
+		return nil
+	}
+
+	err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
+		newRoot := root
+		for typeName, names := range dropByType {
+			typePath := dyn.NewPath(dyn.Key("resources"), dyn.Key(typeName))
+			var err error
+			newRoot, err = dyn.MapByPath(newRoot, typePath, func(_ dyn.Path, resources dyn.Value) (dyn.Value, error) {
+				return merge.AntiSelect(resources, names)
+			})
+			if err != nil {
+				return dyn.InvalidValue, err
+			}
+		}
+		return newRoot, nil
+	})
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	return nil
+}
+
+// slicesContains reports whether target is present in list. Kept local to avoid
+// widening imports for a single call.
+func slicesContains(list []string, target string) bool {
+	for _, v := range list {
+		if v == target {
+			return true
+		}
+	}
+	return false
+}

bundle/config/mutator/resourcemutator/filter_deploy_targets_test.go

@@ -0,0 +1,118 @@
+package resourcemutator_test
+
+import (
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/mutator/resourcemutator"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/databricks/databricks-sdk-go/service/pipelines"
+	"github.com/stretchr/testify/assert"
+)
+
+func filterDeployTargetsBundle(target string) *bundle.Bundle {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					// No deploy_targets: deployed everywhere.
+					"everywhere": {
+						JobSettings: jobs.JobSettings{Name: "everywhere"},
+					},
+					// Restricted to dev and staging.
+					"dev_and_staging": {
+						BaseResource: resources.BaseResource{DeployTargets: []string{"dev", "staging"}},
+						JobSettings:  jobs.JobSettings{Name: "dev_and_staging"},
+					},
+					// Restricted to prod only.
+					"prod_only": {
+						BaseResource: resources.BaseResource{DeployTargets: []string{"prod"}},
+						JobSettings:  jobs.JobSettings{Name: "prod_only"},
+					},
+				},
+				Pipelines: map[string]*resources.Pipeline{
+					// A different resource type, restricted to prod, to confirm
+					// filtering is applied uniformly across resource kinds.
+					"pipeline_prod_only": {
+						BaseResource:   resources.BaseResource{DeployTargets: []string{"prod"}},
+						CreatePipeline: pipelines.CreatePipeline{Name: "pipeline_prod_only"},
+					},
+				},
+			},
+		},
+	}
+	b.Config.Bundle.Target = target
+	return b
+}
+
+func TestFilterDeployTargetsDev(t *testing.T) {
+	b := filterDeployTargetsBundle("dev")
+
+	diags := bundle.Apply(t.Context(), b, resourcemutator.FilterDeployTargets())
+	assert.NoError(t, diags.Error())
+
+	// everywhere has no list so it stays; dev_and_staging includes dev so it
+	// stays; prod_only does not include dev so it is dropped.
+	assert.Contains(t, b.Config.Resources.Jobs, "everywhere")
+	assert.Contains(t, b.Config.Resources.Jobs, "dev_and_staging")
+	assert.NotContains(t, b.Config.Resources.Jobs, "prod_only")
+	assert.Len(t, b.Config.Resources.Jobs, 2)
+
+	// The prod-only pipeline is dropped in dev as well.
+	assert.NotContains(t, b.Config.Resources.Pipelines, "pipeline_prod_only")
+	assert.Empty(t, b.Config.Resources.Pipelines)
+}
+
+func TestFilterDeployTargetsProd(t *testing.T) {
+	b := filterDeployTargetsBundle("prod")
+
+	diags := bundle.Apply(t.Context(), b, resourcemutator.FilterDeployTargets())
+	assert.NoError(t, diags.Error())
+
+	// everywhere stays; dev_and_staging does not include prod so it is dropped;
+	// prod_only includes prod so it stays.
+	assert.Contains(t, b.Config.Resources.Jobs, "everywhere")
+	assert.NotContains(t, b.Config.Resources.Jobs, "dev_and_staging")
+	assert.Contains(t, b.Config.Resources.Jobs, "prod_only")
+	assert.Len(t, b.Config.Resources.Jobs, 2)
+
+	// The prod-only pipeline is kept in prod.
+	assert.Contains(t, b.Config.Resources.Pipelines, "pipeline_prod_only")
+	assert.Len(t, b.Config.Resources.Pipelines, 1)
+}
+
+func TestFilterDeployTargetsUnlistedTarget(t *testing.T) {
+	// A target that appears in no deploy_targets list keeps only the resources
+	// that have no list at all.
+	b := filterDeployTargetsBundle("qa")
+
+	diags := bundle.Apply(t.Context(), b, resourcemutator.FilterDeployTargets())
+	assert.NoError(t, diags.Error())
+
+	assert.Contains(t, b.Config.Resources.Jobs, "everywhere")
+	assert.NotContains(t, b.Config.Resources.Jobs, "dev_and_staging")
+	assert.NotContains(t, b.Config.Resources.Jobs, "prod_only")
+	assert.Len(t, b.Config.Resources.Jobs, 1)
+	assert.Empty(t, b.Config.Resources.Pipelines)
+}
+
+func TestFilterDeployTargetsNoFieldsSet(t *testing.T) {
+	// When no resource declares deploy_targets, the mutator makes no changes.
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"a": {JobSettings: jobs.JobSettings{Name: "a"}},
+					"b": {JobSettings: jobs.JobSettings{Name: "b"}},
+				},
+			},
+		},
+	}
+	b.Config.Bundle.Target = "dev"
+
+	diags := bundle.Apply(t.Context(), b, resourcemutator.FilterDeployTargets())
+	assert.NoError(t, diags.Error())
+	assert.Len(t, b.Config.Resources.Jobs, 2)
+}

bundle/config/resources.go

@@ -64,6 +64,10 @@ type ConfigResource interface {
 
 	// GetLifecycle returns the lifecycle settings for the resource.
 	GetLifecycle() resources.LifecycleConfig
+
+	// GetDeployTargets returns the targets the resource is restricted to, or an
+	// empty slice when the resource is not target-scoped.
+	GetDeployTargets() []string
 }
 
 // ResourceGroup represents a group of resources of the same type.

bundle/config/resources/base.go

@@ -6,9 +6,25 @@ type BaseResource struct {
 	ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
 	URL            string         `json:"url,omitempty" bundle:"internal"`
 	Lifecycle      Lifecycle      `json:"lifecycle,omitempty"`
+
+	// DeployTargets optionally restricts the resource to a subset of targets.
+	// When empty (the default) the resource is deployed to every target, matching
+	// the historical behavior of a top-level resource. When set, the resource is
+	// only deployed to a target whose name appears in the list; for any other
+	// target it is dropped from the configuration before deployment. This lets a
+	// single top-level definition be scoped to specific targets without nesting
+	// the body under targets.<name>.resources. It is a bundle-only field and is
+	// never emitted to the deployment engine.
+	DeployTargets []string `json:"deploy_targets,omitempty"`
 }
 
 // GetLifecycle returns the lifecycle settings for the resource.
 func (b *BaseResource) GetLifecycle() LifecycleConfig {
 	return b.Lifecycle
 }
+
+// GetDeployTargets returns the targets the resource is restricted to, or an
+// empty slice when the resource is not target-scoped.
+func (b *BaseResource) GetDeployTargets() []string {
+	return b.DeployTargets
+}

bundle/config/resources/external_location.go

@@ -17,7 +17,8 @@ type ExternalLocation struct {
 	ID             string         `json:"id,omitempty" bundle:"readonly"`
 	ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
 	// Note: We intentionally don't include BaseResource.URL here to avoid conflict with Url field below
-	Lifecycle Lifecycle `json:"lifecycle,omitempty"`
+	Lifecycle     Lifecycle `json:"lifecycle,omitempty"`
+	DeployTargets []string  `json:"deploy_targets,omitempty"`
 
 	catalog.CreateExternalLocation
 
@@ -67,6 +68,12 @@ func (e *ExternalLocation) GetLifecycle() LifecycleConfig {
 	return e.Lifecycle
 }
 
+// GetDeployTargets returns the targets the resource is restricted to, or an
+// empty slice when the resource is not target-scoped.
+func (e *ExternalLocation) GetDeployTargets() []string {
+	return e.DeployTargets
+}
+
 func (e *ExternalLocation) UnmarshalJSON(b []byte) error {
 	return marshal.Unmarshal(b, e)
 }

bundle/direct/dresources/type_test.go

@@ -59,6 +59,7 @@ var knownMissingInRemoteType = map[string][]string{
 // commonMissingInStateType lists fields that are commonly missing across all resource types.
 // These are bundle-specific fields that exist in InputType but not in StateType.
 var commonMissingInStateType = []string{
+	"deploy_targets",
 	"grants",
 	"lifecycle",
 	"permissions",