bundle: warn when a task parameter is an empty string under the terraform engine

.nextchanges/bundles/empty-task-parameters-warning.md

@@ -0,0 +1 @@
+Warn at validate time when a task parameter is an empty string and the bundle deploys with the Terraform engine. Such deploys fail while creating the job with a provider panic (`parameters[<nil>] is not a string`); the warning names the config location before terraform runs.

acceptance/bundle/variables/empty-in-task-params-variants/databricks.yml

@@ -0,0 +1,48 @@
+bundle:
+  name: empty-variants
+
+variables:
+  user_prefix:
+    description: Optional prefix, empty everywhere but local dev
+    default: ""
+  derived_prefix:
+    description: Variable whose default is another variable
+    default: ${var.user_prefix}
+  settings:
+    description: Complex variable with an empty string field
+    type: complex
+    default:
+      prefix: ""
+
+resources:
+  jobs:
+    demo_job:
+      name: demo-job
+      tasks:
+        - task_key: main
+          spark_python_task:
+            python_file: ./src/main.py
+            parameters:
+              - "--a"
+              - ${var.user_prefix}
+              - "--b"
+              - ${var.derived_prefix}
+              - "--c"
+              - ${var.settings.prefix}
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
+            node_type_id: i3.xlarge
+            num_workers: 1
+
+targets:
+  short_form_override:
+    default: true
+    variables:
+      user_prefix: ""
+  long_form_override:
+    variables:
+      user_prefix:
+        default: ""
+  null_override:
+    variables:
+      user_prefix:

acceptance/bundle/variables/empty-in-task-params-variants/output.txt

@@ -0,0 +1,55 @@
+
+=== Target with short-form empty override (targets.X.variables.user_prefix: "")
+>>> [CLI] bundle validate -o json -t short_form_override
+[
+  "--a",
+  "",
+  "--b",
+  "",
+  "--c",
+  ""
+]
+
+=== Target with long-form empty override (default: "")
+>>> [CLI] bundle validate -o json -t long_form_override
+[
+  "--a",
+  "",
+  "--b",
+  "",
+  "--c",
+  ""
+]
+
+=== Target with null override (targets.X.variables.user_prefix:)
+>>> errcode [CLI] bundle validate -o json -t null_override
+[
+  "--a",
+  "",
+  "--b",
+  "",
+  "--c",
+  ""
+]
+
+=== Empty value via BUNDLE_VAR env var
+>>> env BUNDLE_VAR_user_prefix= [CLI] bundle validate -o json -t short_form_override
+[
+  "--a",
+  "",
+  "--b",
+  "",
+  "--c",
+  ""
+]
+
+=== Empty value via --var flag
+>>> [CLI] bundle validate -o json -t short_form_override --var=user_prefix=
+[
+  "--a",
+  "",
+  "--b",
+  "",
+  "--c",
+  ""
+]

acceptance/bundle/variables/empty-in-task-params-variants/script

@@ -0,0 +1,14 @@
+title "Target with short-form empty override (targets.X.variables.user_prefix: \"\")"
+trace $CLI bundle validate -o json -t short_form_override | jq '.resources.jobs.demo_job.tasks[0].spark_python_task.parameters'
+
+title "Target with long-form empty override (default: \"\")"
+trace $CLI bundle validate -o json -t long_form_override | jq '.resources.jobs.demo_job.tasks[0].spark_python_task.parameters'
+
+title "Target with null override (targets.X.variables.user_prefix:)"
+trace errcode $CLI bundle validate -o json -t null_override | jq '.resources.jobs.demo_job.tasks[0].spark_python_task.parameters'
+
+title "Empty value via BUNDLE_VAR env var"
+trace env BUNDLE_VAR_user_prefix= $CLI bundle validate -o json -t short_form_override | jq '.resources.jobs.demo_job.tasks[0].spark_python_task.parameters'
+
+title "Empty value via --var flag"
+trace $CLI bundle validate -o json -t short_form_override --var="user_prefix=" | jq '.resources.jobs.demo_job.tasks[0].spark_python_task.parameters'

acceptance/bundle/variables/empty-in-task-params-variants/src/main.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/variables/empty-in-task-params-variants/test.toml

@@ -0,0 +1 @@
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/variables/empty-in-task-params/databricks.yml

@@ -0,0 +1,23 @@
+bundle:
+  name: empty-in-task-params
+
+variables:
+  run_suffix:
+    description: Optional suffix appended to job parameters
+    default: ""
+
+resources:
+  jobs:
+    demo_job:
+      name: demo-job
+      tasks:
+        - task_key: main
+          spark_python_task:
+            python_file: ./src/main.py
+            parameters:
+              - "--suffix"
+              - ${var.run_suffix}
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
+            node_type_id: i3.xlarge
+            num_workers: 1

acceptance/bundle/variables/empty-in-task-params/out.deploy.direct.txt

@@ -0,0 +1,6 @@
+
+>>> errcode [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/empty-in-task-params/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!

acceptance/bundle/variables/empty-in-task-params/out.deploy.terraform.txt

@@ -0,0 +1,23 @@
+
+>>> errcode [CLI] bundle deploy
+Warning: empty parameter in spark_python_task fails deploys that use the Terraform engine
+  at resources.jobs.demo_job.tasks[0].spark_python_task.parameters[1]
+  in databricks.yml:19:17
+
+The Terraform provider reads an empty string element in a parameters list back as nil, and creating the job panics with "parameters[<nil>] is not a string". Remove the empty element or make its value non-empty. The direct engine (bundle.engine: direct) deploys empty parameters correctly.
+
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/empty-in-task-params/default/files...
+Deploying resources...
+Error: terraform apply: exit status 1
+
+Error: cannot create job: panic: task: spark_python_task: parameters: task.0.spark_python_task.0.parameters[<nil>] is not a string
+
+  with databricks_job.demo_job,
+  on bundle.tf.json line 45, in resource.databricks_job.demo_job:
+  45:       }
+
+
+
+Updating deployment state...
+
+Exit code: 1

acceptance/bundle/variables/empty-in-task-params/out.jobs-create-params.direct.json

@@ -0,0 +1,6 @@
+[
+  [
+    "--suffix",
+    ""
+  ]
+]

acceptance/bundle/variables/empty-in-task-params/out.jobs-create-params.terraform.json

@@ -0,0 +1 @@
+[]

acceptance/bundle/variables/empty-in-task-params/out.tf-params.terraform.json

@@ -0,0 +1,4 @@
+[
+  "--suffix",
+  ""
+]

acceptance/bundle/variables/empty-in-task-params/out.validate.direct.txt

@@ -0,0 +1,9 @@
+
+>>> errcode [CLI] bundle validate
+Name: empty-in-task-params
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/empty-in-task-params/default
+
+Validation OK!

acceptance/bundle/variables/empty-in-task-params/out.validate.terraform.txt

@@ -0,0 +1,15 @@
+
+>>> errcode [CLI] bundle validate
+Warning: empty parameter in spark_python_task fails deploys that use the Terraform engine
+  at resources.jobs.demo_job.tasks[0].spark_python_task.parameters[1]
+  in databricks.yml:19:17
+
+The Terraform provider reads an empty string element in a parameters list back as nil, and creating the job panics with "parameters[<nil>] is not a string". Remove the empty element or make its value non-empty. The direct engine (bundle.engine: direct) deploys empty parameters correctly.
+
+Name: empty-in-task-params
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/empty-in-task-params/default
+
+Found 1 warning

acceptance/bundle/variables/empty-in-task-params/output.txt

@@ -0,0 +1,5 @@
+
+=== Validate
+=== Deploy
+=== Parameters sent to jobs/create
+>>> jq -s [.[] | select(.path | test("jobs/create")) | .body.tasks[0].spark_python_task.parameters] out.requests.txt

acceptance/bundle/variables/empty-in-task-params/script

@@ -0,0 +1,16 @@
+title "Validate"
+trace errcode $CLI bundle validate > out.validate.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1
+
+title "Deploy"
+trace errcode $CLI bundle deploy > out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1
+
+title "Parameters sent to jobs/create"
+trace jq -s '[.[] | select(.path | test("jobs/create")) | .body.tasks[0].spark_python_task.parameters]' out.requests.txt > out.jobs-create-params.$DATABRICKS_BUNDLE_ENGINE.json
+
+# The terraform engine also records the parameters as rendered into bundle.tf.json,
+# to show the generated configuration is well formed (the empty string is "").
+if [ -f .databricks/bundle/default/terraform/bundle.tf.json ]; then
+  jq '.resource.databricks_job.demo_job.task[0].spark_python_task.parameters' .databricks/bundle/default/terraform/bundle.tf.json > out.tf-params.terraform.json
+fi
+
+rm out.requests.txt

acceptance/bundle/variables/empty-in-task-params/src/main.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/variables/empty-in-task-params/test.toml

@@ -0,0 +1 @@
+RecordRequests = true

bundle/config/validate/empty_task_parameters.go

@@ -0,0 +1,107 @@
+package validate
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/engine"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+)
+
+// EmptyTaskParameters warns when a task parameter is an empty string and the
+// bundle deploys with the Terraform engine. The Terraform provider reads an
+// empty list element back as nil and job creation panics with
+// "parameters[<nil>] is not a string". The Jobs API accepts empty string
+// parameters and the direct engine deploys them, so this is a warning and it
+// is only emitted for the Terraform engine.
+//
+// Empty parameters usually come from a variable that is intentionally blank
+// in some targets, e.g. a per-developer prefix passed as its own list element.
+func EmptyTaskParameters() bundle.ReadOnlyMutator {
+	return &emptyTaskParameters{}
+}
+
+type emptyTaskParameters struct{ bundle.RO }
+
+func (v *emptyTaskParameters) Name() string {
+	return "validate:empty_task_parameters"
+}
+
+// Apply walks every job task (including tasks nested in a for_each_task) and
+// collects a warning for each empty string element in a parameters list. It
+// returns nothing when the effective engine is direct, because the direct
+// engine deploys empty parameters correctly; the panic is specific to the
+// Terraform provider. It must run after variable references in resources are
+// resolved, so that a parameter written as ${var.x} is seen as its final
+// string value.
+func (v *emptyTaskParameters) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	// Resolve effective engine: config takes precedence over env var.
+	effectiveEngine := b.Config.Bundle.Engine
+	if effectiveEngine == engine.EngineNotSet {
+		if envEngine, err := engine.FromEnv(ctx); err == nil {
+			effectiveEngine = envEngine
+		}
+	}
+	if effectiveEngine.IsDirect() {
+		return nil
+	}
+
+	var diags diag.Diagnostics
+	for key, job := range b.Config.Resources.Jobs {
+		for i, task := range job.Tasks {
+			diags = diags.Extend(checkEmptyTaskParameters(b, &task, fmt.Sprintf("resources.jobs.%s.tasks[%d]", key, i)))
+			if task.ForEachTask != nil {
+				diags = diags.Extend(checkEmptyTaskParameters(b, &task.ForEachTask.Task, fmt.Sprintf("resources.jobs.%s.tasks[%d].for_each_task.task", key, i)))
+			}
+		}
+	}
+	return diags
+}
+
+// checkEmptyTaskParameters returns one warning per empty string element in the
+// given task's parameters list, covering the four task types whose parameters
+// are a plain list of strings (spark_python_task, python_wheel_task,
+// spark_jar_task, spark_submit_task). basePath is the config path of the task,
+// e.g. "resources.jobs.my_job.tasks[0]"; it anchors each warning to the exact
+// element (".<task_type>.parameters[<i>]") so the rendered diagnostic points
+// at the offending line in the user's YAML.
+func checkEmptyTaskParameters(b *bundle.Bundle, task *jobs.Task, basePath string) diag.Diagnostics {
+	var diags diag.Diagnostics
+
+	check := func(params []string, taskType string) {
+		for i, p := range params {
+			if p != "" {
+				continue
+			}
+			path := fmt.Sprintf("%s.%s.parameters[%d]", basePath, taskType, i)
+			diags = append(diags, diag.Diagnostic{
+				Severity: diag.Warning,
+				Summary:  fmt.Sprintf("empty parameter in %s fails deploys that use the Terraform engine", taskType),
+				Detail: "The Terraform provider reads an empty string element in a parameters list back as nil, " +
+					"and creating the job panics with \"parameters[<nil>] is not a string\". " +
+					"Remove the empty element or make its value non-empty. " +
+					"The direct engine (bundle.engine: direct) deploys empty parameters correctly.",
+				Locations: b.Config.GetLocations(path),
+				Paths:     []dyn.Path{dyn.MustPathFromString(path)},
+			})
+		}
+	}
+
+	if task.SparkPythonTask != nil {
+		check(task.SparkPythonTask.Parameters, "spark_python_task")
+	}
+	if task.PythonWheelTask != nil {
+		check(task.PythonWheelTask.Parameters, "python_wheel_task")
+	}
+	if task.SparkJarTask != nil {
+		check(task.SparkJarTask.Parameters, "spark_jar_task")
+	}
+	if task.SparkSubmitTask != nil {
+		check(task.SparkSubmitTask.Parameters, "spark_submit_task")
+	}
+
+	return diags
+}