[bundle] Report deploy operations to Deployment Metadata Service (step 5)

acceptance/bin/print_requests.py

@@ -168,7 +168,7 @@ def main():
     if not requests_file.exists():
         sys.exit(f"File {requests_file} not found")
 
-    with open(requests_file) as fobj:
+    with open(requests_file, encoding="utf-8") as fobj:
         data = fobj.read()
 
     if not data:

acceptance/bundle/dms/add-resources/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: dms-add-resources
+
+resources:
+  jobs:
+    job_a:
+      name: job-a

acceptance/bundle/dms/add-resources/output.txt

@@ -0,0 +1,127 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-add-resources/default/files...
+Deploying resources...
+Deployment complete!
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-add-resources/default/files...
+Deploying resources...
+Deployment complete!
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+>>> print_requests.py --get //bundle ^//workspace-files ^//import-file
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "q": {
+    "deployment_id": "[UUID]"
+  },
+  "body": {
+    "target_name": "default"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations",
+  "q": {
+    "resource_key": "jobs.job_a"
+  },
+  "body": {
+    "action_type": "OPERATION_ACTION_TYPE_CREATE",
+    "resource_id": "[NUMID]",
+    "resource_key": "jobs.job_a",
+    "state": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-add-resources/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "job-a",
+      "queue": {
+        "enabled": true
+      }
+    },
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]/resources"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "2"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/operations",
+  "q": {
+    "resource_key": "jobs.job_b"
+  },
+  "body": {
+    "action_type": "OPERATION_ACTION_TYPE_CREATE",
+    "resource_id": "[NUMID]",
+    "resource_key": "jobs.job_b",
+    "state": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-add-resources/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "job-b",
+      "queue": {
+        "enabled": true
+      }
+    },
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]/resources"
+}

acceptance/bundle/dms/add-resources/script

@@ -0,0 +1,34 @@
+# Deploy a fresh bundle with one job — DMS records a single CREATE operation.
+trace $CLI bundle deploy
+
+# Drop the local cache so the next deploy re-reads state from DMS via ListResources.
+rm -rf .databricks
+
+# Add a second job and redeploy. The new deploy should:
+#  - read existing state from DMS,
+#  - skip job_a (no change),
+#  - create job_b and report exactly one CREATE operation under version 2.
+cat > databricks.yml << 'EOF'
+bundle:
+  name: dms-add-resources
+
+resources:
+  jobs:
+    job_a:
+      name: job-a
+    job_b:
+      name: job-b
+EOF
+trace $CLI bundle deploy
+
+# Drop the cache one more time and run plan — both resources should show as
+# unchanged because their reported state matches the local config.
+rm -rf .databricks
+trace $CLI bundle plan
+
+# Inspect the recorded DMS traffic. Expect:
+#  - Deploy 1: CreateDeployment + CreateVersion(1) + CreateOperation(job_a, CREATE) + CompleteVersion(1)
+#  - Deploy 2: ListResources + GetDeployment + CreateVersion(2) + CreateOperation(job_b, CREATE) + CompleteVersion(2)
+#  - Plan:    ListResources
+trace print_requests.py --get //bundle ^//workspace-files ^//import-file
+print_requests.py --get > /dev/null 2>&1 || true

acceptance/bundle/dms/add-resources/test.toml

@@ -0,0 +1,6 @@
+Ignore = [".databricks"]
+
+# Exercise DMS-side operation reporting across two deploys:
+#  - first deploy creates job_a and reports one CREATE operation,
+#  - second deploy adds job_b without retouching job_a,
+#  - subsequent plan reads from DMS and sees both jobs already deployed.

acceptance/bundle/dms/deploy-error/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: dms-deploy-error
+
+resources:
+  jobs:
+    test_job:
+      name: test-job

acceptance/bundle/dms/deploy-error/output.txt

@@ -0,0 +1,56 @@
+
+>>> musterr [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-deploy-error/default/files...
+Deploying resources...
+Error: cannot create resources.jobs.test_job: Invalid job configuration. (400 INVALID_PARAMETER_VALUE)
+
+Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create
+HTTP Status: 400 Bad Request
+API error_code: INVALID_PARAMETER_VALUE
+API message: Invalid job configuration.
+
+
+>>> print_requests.py --get //bundle ^//workspace-files ^//import-file
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "q": {
+    "deployment_id": "[UUID]"
+  },
+  "body": {
+    "target_name": "default"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations",
+  "q": {
+    "resource_key": "jobs.test_job"
+  },
+  "body": {
+    "action_type": "OPERATION_ACTION_TYPE_CREATE",
+    "error_message": "Invalid job configuration.",
+    "resource_id": "",
+    "resource_key": "jobs.test_job",
+    "status": "OPERATION_STATUS_FAILED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_FAILURE"
+  }
+}

acceptance/bundle/dms/deploy-error/script

@@ -0,0 +1,10 @@
+# Deploy with a server-side job creation failure injected via test.toml.
+# Expect the CLI to:
+#  - acquire the DMS lock (CreateDeployment + CreateVersion(1)),
+#  - attempt the job creation and fail,
+#  - report a FAILED CREATE operation carrying the error message,
+#  - release the lock with completion_reason=VERSION_COMPLETE_FAILURE.
+trace musterr $CLI bundle deploy
+
+trace print_requests.py --get //bundle ^//workspace-files ^//import-file
+print_requests.py --get > /dev/null 2>&1 || true

acceptance/bundle/dms/deploy-error/test.toml

@@ -0,0 +1,10 @@
+Ignore = [".databricks"]
+
+# Inject a server-side failure on job creation and confirm that:
+#  - the failed operation is reported to DMS with OPERATION_STATUS_FAILED,
+#  - the lock is released with VERSION_COMPLETE_FAILURE.
+
+[[Server]]
+Pattern = "POST /api/2.2/jobs/create"
+Response.StatusCode = 400
+Response.Body = '{"error_code": "INVALID_PARAMETER_VALUE", "message": "Invalid job configuration."}'

acceptance/bundle/dms/plan-and-summary/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: dms-plan-and-summary
+
+resources:
+  jobs:
+    test_job:
+      name: test-job

acceptance/bundle/dms/plan-and-summary/output.txt

@@ -0,0 +1,85 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-plan-and-summary/default/files...
+Deploying resources...
+Deployment complete!
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+>>> [CLI] bundle summary
+Name: dms-plan-and-summary
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/dms-plan-and-summary/default
+Resources:
+  Jobs:
+    test_job:
+      Name: test-job
+      URL:  [DATABRICKS_URL]/jobs/[NUMID]?o=[NUMID]
+
+>>> print_requests.py --get //bundle ^//workspace-files ^//import-file
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "q": {
+    "deployment_id": "[UUID]"
+  },
+  "body": {
+    "target_name": "default"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations",
+  "q": {
+    "resource_key": "jobs.test_job"
+  },
+  "body": {
+    "action_type": "OPERATION_ACTION_TYPE_CREATE",
+    "resource_id": "[NUMID]",
+    "resource_key": "jobs.test_job",
+    "state": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-plan-and-summary/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "test-job",
+      "queue": {
+        "enabled": true
+      }
+    },
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]/resources"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]/resources"
+}