Normalize clusters in jobs according to Terraform provider logic

acceptance/bundle/integration_whl/base/out.test.toml

@@ -3,4 +3,4 @@ Cloud = false
 CloudSlow = true
 
 [EnvMatrix]
- DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+ DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/integration_whl/custom_params/out.test.toml

@@ -3,4 +3,4 @@ Cloud = false
 CloudSlow = true
 
 [EnvMatrix]
- DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+ DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/integration_whl/interactive_cluster/out.test.toml

@@ -3,4 +3,4 @@ Cloud = false
 CloudSlow = true
 
 [EnvMatrix]
- DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+ DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/integration_whl/interactive_single_user/out.test.toml

@@ -3,4 +3,4 @@ Cloud = false
 CloudSlow = true
 
 [EnvMatrix]
- DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+ DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/integration_whl/wrapper/out.test.toml

@@ -6,4 +6,4 @@ CloudSlow = true
  gcp = false
 
 [EnvMatrix]
- DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+ DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/integration_whl/wrapper/test.toml

@@ -1,4 +1,2 @@
 # Temporarily disabling due to DBR release breakage.
 CloudEnvs.gcp = false
-
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # Error: deploying jobs.some_other_job: creating: Method=Jobs.Create *retries.Err *apierr.APIError StatusCode=400 ErrorCode="INVALID_PARAMETER_VALUE" Message="The field 'node_type_id' cannot be supplied when an instance pool ID is provided."

acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml

@@ -6,4 +6,4 @@ CloudSlow = true
  gcp = false
 
 [EnvMatrix]
- DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+ DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/integration_whl/wrapper_custom_params/test.toml

@@ -1,4 +1,2 @@
 # Temporarily disabling due to DBR release breakage.
 CloudEnvs.gcp = false
-
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # Error: deploying jobs.some_other_job: creating: Method=Jobs.Create *retries.Err *apierr.APIError StatusCode=400 ErrorCode="INVALID_PARAMETER_VALUE" Message="The field 'node_type_id' cannot be supplied when an instance pool ID is provided."

bundle/config/mutator/resourcemutator/job_cluster_fixups.go

@@ -0,0 +1,78 @@
+package resourcemutator
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+)
+
+type jobClustersFixups struct{}
+
+func JobClustersFixups() bundle.Mutator {
+	return &jobClustersFixups{}
+}
+
+func (m *jobClustersFixups) Name() string {
+	return "JobClustersFixups"
+}
+
+func (m *jobClustersFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	for _, job := range b.Config.Resources.Jobs {
+ if job == nil {
+ continue
+ }
+ // TODO: we should raise a warning when user specify both InstancePoolId and NodeTypeId since it's illegal.
+ // Once we remove TF backend and had warning for some time, we can remove this transformation, the backend
+ // will reject such configs.
+ prepareJobSettingsForUpdate(&job.JobSettings)
+	}
+
+	return nil
+}
+
+// Copied from
+// https://github.com/databricks/terraform-provider-databricks/blob/a8c92bb/clusters/resource_cluster.go
+// https://github.com/databricks/terraform-provider-databricks/blob/a8c92bb/clusters/clusters_api.go#L440
+func ModifyRequestOnInstancePool(c *compute.ClusterSpec) {
+	// Instance profile id does not exist or not set
+	if c.InstancePoolId == "" {
+ // Worker must use an instance pool if driver uses an instance pool,
+ // therefore empty the computed value for driver instance pool.
+ c.DriverInstancePoolId = ""
+ return
+	}
+	if c.AwsAttributes != nil {
+ // Reset AwsAttributes
+ awsAttributes := compute.AwsAttributes{
+ InstanceProfileArn: c.AwsAttributes.InstanceProfileArn,
+ }
+ c.AwsAttributes = &awsAttributes
+	}
+	if c.AzureAttributes != nil {
+ c.AzureAttributes = &compute.AzureAttributes{}
+	}
+	if c.GcpAttributes != nil {
+ gcpAttributes := compute.GcpAttributes{
+ GoogleServiceAccount: c.GcpAttributes.GoogleServiceAccount,
+ }
+ c.GcpAttributes = &gcpAttributes
+	}
+	c.EnableElasticDisk = false
+	c.NodeTypeId = ""
+	c.DriverNodeTypeId = ""
+}
+
+// Copied https://github.com/databricks/terraform-provider-databricks/blob/a8c92bb130def431b3fadd9fd533c463e8d4813b/jobs/resource_job.go#L1016
+func prepareJobSettingsForUpdate(js *jobs.JobSettings) {
+	for _, task := range js.Tasks {
+ if task.NewCluster != nil {
+ ModifyRequestOnInstancePool(task.NewCluster)
+ }
+	}
+	for ind := range js.JobClusters {
+ ModifyRequestOnInstancePool(&js.JobClusters[ind].NewCluster)
+	}
+}

bundle/config/mutator/resourcemutator/resource_mutator.go

@@ -158,6 +158,9 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) {
  // Updates (dynamic): resources.dashboards.*.serialized_dashboard
  // Drops (dynamic): resources.dashboards.*.file_path
  ConfigureDashboardSerializedDashboard(),
+
+ // Reads and updates (typed): resources.jobs.*.**
+ JobClustersFixups(),
 	)
 }