direct: make migration ignore direct-only resources

.nextchanges/bundles/migrate-direct-only-resources.md

@@ -0,0 +1 @@
+When migrating a bundle to the direct deployment engine, resources that only the direct engine supports (e.g. instance pools, catalogs) are now skipped by the deploy that migrates the state instead of failing it. They are created by the next deploy, which runs on the migrated state.

acceptance/bundle/migrate/added-direct-only/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+ name: test-bundle
+
+resources:
+ #instance_pools: {pool: {instance_pool_name: "My Pool", node_type_id: "Standard_DS3_v2", idle_instance_autotermination_minutes: 60}}
+ jobs:
+ job_a: {name: "Job A"}

acceptance/bundle/migrate/added-direct-only/output.txt

@@ -0,0 +1,63 @@
+
+=== Deploy with terraform (instance_pools is commented out; terraform cannot deploy it)
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Add a direct-only resource to the config without deploying it
+>>> update_file.py databricks.yml #instance_pools instance_pools
+
+=== Deploying on terraform rejects it, which is what sends the user to migrate
+>>> musterr [CLI] bundle deploy
+Error: Instance Pool resources are only supported with direct deployment mode
+ in databricks.yml:5:19
+
+Instance Pool resources require direct deployment mode. Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use instance_pool resources.
+Learn more at https://docs.databricks.com/dev-tools/bundles/direct
+
+
+=== Migration ignores the direct-only resource; it will be created on the next deploy
+>>> [CLI] bundle deployment migrate
+Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json
+
+Validate the migration by running "databricks bundle plan", there should be no actions planned.
+
+The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy".
+
+To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/default/resources.json and rename [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate
+
+
+=== After migration: plan shows the direct-only resource as "to add"
+>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
+create instance_pools.pool
+
+Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged
+
+=== Deploy creates it; verify via recorded requests
+>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //instance-pools/create
+{
+ "headers": {
+ "User-Agent": [
+ "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat"
+ ]
+ },
+ "method": "POST",
+ "path": "/api/2.0/instance-pools/create",
+ "body": {
+ "idle_instance_autotermination_minutes": 60,
+ "instance_pool_name": "My Pool",
+ "node_type_id": "Standard_DS3_v2"
+ }
+}
+
+=== No further actions planned
+>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged

acceptance/bundle/migrate/added-direct-only/script

@@ -0,0 +1,26 @@
+export DATABRICKS_BUNDLE_ENGINE=terraform
+
+title "Deploy with terraform (instance_pools is commented out; terraform cannot deploy it)"
+trace $CLI bundle deploy
+
+title "Add a direct-only resource to the config without deploying it"
+trace update_file.py databricks.yml "#instance_pools" "instance_pools"
+
+title "Deploying on terraform rejects it, which is what sends the user to migrate"
+trace musterr $CLI bundle deploy
+
+title "Migration ignores the direct-only resource; it will be created on the next deploy"
+trace $CLI bundle deployment migrate
+
+title "After migration: plan shows the direct-only resource as \"to add\""
+trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan
+
+title "Deploy creates it; verify via recorded requests"
+rm out.requests.txt
+trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy
+trace print_requests.py //instance-pools/create
+
+title "No further actions planned"
+trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan
+
+rm out.requests.txt

acceptance/bundle/migrate/auto-migrate-direct-only-envvar/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+ name: test-bundle
+
+resources:
+ #instance_pools: {pool: {instance_pool_name: "My Pool", node_type_id: "Standard_DS3_v2", idle_instance_autotermination_minutes: 60}}
+ jobs:
+ job_a: {name: "Job A"}

acceptance/bundle/migrate/auto-migrate-direct-only-envvar/output.txt

@@ -0,0 +1,55 @@
+
+=== Deploy with terraform (instance_pools is commented out; terraform cannot deploy it)
+>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Add a direct-only resource to the config
+>>> update_file.py databricks.yml #instance_pools instance_pools
+
+=== Opt in via env var only: deploy migrates the state, ignoring the direct-only resource
+>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy
+Warn: Direct engine requested in DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy.
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+Migrating state to direct deployment engine (opted in via DATABRICKS_BUNDLE_ENGINE environment variable)...
+Migrated 1 resource to direct deployment engine.
+
+>>> print_migration_telemetry
+direct_migrated_via_env true
+
+=== The direct-only resource is created on the next deploy
+>>> [CLI] bundle plan
+create instance_pools.pool
+
+Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //instance-pools/create
+{
+ "headers": {
+ "User-Agent": [
+ "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat"
+ ]
+ },
+ "method": "POST",
+ "path": "/api/2.0/instance-pools/create",
+ "body": {
+ "idle_instance_autotermination_minutes": 60,
+ "instance_pool_name": "My Pool",
+ "node_type_id": "Standard_DS3_v2"
+ }
+}
+
+=== No further actions planned
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged

acceptance/bundle/migrate/auto-migrate-direct-only-envvar/script

@@ -0,0 +1,23 @@
+export DATABRICKS_BUNDLE_ENGINE=
+
+title "Deploy with terraform (instance_pools is commented out; terraform cannot deploy it)"
+trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy
+rm -f out.requests.txt
+
+title "Add a direct-only resource to the config"
+trace update_file.py databricks.yml "#instance_pools" "instance_pools"
+
+title "Opt in via env var only: deploy migrates the state, ignoring the direct-only resource"
+trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy
+trace print_migration_telemetry
+rm -f out.requests.txt
+
+title "The direct-only resource is created on the next deploy"
+trace $CLI bundle plan
+trace $CLI bundle deploy
+trace print_requests.py //instance-pools/create
+
+title "No further actions planned"
+trace $CLI bundle plan
+
+rm -f out.requests.txt

acceptance/bundle/migrate/auto-migrate-direct-only/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+ name: test-bundle
+
+resources:
+ #instance_pools: {pool: {instance_pool_name: "My Pool", node_type_id: "Standard_DS3_v2", idle_instance_autotermination_minutes: 60}}
+ jobs:
+ job_a: {name: "Job A"}

acceptance/bundle/migrate/auto-migrate-direct-only/output.txt

@@ -0,0 +1,65 @@
+
+=== Deploy with terraform (instance_pools is commented out; terraform cannot deploy it)
+>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Add a direct-only resource and opt in to the direct engine via config
+>>> update_file.py databricks.yml #instance_pools instance_pools
+
+>>> update_file.py databricks.yml name: test-bundle name: test-bundle
+ engine: direct
+
+=== Plan skips the direct-only resource, since this run still uses terraform
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle plan -o json
+json.plan.resources.instance_pools.pool.action = "skip";
+
+=== Deploy migrates the state, skipping the direct-only resource
+>>> [CLI] bundle deploy
+Warn: Direct engine requested in bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy.
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+Migrating state to direct deployment engine (opted in via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)...
+Migrated 1 resource to direct deployment engine.
+
+>>> print_migration_telemetry
+direct_migrated_via_config true
+
+=== The direct-only resource is created on the next deploy
+>>> [CLI] bundle plan
+create instance_pools.pool
+
+Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //instance-pools/create
+{
+ "headers": {
+ "User-Agent": [
+ "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat"
+ ]
+ },
+ "method": "POST",
+ "path": "/api/2.0/instance-pools/create",
+ "body": {
+ "idle_instance_autotermination_minutes": 60,
+ "instance_pool_name": "My Pool",
+ "node_type_id": "Standard_DS3_v2"
+ }
+}
+
+=== No further actions planned
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged

acceptance/bundle/migrate/auto-migrate-direct-only/script

@@ -0,0 +1,28 @@
+export DATABRICKS_BUNDLE_ENGINE=
+
+title "Deploy with terraform (instance_pools is commented out; terraform cannot deploy it)"
+trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy
+rm -f out.requests.txt
+
+title "Add a direct-only resource and opt in to the direct engine via config"
+trace update_file.py databricks.yml "#instance_pools" "instance_pools"
+trace update_file.py databricks.yml "name: test-bundle" $'name: test-bundle\n engine: direct'
+
+title "Plan skips the direct-only resource, since this run still uses terraform"
+trace $CLI bundle plan
+trace $CLI bundle plan -o json | gron.py | grep instance_pools
+
+title "Deploy migrates the state, skipping the direct-only resource"
+trace $CLI bundle deploy
+trace print_migration_telemetry
+rm -f out.requests.txt
+
+title "The direct-only resource is created on the next deploy"
+trace $CLI bundle plan
+trace $CLI bundle deploy
+trace print_requests.py //instance-pools/create
+
+title "No further actions planned"
+trace $CLI bundle plan
+
+rm -f out.requests.txt

bundle/bundle.go

@@ -172,6 +172,14 @@ type Bundle struct {
 	// When non-empty, only the specified resources are included in deployment.
 	Select []string
 
+	// MigratingToDirect is set when the direct engine is requested but the existing
+	// state still uses terraform, so the state is migrated to the direct engine after
+	// this deploy. Resources that only the direct engine supports are skipped by this
+	// run rather than rejected: terraform cannot deploy them, and since terraform
+	// could never have deployed them they are absent from its state. The next deploy,
+	// which runs on the migrated state, creates them.
+	MigratingToDirect bool
+
 	// SkipLocalFileValidation makes path translation tolerant of missing local files.
 	// When set, TranslatePaths computes workspace paths without verifying files exist.
 	// Used by config-remote-sync: a user may modify resource paths remotely (e.g.,

bundle/config/mutator/validate_direct_only_resources.go

@@ -34,11 +34,18 @@ func isDirectOnly(pluralName string) bool {
 	return hasDirect && !hasTerraform
 }
 
-func (m *validateDirectOnlyResources) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+func (m *validateDirectOnlyResources) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
 	if m.engine.IsDirect() {
  return nil
 	}
 
+	// The state is migrated to the direct engine right after this deploy, so
+	// direct-only resources are skipped by this run instead of rejected.
+	// BundleToTerraformWithDynValue logs each resource it skips.
+	if b.MigratingToDirect {
+ return nil
+	}
+
 	var diags diag.Diagnostics
 	for _, group := range b.Config.Resources.AllResources() {
  if len(group.Resources) == 0 {

bundle/deploy/terraform/convert.go

@@ -8,11 +8,17 @@ import (
 	"github.com/databricks/cli/bundle/deploy/terraform/tfdyn"
 	"github.com/databricks/cli/bundle/internal/tf/schema"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/log"
 )
 
 // BundleToTerraformWithDynValue converts resources in a bundle configuration
 // to the equivalent Terraform JSON representation.
-func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema.Root, error) {
+//
+// Resource types without a terraform converter are only supported by the direct
+// engine. They are an error here, except when skipUnsupported is set: the state is
+// migrated to the direct engine right after this deploy, so they are skipped by
+// this run and deployed by the next one. See bundle.Bundle.MigratingToDirect.
+func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value, skipUnsupported bool) (*schema.Root, error) {
 	tfroot := schema.NewRoot()
 	tfroot.Provider = schema.NewProviders()
 
@@ -49,7 +55,11 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema
  // Lookup the converter based on the resource type.
  c, ok := tfdyn.GetConverter(typ)
  if !ok {
- return dyn.InvalidValue, fmt.Errorf("no converter for resource type %s", typ)
+ if !skipUnsupported {
+ return dyn.InvalidValue, fmt.Errorf("no converter for resource type %s", typ)
+ }
+ log.Infof(ctx, "%s.%s: skipping, only supported by the direct engine; will be deployed after the state is migrated", typ, key)
+ return v, dyn.ErrSkip
  }
 
  // Convert resource to Terraform representation.

bundle/deploy/terraform/convert_test.go

@@ -21,7 +21,7 @@ import (
 func produceTerraformConfiguration(t *testing.T, config config.Root) *schema.Root {
 	vin, err := convert.FromTyped(config, dyn.NilValue)
 	require.NoError(t, err)
-	out, err := BundleToTerraformWithDynValue(t.Context(), vin)
+	out, err := BundleToTerraformWithDynValue(t.Context(), vin, false)
 	require.NoError(t, err)
 	return out
 }
@@ -588,7 +588,7 @@ func TestBundleToTerraformDeletedResources(t *testing.T) {
 
 	vin, err := convert.FromTyped(config, dyn.NilValue)
 	require.NoError(t, err)
-	out, err := BundleToTerraformWithDynValue(t.Context(), vin)
+	out, err := BundleToTerraformWithDynValue(t.Context(), vin, false)
 	require.NoError(t, err)
 
 	_, ok := out.Resource.Job["my_job1"]

bundle/deploy/terraform/lifecycle_test.go

@@ -45,7 +45,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) {
  }, nil),
  }, nil)
 
- tfroot, err := BundleToTerraformWithDynValue(ctx, vin)
+ tfroot, err := BundleToTerraformWithDynValue(ctx, vin, false)
  require.NoError(t, err)
 
  bytes, err := json.Marshal(tfroot.Resource)