diff --git a/.nextchanges/bundles/migrate-direct-only-resources.md b/.nextchanges/bundles/migrate-direct-only-resources.md new file mode 100644 index 00000000000..7da46efea12 --- /dev/null +++ b/.nextchanges/bundles/migrate-direct-only-resources.md @@ -0,0 +1 @@ +When migrating a bundle to the direct deployment engine, resources that only the direct engine supports (e.g. instance pools, catalogs) are now skipped by the deploy that migrates the state instead of failing it. They are created by the next deploy, which runs on the migrated state. diff --git a/acceptance/bundle/migrate/added-direct-only/databricks.yml b/acceptance/bundle/migrate/added-direct-only/databricks.yml new file mode 100644 index 00000000000..8f2a40d4f2b --- /dev/null +++ b/acceptance/bundle/migrate/added-direct-only/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + #instance_pools: {pool: {instance_pool_name: "My Pool", node_type_id: "Standard_DS3_v2", idle_instance_autotermination_minutes: 60}} + jobs: + job_a: {name: "Job A"} diff --git a/acceptance/bundle/migrate/added-direct-only/out.test.toml b/acceptance/bundle/migrate/added-direct-only/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/migrate/added-direct-only/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/added-direct-only/output.txt b/acceptance/bundle/migrate/added-direct-only/output.txt new file mode 100644 index 00000000000..13ddf672f43 --- /dev/null +++ b/acceptance/bundle/migrate/added-direct-only/output.txt @@ -0,0 +1,63 @@ + +=== Deploy with terraform (instance_pools is commented out; terraform cannot deploy it) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Add a direct-only resource to the config without deploying it +>>> update_file.py databricks.yml #instance_pools instance_pools + +=== Deploying on terraform rejects it, which is what sends the user to migrate +>>> musterr [CLI] bundle deploy +Error: Instance Pool resources are only supported with direct deployment mode + in databricks.yml:5:19 + +Instance Pool resources require direct deployment mode. Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use instance_pool resources. +Learn more at https://docs.databricks.com/dev-tools/bundles/direct + + +=== Migration ignores the direct-only resource; it will be created on the next deploy +>>> [CLI] bundle deployment migrate +Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json + +Validate the migration by running "databricks bundle plan", there should be no actions planned. + +The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy". + +To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/default/resources.json and rename [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate + + +=== After migration: plan shows the direct-only resource as "to add" +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan +create instance_pools.pool + +Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged + +=== Deploy creates it; verify via recorded requests +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //instance-pools/create +{ + "headers": { + "User-Agent": [ + "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" + ] + }, + "method": "POST", + "path": "/api/2.0/instance-pools/create", + "body": { + "idle_instance_autotermination_minutes": 60, + "instance_pool_name": "My Pool", + "node_type_id": "Standard_DS3_v2" + } +} + +=== No further actions planned +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/migrate/added-direct-only/script b/acceptance/bundle/migrate/added-direct-only/script new file mode 100644 index 00000000000..5302ea89b89 --- /dev/null +++ b/acceptance/bundle/migrate/added-direct-only/script @@ -0,0 +1,26 @@ +export DATABRICKS_BUNDLE_ENGINE=terraform + +title "Deploy with terraform (instance_pools is commented out; terraform cannot deploy it)" +trace $CLI bundle deploy + +title "Add a direct-only resource to the config without deploying it" +trace update_file.py databricks.yml "#instance_pools" "instance_pools" + +title "Deploying on terraform rejects it, which is what sends the user to migrate" +trace musterr $CLI bundle deploy + +title "Migration ignores the direct-only resource; it will be created on the next deploy" +trace $CLI bundle deployment migrate + +title "After migration: plan shows the direct-only resource as \"to add\"" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan + +title "Deploy creates it; verify via recorded requests" +rm out.requests.txt +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy +trace print_requests.py //instance-pools/create + +title "No further actions planned" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan + +rm out.requests.txt diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/databricks.yml b/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/databricks.yml new file mode 100644 index 00000000000..8f2a40d4f2b --- /dev/null +++ b/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + #instance_pools: {pool: {instance_pool_name: "My Pool", node_type_id: "Standard_DS3_v2", idle_instance_autotermination_minutes: 60}} + jobs: + job_a: {name: "Job A"} diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/out.test.toml b/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/output.txt b/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/output.txt new file mode 100644 index 00000000000..dcc6efc8264 --- /dev/null +++ b/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/output.txt @@ -0,0 +1,55 @@ + +=== Deploy with terraform (instance_pools is commented out; terraform cannot deploy it) +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Add a direct-only resource to the config +>>> update_file.py databricks.yml #instance_pools instance_pools + +=== Opt in via env var only: deploy migrates the state, ignoring the direct-only resource +>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy +Warn: Direct engine requested in DATABRICKS_BUNDLE_ENGINE environment variable but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +Migrating state to direct deployment engine (opted in via DATABRICKS_BUNDLE_ENGINE environment variable)... +Migrated 1 resource to direct deployment engine. + +>>> print_migration_telemetry +direct_migrated_via_env true + +=== The direct-only resource is created on the next deploy +>>> [CLI] bundle plan +create instance_pools.pool + +Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //instance-pools/create +{ + "headers": { + "User-Agent": [ + "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" + ] + }, + "method": "POST", + "path": "/api/2.0/instance-pools/create", + "body": { + "idle_instance_autotermination_minutes": 60, + "instance_pool_name": "My Pool", + "node_type_id": "Standard_DS3_v2" + } +} + +=== No further actions planned +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/script b/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/script new file mode 100644 index 00000000000..b605d8edb3c --- /dev/null +++ b/acceptance/bundle/migrate/auto-migrate-direct-only-envvar/script @@ -0,0 +1,23 @@ +export DATABRICKS_BUNDLE_ENGINE= + +title "Deploy with terraform (instance_pools is commented out; terraform cannot deploy it)" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy +rm -f out.requests.txt + +title "Add a direct-only resource to the config" +trace update_file.py databricks.yml "#instance_pools" "instance_pools" + +title "Opt in via env var only: deploy migrates the state, ignoring the direct-only resource" +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy +trace print_migration_telemetry +rm -f out.requests.txt + +title "The direct-only resource is created on the next deploy" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //instance-pools/create + +title "No further actions planned" +trace $CLI bundle plan + +rm -f out.requests.txt diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only/databricks.yml b/acceptance/bundle/migrate/auto-migrate-direct-only/databricks.yml new file mode 100644 index 00000000000..8f2a40d4f2b --- /dev/null +++ b/acceptance/bundle/migrate/auto-migrate-direct-only/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + #instance_pools: {pool: {instance_pool_name: "My Pool", node_type_id: "Standard_DS3_v2", idle_instance_autotermination_minutes: 60}} + jobs: + job_a: {name: "Job A"} diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only/out.test.toml b/acceptance/bundle/migrate/auto-migrate-direct-only/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/migrate/auto-migrate-direct-only/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only/output.txt b/acceptance/bundle/migrate/auto-migrate-direct-only/output.txt new file mode 100644 index 00000000000..95b2067d012 --- /dev/null +++ b/acceptance/bundle/migrate/auto-migrate-direct-only/output.txt @@ -0,0 +1,65 @@ + +=== Deploy with terraform (instance_pools is commented out; terraform cannot deploy it) +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Add a direct-only resource and opt in to the direct engine via config +>>> update_file.py databricks.yml #instance_pools instance_pools + +>>> update_file.py databricks.yml name: test-bundle name: test-bundle + engine: direct + +=== Plan skips the direct-only resource, since this run still uses terraform +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle plan -o json +json.plan.resources.instance_pools.pool.action = "skip"; + +=== Deploy migrates the state, skipping the direct-only resource +>>> [CLI] bundle deploy +Warn: Direct engine requested in bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +Migrating state to direct deployment engine (opted in via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)... +Migrated 1 resource to direct deployment engine. + +>>> print_migration_telemetry +direct_migrated_via_config true + +=== The direct-only resource is created on the next deploy +>>> [CLI] bundle plan +create instance_pools.pool + +Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //instance-pools/create +{ + "headers": { + "User-Agent": [ + "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] interactive/none engine/direct auth/pat" + ] + }, + "method": "POST", + "path": "/api/2.0/instance-pools/create", + "body": { + "idle_instance_autotermination_minutes": 60, + "instance_pool_name": "My Pool", + "node_type_id": "Standard_DS3_v2" + } +} + +=== No further actions planned +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only/script b/acceptance/bundle/migrate/auto-migrate-direct-only/script new file mode 100644 index 00000000000..3cb22cb053a --- /dev/null +++ b/acceptance/bundle/migrate/auto-migrate-direct-only/script @@ -0,0 +1,28 @@ +export DATABRICKS_BUNDLE_ENGINE= + +title "Deploy with terraform (instance_pools is commented out; terraform cannot deploy it)" +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy +rm -f out.requests.txt + +title "Add a direct-only resource and opt in to the direct engine via config" +trace update_file.py databricks.yml "#instance_pools" "instance_pools" +trace update_file.py databricks.yml "name: test-bundle" $'name: test-bundle\n engine: direct' + +title "Plan skips the direct-only resource, since this run still uses terraform" +trace $CLI bundle plan +trace $CLI bundle plan -o json | gron.py | grep instance_pools + +title "Deploy migrates the state, skipping the direct-only resource" +trace $CLI bundle deploy +trace print_migration_telemetry +rm -f out.requests.txt + +title "The direct-only resource is created on the next deploy" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //instance-pools/create + +title "No further actions planned" +trace $CLI bundle plan + +rm -f out.requests.txt diff --git a/bundle/bundle.go b/bundle/bundle.go index ff04c3479af..8ce12211c3d 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -172,6 +172,14 @@ type Bundle struct { // When non-empty, only the specified resources are included in deployment. Select []string + // MigratingToDirect is set when the direct engine is requested but the existing + // state still uses terraform, so the state is migrated to the direct engine after + // this deploy. Resources that only the direct engine supports are skipped by this + // run rather than rejected: terraform cannot deploy them, and since terraform + // could never have deployed them they are absent from its state. The next deploy, + // which runs on the migrated state, creates them. + MigratingToDirect bool + // SkipLocalFileValidation makes path translation tolerant of missing local files. // When set, TranslatePaths computes workspace paths without verifying files exist. // Used by config-remote-sync: a user may modify resource paths remotely (e.g., diff --git a/bundle/config/mutator/validate_direct_only_resources.go b/bundle/config/mutator/validate_direct_only_resources.go index a76feb9482f..1c3cd41a76c 100644 --- a/bundle/config/mutator/validate_direct_only_resources.go +++ b/bundle/config/mutator/validate_direct_only_resources.go @@ -34,11 +34,18 @@ func isDirectOnly(pluralName string) bool { return hasDirect && !hasTerraform } -func (m *validateDirectOnlyResources) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *validateDirectOnlyResources) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { if m.engine.IsDirect() { return nil } + // The state is migrated to the direct engine right after this deploy, so + // direct-only resources are skipped by this run instead of rejected. + // BundleToTerraformWithDynValue logs each resource it skips. + if b.MigratingToDirect { + return nil + } + var diags diag.Diagnostics for _, group := range b.Config.Resources.AllResources() { if len(group.Resources) == 0 { diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go index 551cbb0a762..8303c2b3a0d 100644 --- a/bundle/deploy/terraform/convert.go +++ b/bundle/deploy/terraform/convert.go @@ -8,11 +8,17 @@ import ( "github.com/databricks/cli/bundle/deploy/terraform/tfdyn" "github.com/databricks/cli/bundle/internal/tf/schema" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/log" ) // BundleToTerraformWithDynValue converts resources in a bundle configuration // to the equivalent Terraform JSON representation. -func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema.Root, error) { +// +// Resource types without a terraform converter are only supported by the direct +// engine. They are an error here, except when skipUnsupported is set: the state is +// migrated to the direct engine right after this deploy, so they are skipped by +// this run and deployed by the next one. See bundle.Bundle.MigratingToDirect. +func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value, skipUnsupported bool) (*schema.Root, error) { tfroot := schema.NewRoot() tfroot.Provider = schema.NewProviders() @@ -49,7 +55,11 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema // Lookup the converter based on the resource type. c, ok := tfdyn.GetConverter(typ) if !ok { - return dyn.InvalidValue, fmt.Errorf("no converter for resource type %s", typ) + if !skipUnsupported { + return dyn.InvalidValue, fmt.Errorf("no converter for resource type %s", typ) + } + log.Infof(ctx, "%s.%s: skipping, only supported by the direct engine; will be deployed after the state is migrated", typ, key) + return v, dyn.ErrSkip } // Convert resource to Terraform representation. diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index 564340190d4..2b709787969 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -21,7 +21,7 @@ import ( func produceTerraformConfiguration(t *testing.T, config config.Root) *schema.Root { vin, err := convert.FromTyped(config, dyn.NilValue) require.NoError(t, err) - out, err := BundleToTerraformWithDynValue(t.Context(), vin) + out, err := BundleToTerraformWithDynValue(t.Context(), vin, false) require.NoError(t, err) return out } @@ -588,7 +588,7 @@ func TestBundleToTerraformDeletedResources(t *testing.T) { vin, err := convert.FromTyped(config, dyn.NilValue) require.NoError(t, err) - out, err := BundleToTerraformWithDynValue(t.Context(), vin) + out, err := BundleToTerraformWithDynValue(t.Context(), vin, false) require.NoError(t, err) _, ok := out.Resource.Job["my_job1"] diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index e44b86509ef..f8a5140576f 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -45,7 +45,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { }, nil), }, nil) - tfroot, err := BundleToTerraformWithDynValue(ctx, vin) + tfroot, err := BundleToTerraformWithDynValue(ctx, vin, false) require.NoError(t, err) bytes, err := json.Marshal(tfroot.Resource) diff --git a/bundle/deploy/terraform/write.go b/bundle/deploy/terraform/write.go index bee777ffe00..3a87d1baabe 100644 --- a/bundle/deploy/terraform/write.go +++ b/bundle/deploy/terraform/write.go @@ -26,7 +26,7 @@ func (w *write) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var root *schema.Root err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - root, err = BundleToTerraformWithDynValue(ctx, v) + root, err = BundleToTerraformWithDynValue(ctx, v, b.MigratingToDirect) return v, err }) if err != nil { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index e4f232605ce..8bbe98bd1a0 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -189,12 +189,14 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } cmd.SetContext(ctx) + b.MigratingToDirect = requiredEngine.Type == engine.EngineDirect && !stateDesc.Engine.IsDirect() + // Announce the auto-migration path here (only on deploy) so the user // isn't surprised when MigrateToDirect commits state changes at the // end. PullResourcesState is shared with non-deploy commands like // `bundle debug states`, which would otherwise print the same hint // even though they will not migrate. - if opts.Deploy && requiredEngine.Type == engine.EngineDirect && !stateDesc.Engine.IsDirect() { + if opts.Deploy && b.MigratingToDirect { log.Warnf(ctx, "Direct engine requested in %s but the existing state uses %q. Deploying on %q; will attempt to migrate the state to the direct engine after this deploy.", requiredEngine.Source, stateDesc.Engine, stateDesc.Engine) }