Bump Terraform provider to v1.98.0

NEXT_CHANGELOG.md

@@ -7,10 +7,12 @@
 
 ### CLI
 
-### Dependency updates
-
 ### Bundles
 * Add support for alerts to DABs ([#4004](https://github.com/databricks/cli/pull/4004))
 * Allow `file://` URIs in job libraries to reference runtime filesystem paths (e.g., JARs pre-installed on clusters via init scripts). These paths are no longer treated as local files to upload. ([#3884](https://github.com/databricks/cli/pull/3884))
+* Pipeline catalog changes now trigger in-place updates instead of recreation (Terraform provider v1.98.0 behavior change) ([#4082](https://github.com/databricks/cli/pull/4082))
+
+### Dependency updates
+* Bump Terraform provider to v1.98.0 ([#4082](https://github.com/databricks/cli/pull/4082))
 
 ### API Changes

acceptance/bin/edit_resource.py

@@ -50,7 +50,7 @@ def main():
  my_locals = {"r": data}
 
  try:
- exec(script, locals=my_locals)
+ exec(script, my_locals)
  except Exception:
  pprint.pprint(my_locals)
  raise

acceptance/bundle/lifecycle/prevent-destroy/script

@@ -5,8 +5,10 @@ trace $CLI bundle deploy
 trace errcode $CLI bundle plan
 trace musterr $CLI bundle destroy --auto-approve >>out.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1
 
-# Changing the catalog name, deploy must fail because pipeline will be recreated
-update_file.py resources/pipeline.yml 'catalog: main' 'catalog: mainnew'
+# Note: In Terraform provider v1.98.0+, changing the catalog no longer triggers recreation.
+# We use storage location instead, which still requires recreation.
+# Changing from catalog to storage, deploy must fail because pipeline will be recreated
+update_file.py resources/pipeline.yml ' catalog: main' ' storage: "dbfs:/my-storage"'
 trace errcode $CLI bundle plan >>out.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1
 trace musterr $CLI bundle deploy >>out.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1
 

acceptance/bundle/migrate/default-python/out.state_original.json

@@ -313,6 +313,7 @@
  "trigger": [
  {
  "file_arrival": [],
+ "model": [],
  "pause_status": "PAUSED",
  "periodic": [
  {

acceptance/bundle/migrate/runas/out.create_requests.json

@@ -1,7 +1,7 @@
 {
  "headers": {
  "User-Agent": [
- "databricks-tf-provider/1.97.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/pipeline auth/pat"
+ "databricks-tf-provider/1.98.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/pipeline auth/pat"
  ]
  },
  "method": "POST",
@@ -32,7 +32,7 @@
 {
  "headers": {
  "User-Agent": [
- "databricks-tf-provider/1.97.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/permissions auth/pat"
+ "databricks-tf-provider/1.98.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/permissions auth/pat"
  ]
  },
  "method": "PUT",

acceptance/bundle/resource_deps/pipelines_recreate/databricks.yml

@@ -2,7 +2,7 @@ resources:
  pipelines:
  foo:
  name: pipeline foo
-catalog: mycatalog
+storage: dbfs:/my-storage
  jobs:
  bar:
  name: job bar

acceptance/bundle/resource_deps/pipelines_recreate/out.create.requests.json

@@ -2,14 +2,14 @@
  "method": "POST",
  "path": "/api/2.0/pipelines",
  "body": {
- "catalog": "mycatalog",
  "channel": "CURRENT",
  "deployment": {
  "kind": "BUNDLE",
  "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
  },
  "edition": "ADVANCED",
- "name": "pipeline foo"
+ "name": "pipeline foo",
+ "storage": "dbfs:/my-storage"
  }
 }
 {

acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json

@@ -32,14 +32,14 @@
  "action": "create",
  "new_state": {
  "value": {
- "catalog": "mycatalog",
  "channel": "CURRENT",
  "deployment": {
  "kind": "BUNDLE",
  "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
  },
  "edition": "ADVANCED",
- "name": "pipeline foo"
+ "name": "pipeline foo",
+ "storage": "dbfs:/my-storage"
  }
  }
  }

acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json

@@ -57,15 +57,15 @@
  "pipeline_id": "[UUID]",
  "run_as_user_name": "[USERNAME]",
  "spec": {
- "catalog": "mynewcatalog",
  "channel": "CURRENT",
  "deployment": {
  "kind": "BUNDLE",
  "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
  },
  "edition": "ADVANCED",
  "id": "[UUID]",
- "name": "pipeline foo"
+ "name": "pipeline foo",
+ "storage": "dbfs:/my-new-storage"
  },
  "state": "IDLE"
  }

acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json

@@ -78,22 +78,22 @@
  "action": "recreate",
  "new_state": {
  "value": {
- "catalog": "mynewcatalog",
  "channel": "CURRENT",
  "deployment": {
  "kind": "BUNDLE",
  "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
  },
  "edition": "ADVANCED",
- "name": "pipeline foo"
+ "name": "pipeline foo",
+ "storage": "dbfs:/my-new-storage"
  }
  },
  "changes": {
  "local": {
- "catalog": {
+ "storage": {
  "action": "recreate",
- "old": "mycatalog",
- "new": "mynewcatalog"
+ "old": "dbfs:/my-storage",
+ "new": "dbfs:/my-new-storage"
  }
  }
  }

acceptance/bundle/resource_deps/pipelines_recreate/out.update.requests.direct.json

@@ -6,14 +6,14 @@
  "method": "POST",
  "path": "/api/2.0/pipelines",
  "body": {
- "catalog": "mynewcatalog",
  "channel": "CURRENT",
  "deployment": {
  "kind": "BUNDLE",
  "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
  },
  "edition": "ADVANCED",
- "name": "pipeline foo"
+ "name": "pipeline foo",
+ "storage": "dbfs:/my-new-storage"
  }
 }
 {

acceptance/bundle/resource_deps/pipelines_recreate/out.update.requests.terraform.json

@@ -6,14 +6,14 @@
  "method": "POST",
  "path": "/api/2.0/pipelines",
  "body": {
- "catalog": "mynewcatalog",
  "channel": "CURRENT",
  "deployment": {
  "kind": "BUNDLE",
  "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
  },
  "edition": "ADVANCED",
- "name": "pipeline foo"
+ "name": "pipeline foo",
+ "storage": "dbfs:/my-new-storage"
  }
 }
 {

acceptance/bundle/resource_deps/pipelines_recreate/output.txt

@@ -16,8 +16,8 @@ Deployment complete!
 >>> [CLI] bundle plan
 Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
 
-=== Update catalog, triggering recreate for pipeline; this means updating downstream deps
->>> update_file.py databricks.yml mycatalog mynewcatalog
+=== Update storage, triggering recreate for pipeline; this means updating downstream deps
+>>> update_file.py databricks.yml storage: dbfs:/my-storage storage: dbfs:/my-new-storage
 
 >>> [CLI] bundle plan
 update jobs.bar
@@ -51,15 +51,15 @@ Error: The specified pipeline [FOO_ID] was not found.
  "pipeline_id":"[UUID]",
  "run_as_user_name":"[USERNAME]",
  "spec": {
- "catalog":"mynewcatalog",
  "channel":"CURRENT",
  "deployment": {
  "kind":"BUNDLE",
  "metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
  },
  "edition":"ADVANCED",
  "id":"[UUID]",
- "name":"pipeline foo"
+ "name":"pipeline foo",
+ "storage":"dbfs:/my-new-storage"
  },
  "state":"IDLE"
 }

acceptance/bundle/resource_deps/pipelines_recreate/script

@@ -13,8 +13,8 @@ echo "$bar_id:BAR_ID" >> ACC_REPLS
 
 trace $CLI bundle plan # empty
 
-title "Update catalog, triggering recreate for pipeline; this means updating downstream deps"
-trace update_file.py databricks.yml mycatalog mynewcatalog
+title "Update storage, triggering recreate for pipeline; this means updating downstream deps"
+trace update_file.py databricks.yml "storage: dbfs:/my-storage" "storage: dbfs:/my-new-storage"
 trace $CLI bundle plan
 $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json
 trace $CLI bundle deploy --auto-approve

acceptance/bundle/resources/jobs/update/out.state.terraform.json

@@ -126,6 +126,7 @@
  "trigger": [
  {
  "file_arrival": [],
+ "model": [],
  "pause_status": "UNPAUSED",
  "periodic": [
  {