[Python] Move 'python' outside of experimental

NEXT_CHANGELOG.md

@@ -3,14 +3,16 @@
 ## Release v0.275.0
 
 ### Notable Changes
+* Python support for Databricks Asset Bundles is now generally available
 
 ### CLI
-
 * Remove `inplace` mode for the `--progress-format` flag. ([#3811](https://github.com/databricks/cli/pull/3811))
 * Remove `json` mode for the `--progress-format` flag. ([#3812](https://github.com/databricks/cli/pull/3812))
 
 ### Dependency updates
 
 ### Bundles
+* Add `pydabs` template replacing `experimental-jobs-as-code` template ([#3806](https://github.com/databricks/cli/pull/3806))
+* You can now use `python` section instead of `experimental/python` ([#3540](https://github.com/databricks/cli/pull/3540))
 
 ### API Changes

acceptance/bundle/help/bundle-init/output.txt

@@ -7,7 +7,7 @@ TEMPLATE_PATH optionally specifies which template to use. It can be one of the f
 - default-sql: The default SQL template for .sql files that run with Databricks SQL
 - dbt-sql: The dbt SQL template (databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks)
 - mlops-stacks: The Databricks MLOps Stacks template (github.com/databricks/mlops-stacks)
-- experimental-jobs-as-code: Jobs as code template (experimental)
+- pydabs: A variant of 'default-python' template that defines resources in Python instead of YAML
 - a local file system path with a template directory
 - a Git repository URL, e.g. https://github.com/my/repository
 

acceptance/bundle/python/experimental-compatibility-both-equal/databricks.yml

@@ -0,0 +1,15 @@
+bundle:
+ name: my_project
+
+sync: {paths: []} # don't need to copy files
+
+# We DO NOT allow specifying both 'python' and 'experimental/python' if they are not equal.
+
+experimental:
+ python:
+ resources:
+ - "resources:load_resources"
+
+python:
+ resources:
+ - "resources:load_resources"

acceptance/bundle/python/experimental-compatibility-both-equal/output.txt

@@ -0,0 +1,86 @@
+
+>>> uv run [UV_ARGS] -q [CLI] bundle validate --output json --include-locations
+{
+ "experimental": {
+ "python": {
+ "resources": [
+ "resources:load_resources"
+ ]
+ }
+ },
+ "resources": {
+ "jobs": {
+ "my_job": {
+ "deployment": {
+ "kind": "BUNDLE",
+ "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_project/default/state/metadata.json"
+ },
+ "edit_mode": "UI_LOCKED",
+ "format": "MULTI_TASK",
+ "max_concurrent_runs": 1,
+ "name": "My Job",
+ "queue": {
+ "enabled": true
+ }
+ }
+ }
+ },
+ "__locations": {
+ "files": [
+ "__generated_by_python__.yml",
+ "databricks.yml",
+ "resources.py"
+ ],
+ "locations": {
+ "bundle": [
+ [
+ 1,
+ 2,
+ 3
+ ]
+ ],
+ "experimental": [
+ [
+ 1,
+ 9,
+ 3
+ ]
+ ],
+ "python": [
+ [
+ 1,
+ 14,
+ 3
+ ]
+ ],
+ "resources": [
+ [
+ 0,
+ 1,
+ 880
+ ]
+ ],
+ "resources.jobs": [
+ [
+ 0,
+ 1,
+ 889
+ ]
+ ],
+ "resources.jobs.my_job": [
+ [
+ 2,
+ 6,
+ 1
+ ]
+ ],
+ "sync": [
+ [
+ 1,
+ 4,
+ 7
+ ]
+ ]
+ }
+ }
+}

acceptance/bundle/python/experimental-compatibility-both-equal/resources.py

@@ -0,0 +1,11 @@
+from databricks.bundles.core import Resources
+
+
+def load_resources() -> Resources:
+ resources = Resources()
+ resources.add_job(
+ resource_name="my_job",
+ job={"name": "My Job"},
+ )
+
+ return resources

acceptance/bundle/python/experimental-compatibility-both-equal/script

@@ -0,0 +1,8 @@
+echo "$DATABRICKS_BUNDLES_WHEEL" > "requirements-latest.txt"
+
+# each job should record location where add_job function was called
+
+trace uv run $UV_ARGS -q $CLI bundle validate --output json --include-locations | \
+ jq "pick(.experimental.python, .resources, .__locations.files, .__locations.locations)"
+
+rm -fr .databricks __pycache__

acceptance/bundle/python/experimental-compatibility-both-error/databricks.yml

@@ -0,0 +1,15 @@
+bundle:
+ name: my_project
+
+sync: {paths: []} # don't need to copy files
+
+# We allow specifying both 'python' and 'experimental/python' if they are equal.
+
+experimental:
+ python:
+ resources:
+ - "resources:load_resources_1"
+
+python:
+ resources:
+ - "resources:load_resources_2"

acceptance/bundle/python/experimental-compatibility-both-error/output.txt

@@ -0,0 +1,20 @@
+
+>>> uv run [UV_ARGS] -q [CLI] bundle validate --output json --include-locations
+Error: failed to apply python mutator: 'experimental/python' and 'python' configuration properties are mutually exclusive, use only 'python'
+
+{
+ "experimental": {
+ "python": {
+ "resources": [
+ "resources:load_resources_1"
+ ]
+ }
+ },
+ "resources": null,
+ "__locations": {
+ "files": null,
+ "locations": null
+ }
+}
+
+Exit code: 1

acceptance/bundle/python/experimental-compatibility-both-error/script

@@ -0,0 +1,8 @@
+echo "$DATABRICKS_BUNDLES_WHEEL" > "requirements-latest.txt"
+
+# each job should record location where add_job function was called
+
+trace uv run $UV_ARGS -q $CLI bundle validate --output json --include-locations | \
+ jq "pick(.experimental.python, .resources, .__locations.files, .__locations.locations)"
+
+rm -fr .databricks __pycache__

acceptance/bundle/python/experimental-compatibility/databricks.yml

@@ -0,0 +1,13 @@
+bundle:
+ name: my_project
+
+sync: {paths: []} # don't need to copy files
+
+# We keep support for experimental section for backward-compatibility until it's fully deprecated.
+#
+# The next step until deletion is returning an error when 'experimental/python' is used.
+
+experimental:
+ python:
+ resources:
+ - "resources:load_resources"

acceptance/bundle/python/experimental-compatibility/output.txt

@@ -0,0 +1,79 @@
+
+>>> uv run [UV_ARGS] -q [CLI] bundle validate --output json --include-locations
+{
+ "experimental": {
+ "python": {
+ "resources": [
+ "resources:load_resources"
+ ]
+ }
+ },
+ "resources": {
+ "jobs": {
+ "my_job": {
+ "deployment": {
+ "kind": "BUNDLE",
+ "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_project/default/state/metadata.json"
+ },
+ "edit_mode": "UI_LOCKED",
+ "format": "MULTI_TASK",
+ "max_concurrent_runs": 1,
+ "name": "My Job",
+ "queue": {
+ "enabled": true
+ }
+ }
+ }
+ },
+ "__locations": {
+ "files": [
+ "__generated_by_python__.yml",
+ "databricks.yml",
+ "resources.py"
+ ],
+ "locations": {
+ "bundle": [
+ [
+ 1,
+ 2,
+ 3
+ ]
+ ],
+ "experimental": [
+ [
+ 1,
+ 11,
+ 3
+ ]
+ ],
+ "resources": [
+ [
+ 0,
+ 1,
+ 825
+ ]
+ ],
+ "resources.jobs": [
+ [
+ 0,
+ 1,
+ 834
+ ]
+ ],
+ "resources.jobs.my_job": [
+ [
+ 2,
+ 6,
+ 1
+ ]
+ ],
+ "sync": [
+ [
+ 1,
+ 4,
+ 7
+ ]
+ ]
+ }
+ }
+}

acceptance/bundle/python/experimental-compatibility/resources.py

@@ -0,0 +1,11 @@
+from databricks.bundles.core import Resources
+
+
+def load_resources() -> Resources:
+ resources = Resources()
+ resources.add_job(
+ resource_name="my_job",
+ job={"name": "My Job"},
+ )
+
+ return resources

acceptance/bundle/python/experimental-compatibility/script

@@ -0,0 +1,8 @@
+echo "$DATABRICKS_BUNDLES_WHEEL" > "requirements-latest.txt"
+
+# each job should record location where add_job function was called
+
+trace uv run $UV_ARGS -q $CLI bundle validate --output json --include-locations | \
+ jq "pick(.experimental.python, .resources, .__locations.files, .__locations.locations)"
+
+rm -fr .databricks __pycache__

acceptance/bundle/python/mutator-ordering/databricks.yml

@@ -3,11 +3,10 @@ bundle:
 
 sync: {paths: []} # don't need to copy files
 
-experimental:
- python:
- mutators:
- - "mutators:add_task_1"
- - "mutators:add_task_2"
+python:
+ mutators:
+ - "mutators:add_task_1"
+ - "mutators:add_task_2"
 
 resources:
  jobs:

acceptance/bundle/python/mutator-ordering/output.txt

@@ -47,7 +47,7 @@
  3
  ]
  ],
- "experimental": [
+ "python": [
  [
  0,
  7,
@@ -57,28 +57,28 @@
  "resources": [
  [
  0,
-13,
+12,
  3
  ]
  ],
  "resources.jobs": [
  [
  0,
-14,
+13,
  5
  ]
  ],
  "resources.jobs.my_job": [
  [
  0,
-15,
+14,
  7
  ]
  ],
  "resources.jobs.my_job.tasks": [
  [
  0,
-15,
+14,
  14
  ]
  ],