Implement dynamic_version attribute in artifacts section

NEXT_CHANGELOG.md

@@ -12,5 +12,6 @@
 * Added support for dashboards in deployment bind/unbind commands ([#2516](https://github.com/databricks/cli/pull/2516))
 * Added support for registered models in deployment bind/unbind commands ([#2556](https://github.com/databricks/cli/pull/2556))
 * Added a mismatch check when host is defined in config and as an env variable ([#2549](https://github.com/databricks/cli/pull/2549))
+* New attribute on artifacts entries: `dynamic_version`. When set to true, it patches the wheel with dynamic version suffix so it is always used by Databricks environments, even if original wheel version is the same. Intended for development loop on interactive clusters. ([#2520](https://github.com/databricks/cli/pull/2520))
 
 ### API Changes

acceptance/bin/ziplist.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""
+List files in zip archive
+"""
+
+import sys
+import zipfile
+from pathlib import Path
+
+
+for zip_path in sys.argv[1:]:
+ with zipfile.ZipFile(zip_path.strip()) as z:
+ for info in z.infolist():
+ print(info.filename)

acceptance/bundle/artifacts/whl_dynamic/databricks.yml

@@ -0,0 +1,50 @@
+bundle:
+ name: python-wheel
+
+artifacts:
+ my_test_code:
+ type: whl
+ path: "./my_test_code"
+ # using 'python' there because 'python3' does not exist in virtualenv on windows
+ build: python setup.py bdist_wheel
+ dynamic_version: true
+ my_prebuilt_whl:
+ type: whl
+ files:
+ - source: prebuilt/other_test_code-0.0.1-py3-none-any.whl
+ dynamic_version: true
+
+resources:
+ jobs:
+ test_job:
+ name: "[${bundle.target}] My Wheel Job"
+ tasks:
+ - task_key: TestTask
+ existing_cluster_id: "0717-132531-5opeqon1"
+ python_wheel_task:
+ package_name: "my_test_code"
+ entry_point: "run"
+ libraries:
+ - whl: ./my_test_code/dist/*.whl
+ - whl: prebuilt/other_test_code-0.0.1-py3-none-any.whl
+ for_each_task:
+ inputs: "[1]"
+ task:
+ task_key: SubTask
+ existing_cluster_id: "0717-132531-5opeqon1"
+ python_wheel_task:
+ package_name: "my_test_code"
+ entry_point: "run"
+ libraries:
+ - whl: ./my_test_code/dist/*.whl
+ - task_key: ServerlessTestTask
+ python_wheel_task:
+ package_name: "my_test_code"
+ entry_point: "run"
+ environment_key: "test_env"
+ environments:
+ - environment_key: "test_env"
+ spec:
+ client: "1"
+ dependencies:
+ - ./my_test_code/dist/*.whl

acceptance/bundle/artifacts/whl_dynamic/output.txt

@@ -0,0 +1,170 @@
+
+>>> [CLI] bundle validate -o json
+{
+ "my_prebuilt_whl": {
+ "dynamic_version": true,
+ "files": [
+ {
+ "source": "[TEST_TMP_DIR]/prebuilt/other_test_code-0.0.1-py3-none-any.whl"
+ }
+ ],
+ "path": "[TEST_TMP_DIR]",
+ "type": "whl"
+ },
+ "my_test_code": {
+ "build": "python setup.py bdist_wheel",
+ "dynamic_version": true,
+ "path": "[TEST_TMP_DIR]/my_test_code",
+ "type": "whl"
+ }
+}
+
+>>> [CLI] bundle deploy
+Building my_test_code...
+Uploading .databricks/bundle/default/patched_wheels/my_prebuilt_whl_other_test_code/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl...
+Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== There are 2 original wheels and 2 patched ones
+>>> find.py --expect 4 whl
+.databricks/bundle/default/patched_wheels/my_prebuilt_whl_other_test_code/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl
+.databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl
+my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl
+prebuilt/other_test_code-0.0.1-py3-none-any.whl
+
+=== Verify contents of the zip file
+>>> find.py --expect 1 .databricks/.*my_test_code.*whl
+src/__init__.py
+src/__main__.py
+my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/METADATA
+my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/WHEEL
+my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/entry_points.txt
+my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/top_level.txt
+my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/RECORD
+
+=== Expecting 2 patched wheels in libraries section in /jobs/create
+>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt
+[
+ {
+ "environment_key": "test_env",
+ "python_wheel_task": {
+ "entry_point": "run",
+ "package_name": "my_test_code"
+ },
+ "task_key": "ServerlessTestTask"
+ },
+ {
+ "existing_cluster_id": "0717-132531-5opeqon1",
+ "for_each_task": {
+ "inputs": "[1]",
+ "task": {
+ "existing_cluster_id": "0717-132531-5opeqon1",
+ "libraries": [
+ {
+ "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl"
+ }
+ ],
+ "python_wheel_task": {
+ "entry_point": "run",
+ "package_name": "my_test_code"
+ },
+ "task_key": "SubTask"
+ }
+ },
+ "libraries": [
+ {
+ "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl"
+ },
+ {
+ "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl"
+ }
+ ],
+ "python_wheel_task": {
+ "entry_point": "run",
+ "package_name": "my_test_code"
+ },
+ "task_key": "TestTask"
+ }
+]
+
+=== Expecting 2 patched wheels to be uploaded
+>>> jq .path
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/prebuilt/other_test_code-0.0.1-py3-none-any.whl"
+
+=== Updating the local wheel and deploying again
+>>> [CLI] bundle deploy
+Building my_test_code...
+Uploading .databricks/bundle/default/patched_wheels/my_prebuilt_whl_other_test_code/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl...
+Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Verify contents, it should now have new_module.py
+>>> find.py --expect 1 .databricks/.*my_test_code.*whl
+src/__init__.py
+src/__main__.py
+src/new_module.py
+my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/METADATA
+my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/WHEEL
+my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/entry_points.txt
+my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/top_level.txt
+my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/RECORD
+
+=== Expecting 2 patched wheels in libraries section in /jobs/reset
+>>> jq -s .[] | select(.path=="/api/2.1/jobs/reset") | .body.new_settings.tasks out.requests.txt
+[
+ {
+ "environment_key": "test_env",
+ "python_wheel_task": {
+ "entry_point": "run",
+ "package_name": "my_test_code"
+ },
+ "task_key": "ServerlessTestTask"
+ },
+ {
+ "existing_cluster_id": "0717-132531-5opeqon1",
+ "for_each_task": {
+ "inputs": "[1]",
+ "task": {
+ "existing_cluster_id": "0717-132531-5opeqon1",
+ "libraries": [
+ {
+ "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl"
+ }
+ ],
+ "python_wheel_task": {
+ "entry_point": "run",
+ "package_name": "my_test_code"
+ },
+ "task_key": "SubTask"
+ }
+ },
+ "libraries": [
+ {
+ "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl"
+ },
+ {
+ "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl"
+ }
+ ],
+ "python_wheel_task": {
+ "entry_point": "run",
+ "package_name": "my_test_code"
+ },
+ "task_key": "TestTask"
+ }
+]
+
+=== Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice)
+>>> jq .path
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl"

acceptance/bundle/artifacts/whl_dynamic/script

@@ -0,0 +1,36 @@
+trap "rm -fr out.requests.txt databricks.yml my_test_code prebuilt" EXIT
+
+cp -r $TESTDIR/../whl_explicit/my_test_code .
+mkdir prebuilt
+cp -r $TESTDIR/../whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl prebuilt
+
+trace $CLI bundle validate -o json | jq .artifacts
+
+trace $CLI bundle deploy
+
+title "There are 2 original wheels and 2 patched ones"
+trace find.py --expect 4 whl
+
+title "Verify contents of the zip file"
+trace find.py --expect 1 '.databricks/.*my_test_code.*whl' | xargs ziplist.py
+
+title "Expecting 2 patched wheels in libraries section in /jobs/create"
+trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt
+
+title "Expecting 2 patched wheels to be uploaded"
+trace jq .path < out.requests.txt | grep import | grep whl | sort
+
+rm out.requests.txt
+
+title "Updating the local wheel and deploying again"
+touch my_test_code/src/new_module.py
+trace $CLI bundle deploy
+
+title "Verify contents, it should now have new_module.py"
+trace find.py --expect 1 '.databricks/.*my_test_code.*whl' | xargs ziplist.py
+
+title "Expecting 2 patched wheels in libraries section in /jobs/reset"
+trace jq -s '.[] | select(.path=="/api/2.1/jobs/reset") | .body.new_settings.tasks' out.requests.txt
+
+title "Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice)"
+trace jq .path < out.requests.txt | grep import | grep whl | sort

acceptance/bundle/artifacts/whl_dynamic/test.toml

@@ -0,0 +1,7 @@
+[[Repls]]
+Old = '\\\\'
+New = '/'
+
+[[Server]]
+Pattern = "POST /api/2.1/jobs/reset"
+Response.Body = '{}'

acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/databricks.yml.tmpl

@@ -0,0 +1,37 @@
+bundle:
+ name: wheel-task
+
+workspace:
+ root_path: "~/.bundle/$UNIQUE_NAME"
+
+artifacts:
+ python_artifact:
+ type: whl
+ build: uv build --wheel
+ files:
+ - source: dist/*.whl
+ dynamic_version: true
+
+resources:
+ clusters:
+ test_cluster:
+ cluster_name: "test-cluster-$UNIQUE_NAME"
+ spark_version: "$DEFAULT_SPARK_VERSION"
+ node_type_id: "$NODE_TYPE_ID"
+ num_workers: 1
+ data_security_mode: $DATA_SECURITY_MODE
+
+ jobs:
+ some_other_job:
+ name: "[${bundle.target}] Test Wheel Job $UNIQUE_NAME"
+ tasks:
+ - task_key: TestTask
+ existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}"
+ python_wheel_task:
+ package_name: my_test_code
+ entry_point: run
+ parameters:
+ - "one"
+ - "two"
+ libraries:
+ - whl: ./dist/*.whl

acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt

@@ -0,0 +1,45 @@
+
+>>> [CLI] bundle deploy
+Building python_artifact...
+Uploading .databricks/bundle/default/patched_wheels/python_artifact_my_test_code/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle run some_other_job
+Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID]
+
+[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING 
+[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS 
+Hello from my func
+Got arguments:
+['my_test_code', 'one', 'two']
+
+=== Make a change to code without version change and run the job again
+>>> [CLI] bundle deploy
+Building python_artifact...
+Uploading .databricks/bundle/default/patched_wheels/python_artifact_my_test_code/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle run some_other_job
+Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID]
+
+[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING 
+[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS 
+UPDATED MY FUNC
+Got arguments:
+['my_test_code', 'one', 'two']
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+ delete cluster test_cluster
+ delete job some_other_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Deleting files...
+Destroy complete!

acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/script

@@ -0,0 +1,10 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+cp -r $TESTDIR/../interactive_cluster/{setup.py,my_test_code} .
+trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT
+trace $CLI bundle deploy
+trace $CLI bundle run some_other_job
+
+title "Make a change to code without version change and run the job again"
+update_file.py my_test_code/__main__.py 'Hello from my func' 'UPDATED MY FUNC'
+trace $CLI bundle deploy
+trace $CLI bundle run some_other_job

acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml

@@ -0,0 +1,5 @@
+[EnvMatrix]
+DATA_SECURITY_MODE = [
+ "USER_ISOLATION",
+ "SINGLE_USER",
+]