When globbing for wheels, only consider most recent version; do not clean up dist/build/__pycache__

NEXT_CHANGELOG.md

@@ -10,5 +10,7 @@
 
 ### Bundles
 * Fix reading dashboard contents when the sync root is different than the bundle root ([#3006](https://github.com/databricks/cli/pull/3006))
+* When glob for wheels is used, like "\*.whl", it will filter out different version of the same package and will only take the most recent version. ([#2982](https://github.com/databricks/cli/pull/2982))
+* When building Python artifacts as part of "bundle deploy" we no longer delete `dist`, `build`, `*egg-info` and `__pycache__` directories. ([#2982](https://github.com/databricks/cli/pull/2982))
 
 ### API Changes

acceptance/bundle/artifacts/whl_change_version/.gitignore

@@ -0,0 +1,3 @@
+build/
+*.egg-info
+.databricks

acceptance/bundle/artifacts/whl_change_version/databricks.yml

@@ -0,0 +1,12 @@
+resources:
+ jobs:
+ test_job:
+ name: "[${bundle.target}] My Wheel Job"
+ tasks:
+ - task_key: TestTask
+ existing_cluster_id: "0717-aaaaa-bbbbbb"
+ python_wheel_task:
+ package_name: "my_test_code"
+ entry_point: "run"
+ libraries:
+ - whl: ./dist/*.whl

acceptance/bundle/artifacts/whl_change_version/my_test_code/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.1.0"
+__author__ = "Databricks"

acceptance/bundle/artifacts/whl_change_version/my_test_code/__main__.py

@@ -0,0 +1,16 @@
+"""
+The entry point of the Python Wheel
+"""
+
+import sys
+
+
+def main():
+ # This method will print the provided arguments
+ print("Hello from my func")
+ print("Got arguments:")
+ print(sys.argv)
+
+
+if __name__ == "__main__":
+ main()

acceptance/bundle/artifacts/whl_change_version/output.txt

@@ -0,0 +1,138 @@
+
+>>> [CLI] bundle deploy
+Building python_artifact...
+Uploading dist/my_test_code-0.1.0-py3-none-any.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> find.py --expect 1 whl
+dist/my_test_code-0.1.0-py3-none-any.whl
+
+=== Expecting 1 wheel in libraries section in /jobs/create
+>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks out.requests.txt
+[
+ {
+ "existing_cluster_id": "0717-aaaaa-bbbbbb",
+ "libraries": [
+ {
+ "whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl"
+ }
+ ],
+ "python_wheel_task": {
+ "entry_point": "run",
+ "package_name": "my_test_code"
+ },
+ "task_key": "TestTask"
+ }
+]
+
+=== Expecting 1 wheel to be uploaded
+>>> jq .path out.requests.txt
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/.gitignore"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.1.0-py3-none-any.whl"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate"
+
+>>> update_file.py my_test_code/__init__.py 0.1.0 0.2.0
+
+>>> [CLI] bundle deploy
+Building python_artifact...
+Uploading dist/my_test_code-0.2.0-py3-none-any.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> find.py --expect 2 whl
+dist/my_test_code-0.1.0-py3-none-any.whl
+dist/my_test_code-0.2.0-py3-none-any.whl
+
+=== Expecting 1 wheel in libraries section in /jobs/reset
+>>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks out.requests.txt
+[
+ {
+ "existing_cluster_id": "0717-aaaaa-bbbbbb",
+ "libraries": [
+ {
+ "whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl"
+ }
+ ],
+ "python_wheel_task": {
+ "entry_point": "run",
+ "package_name": "my_test_code"
+ },
+ "task_key": "TestTask"
+ }
+]
+
+=== Expecting 1 wheel to be uploaded
+>>> jq .path out.requests.txt
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate"
+
+=== Restore config to target old wheel
+>>> update_file.py databricks.yml ./dist/*.whl ./dist/my*0.1.0*.whl
+
+>>> [CLI] bundle deploy
+Building python_artifact...
+Uploading dist/my_test_code-0.1.0-py3-none-any.whl...
+Uploading dist/my_test_code-0.2.0-py3-none-any.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> find.py --expect 2 whl
+dist/my_test_code-0.1.0-py3-none-any.whl
+dist/my_test_code-0.2.0-py3-none-any.whl
+
+=== Expecting 1 wheel in libraries section in /jobs/reset
+>>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks out.requests.txt
+[
+ {
+ "existing_cluster_id": "0717-aaaaa-bbbbbb",
+ "libraries": [
+ {
+ "whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl"
+ }
+ ],
+ "python_wheel_task": {
+ "entry_point": "run",
+ "package_name": "my_test_code"
+ },
+ "task_key": "TestTask"
+ }
+]
+
+=== Expecting 1 wheel to be uploaded
+>>> jq .path out.requests.txt
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate"

acceptance/bundle/artifacts/whl_change_version/script

@@ -0,0 +1,39 @@
+trace $CLI bundle deploy
+
+trace find.py --expect 1 whl
+
+title "Expecting 1 wheel in libraries section in /jobs/create"
+trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt
+
+title "Expecting 1 wheel to be uploaded"
+trace jq .path out.requests.txt | grep import | sort
+
+rm out.requests.txt
+
+
+trace update_file.py my_test_code/__init__.py 0.1.0 0.2.0
+trace $CLI bundle deploy
+
+trace find.py --expect 2 whl # there are now 2 wheels on disk
+
+title "Expecting 1 wheel in libraries section in /jobs/reset"
+trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt
+
+title "Expecting 1 wheel to be uploaded"
+trace jq .path out.requests.txt | grep import | sort
+
+rm out.requests.txt
+
+
+title 'Restore config to target old wheel'
+trace update_file.py databricks.yml './dist/*.whl' './dist/my*0.1.0*.whl'
+trace $CLI bundle deploy
+trace find.py --expect 2 whl
+
+title "Expecting 1 wheel in libraries section in /jobs/reset"
+trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks' out.requests.txt
+
+title "Expecting 1 wheel to be uploaded"
+trace jq .path out.requests.txt | grep import | sort
+
+rm out.requests.txt

acceptance/bundle/artifacts/whl_change_version/setup.py

@@ -0,0 +1,15 @@
+from setuptools import setup, find_packages
+
+import my_test_code
+
+setup(
+ name="my_test_code",
+ version=my_test_code.__version__,
+ author=my_test_code.__author__,
+ url="https://databricks.com",
+ author_email="john.doe@databricks.com",
+ description="my test wheel",
+ packages=find_packages(include=["my_test_code"]),
+ entry_points={"group_1": "run=my_test_code.__main__:main"},
+ install_requires=["setuptools"],
+)

bundle/artifacts/build.go

@@ -38,8 +38,6 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
  })
 	}
 
-	cleanupPythonDistBuild(ctx, b)
-
 	for _, artifactName := range utils.SortedKeys(b.Config.Artifacts) {
  a := b.Config.Artifacts[artifactName]
 

bundle/artifacts/expand_globs.go

@@ -9,6 +9,7 @@ import (
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/cli/libs/patchwheel"
 )
 
 func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic {
@@ -72,6 +73,12 @@ func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti
  return v, nil
  }
 
+ // Note, we're applying this for all artifact types, not just "whl".
+ // Rationale:
+ // 1. type is optional
+ // 2. if you have wheels in other artifact type, maybe you still want the filter logic? impossible to say.
+ matches = patchwheel.FilterLatestWheels(ctx, matches)
+
  if len(matches) == 1 && matches[0] == source {
  // No glob expansion was performed.
  // Keep node unchanged. We need to ensure that "patched" field remains and not wiped out by code below.

bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go

@@ -9,6 +9,7 @@ import (
 	"github.com/databricks/cli/bundle/libraries"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/patchwheel"
 )
 
 type expandPipelineGlobPaths struct{}
@@ -17,7 +18,7 @@ func ExpandPipelineGlobPaths() bundle.Mutator {
 	return &expandPipelineGlobPaths{}
 }
 
-func (m *expandPipelineGlobPaths) expandLibrary(dir string, v dyn.Value) ([]dyn.Value, error) {
+func (m *expandPipelineGlobPaths) expandLibrary(ctx context.Context, dir string, v dyn.Value) ([]dyn.Value, error) {
 	// Probe for the path field in the library.
 	for _, p := range []dyn.Path{
  dyn.NewPath(dyn.Key("notebook"), dyn.Key("path")),
@@ -47,6 +48,8 @@ func (m *expandPipelineGlobPaths) expandLibrary(dir string, v dyn.Value) ([]dyn.
  return []dyn.Value{v}, nil
  }
 
+ matches = patchwheel.FilterLatestWheels(ctx, matches)
+
  // Emit a new value for each match.
  var ev []dyn.Value
  for _, match := range matches {
@@ -69,15 +72,15 @@ func (m *expandPipelineGlobPaths) expandLibrary(dir string, v dyn.Value) ([]dyn.
 	return []dyn.Value{v}, nil
 }
 
-func (m *expandPipelineGlobPaths) expandSequence(dir string, p dyn.Path, v dyn.Value) (dyn.Value, error) {
+func (m *expandPipelineGlobPaths) expandSequence(ctx context.Context, dir string, p dyn.Path, v dyn.Value) (dyn.Value, error) {
 	s, ok := v.AsSequence()
 	if !ok {
  return dyn.InvalidValue, fmt.Errorf("expected sequence, got %s", v.Kind())
 	}
 
 	var vs []dyn.Value
 	for _, sv := range s {
- v, err := m.expandLibrary(dir, sv)
+ v, err := m.expandLibrary(ctx, dir, sv)
  if err != nil {
  return dyn.InvalidValue, err
  }
@@ -88,7 +91,7 @@ func (m *expandPipelineGlobPaths) expandSequence(dir string, p dyn.Path, v dyn.V
 	return dyn.NewValue(vs, v.Locations()), nil
 }
 
-func (m *expandPipelineGlobPaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
+func (m *expandPipelineGlobPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
  p := dyn.NewPattern(
  dyn.Key("resources"),
@@ -99,7 +102,7 @@ func (m *expandPipelineGlobPaths) Apply(_ context.Context, b *bundle.Bundle) dia
 
  // Visit each pipeline's "libraries" field and expand any glob patterns.
  return dyn.MapByPattern(v, p, func(path dyn.Path, value dyn.Value) (dyn.Value, error) {
- return m.expandSequence(b.BundleRootPath, path, value)
+ return m.expandSequence(ctx, b.BundleRootPath, path, value)
  })
 	})