Add preliminary job_runs resource

NEXT_CHANGELOG.md

@@ -10,6 +10,7 @@
 
 ### Bundles
 
+* direct: add basic version of job_runs resource (experimental) ([#5603](https://github.com/databricks/cli/pull/5603)).
 * Fix permissions added to a job or pipeline by a Python (PyDABs) mutator failing to deploy with "must have exactly one owner"; the deploying identity is now set as owner, matching resources whose permissions are declared in YAML ([#5821](https://github.com/databricks/cli/pull/5821)).
 * Remove duplicate enum values for jsonschema.json ([#5839](https://github.com/databricks/cli/pull/5839)).
 * direct: volumes: support `volume_path` property ([#5550](https://github.com/databricks/cli/pull/5550)).

acceptance/bundle/invariant/configs/job_run.yml.tmpl

@@ -0,0 +1,20 @@
+bundle:
+ name: test-bundle-$UNIQUE_NAME
+
+resources:
+ jobs:
+ foo:
+ name: test-job-$UNIQUE_NAME
+ tasks:
+ - task_key: only_task
+ notebook_task:
+ notebook_path: /Shared/notebook
+ new_cluster:
+ spark_version: $DEFAULT_SPARK_VERSION
+ node_type_id: $NODE_TYPE_ID
+ instance_pool_id: $TEST_INSTANCE_POOL_ID
+ num_workers: 1
+
+ job_runs:
+ foo_run:
+ job_id: ${resources.jobs.foo.id}

acceptance/bundle/invariant/continue_293/test.toml

@@ -13,6 +13,9 @@ EnvMatrixExclude.no_vector_search_index = ["INPUT_CONFIG=vector_search_index.yml
 # genie_spaces resource is not supported on v0.293.0
 EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"]
 
+# job_runs resource is not supported on v0.293.0
+EnvMatrixExclude.no_job_run = ["INPUT_CONFIG=job_run.yml.tmpl"]
+
 # Dotted pipeline configuration keys are not supported on v0.293.0
 EnvMatrixExclude.no_pipeline_config_dots = ["INPUT_CONFIG=pipeline_config_dots.yml.tmpl"]
 

acceptance/bundle/invariant/migrate/test.toml

@@ -2,6 +2,10 @@
 EnvMatrixExclude.no_vector_search_endpoint = ["INPUT_CONFIG=vector_search_endpoint.yml.tmpl"]
 EnvMatrixExclude.no_vector_search_index = ["INPUT_CONFIG=vector_search_index.yml.tmpl"]
 
+# job_runs is a direct-only resource with no terraform converter, so the
+# terraform deploy that seeds the migration fails for it.
+EnvMatrixExclude.no_job_run = ["INPUT_CONFIG=job_run.yml.tmpl"]
+
 # Error: Catalog resources are only supported with direct deployment mode
 EnvMatrixExclude.no_catalog = ["INPUT_CONFIG=catalog.yml.tmpl"]
 EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmpl"]

acceptance/bundle/invariant/test.toml

@@ -40,6 +40,7 @@ EnvMatrix.INPUT_CONFIG = [
  "job_pydabs_1000_tasks.yml.tmpl",
  "job_cross_resource_ref.yml.tmpl",
  "job_permission_ref.yml.tmpl",
+ "job_run.yml.tmpl",
  "job_run_job_ref.yml.tmpl",
  "job_with_depends_on.yml.tmpl",
  "job_with_task.yml.tmpl",

acceptance/bundle/refschema/out.fields.txt

@@ -756,6 +756,55 @@ resources.genie_spaces.*.permissions[*].group_name	string	ALL
 resources.genie_spaces.*.permissions[*].level	iam.PermissionLevel	ALL
 resources.genie_spaces.*.permissions[*].service_principal_name	string	ALL
 resources.genie_spaces.*.permissions[*].user_name	string	ALL
+resources.job_runs.*.dbt_commands	[]string	ALL
+resources.job_runs.*.dbt_commands[*]	string	ALL
+resources.job_runs.*.id	string	INPUT
+resources.job_runs.*.idempotency_token	string	ALL
+resources.job_runs.*.jar_params	[]string	ALL
+resources.job_runs.*.jar_params[*]	string	ALL
+resources.job_runs.*.job_id	int64	ALL
+resources.job_runs.*.job_parameters	map[string]string	ALL
+resources.job_runs.*.job_parameters.*	string	ALL
+resources.job_runs.*.lifecycle	resources.Lifecycle	INPUT
+resources.job_runs.*.lifecycle.prevent_destroy	bool	INPUT
+resources.job_runs.*.modified_status	string	INPUT
+resources.job_runs.*.notebook_params	map[string]string	ALL
+resources.job_runs.*.notebook_params.*	string	ALL
+resources.job_runs.*.only	[]string	ALL
+resources.job_runs.*.only[*]	string	ALL
+resources.job_runs.*.performance_target	jobs.PerformanceTarget	ALL
+resources.job_runs.*.pipeline_params	*jobs.PipelineParams	ALL
+resources.job_runs.*.pipeline_params.full_refresh	bool	ALL
+resources.job_runs.*.pipeline_params.full_refresh_selection	[]string	ALL
+resources.job_runs.*.pipeline_params.full_refresh_selection[*]	string	ALL
+resources.job_runs.*.pipeline_params.refresh_flow_selection	[]string	ALL
+resources.job_runs.*.pipeline_params.refresh_flow_selection[*]	string	ALL
+resources.job_runs.*.pipeline_params.refresh_selection	[]string	ALL
+resources.job_runs.*.pipeline_params.refresh_selection[*]	string	ALL
+resources.job_runs.*.pipeline_params.reset_checkpoint_selection	[]string	ALL
+resources.job_runs.*.pipeline_params.reset_checkpoint_selection[*]	string	ALL
+resources.job_runs.*.python_named_params	map[string]string	ALL
+resources.job_runs.*.python_named_params.*	string	ALL
+resources.job_runs.*.python_params	[]string	ALL
+resources.job_runs.*.python_params[*]	string	ALL
+resources.job_runs.*.queue	*jobs.QueueSettings	ALL
+resources.job_runs.*.queue.enabled	bool	ALL
+resources.job_runs.*.resolved_job_id	int64	INPUT
+resources.job_runs.*.run_id	int64	REMOTE
+resources.job_runs.*.run_name	string	REMOTE
+resources.job_runs.*.run_page_url	string	REMOTE
+resources.job_runs.*.run_type	jobs.RunType	REMOTE
+resources.job_runs.*.spark_submit_params	[]string	ALL
+resources.job_runs.*.spark_submit_params[*]	string	ALL
+resources.job_runs.*.sql_params	map[string]string	ALL
+resources.job_runs.*.sql_params.*	string	ALL
+resources.job_runs.*.state	*jobs.RunState	REMOTE
+resources.job_runs.*.state.life_cycle_state	jobs.RunLifeCycleState	REMOTE
+resources.job_runs.*.state.queue_reason	string	REMOTE
+resources.job_runs.*.state.result_state	jobs.RunResultState	REMOTE
+resources.job_runs.*.state.state_message	string	REMOTE
+resources.job_runs.*.state.user_cancelled_or_timedout	bool	REMOTE
+resources.job_runs.*.url	string	INPUT
 resources.jobs.*.budget_policy_id	string	ALL
 resources.jobs.*.continuous	*jobs.Continuous	ALL
 resources.jobs.*.continuous.pause_status	jobs.PauseStatus	ALL

acceptance/bundle/resources/job_runs/basic/databricks.yml

@@ -0,0 +1,15 @@
+bundle:
+ name: job-runs-basic
+
+resources:
+ jobs:
+ my_job:
+ name: my-job
+ tasks:
+ - task_key: main
+ notebook_task:
+ notebook_path: /Workspace/test
+
+ job_runs:
+ my_run:
+ job_id: ${resources.jobs.my_job.id}

acceptance/bundle/resources/job_runs/basic/output.txt

@@ -0,0 +1,99 @@
+
+=== deploy triggers exactly one run
+>>> [CLI] bundle validate
+Name: job-runs-basic
+Target: default
+Workspace:
+ User: [USERNAME]
+ Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default
+
+Validation OK!
+
+>>> [CLI] bundle plan
+create job_runs.my_run
+create jobs.my_job
+
+Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged
+
+>>> [CLI] bundle summary
+Name: job-runs-basic
+Target: default
+Workspace:
+ User: [USERNAME]
+ Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default
+Resources:
+ Job Runs:
+ my_run:
+ Name: 
+ URL: (not deployed)
+ Jobs:
+ my_job:
+ Name: my-job
+ URL: (not deployed)
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle summary
+Name: job-runs-basic
+Target: default
+Workspace:
+ User: [USERNAME]
+ Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default
+Resources:
+ Job Runs:
+ my_run:
+ Name: 
+ URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?w=[NUMID]
+ Jobs:
+ my_job:
+ Name: my-job
+ URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[NUMID]
+
+=== exactly one run-now request was made
+>>> print_requests.py //jobs/run-now
+{
+ "method": "POST",
+ "path": "/api/2.2/jobs/run-now",
+ "body": {
+ "job_id": [MY_JOB_ID]
+ }
+}
+
+=== the triggered run id is stored in state
+>>> read_id.py my_run
+[MY_RUN_ID]
+
+>>> read_id.py my_job
+[MY_JOB_ID]
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+ delete resources.job_runs.my_run
+ delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default
+
+Deleting files...
+Destroy complete!
+
+>>> print_requests.py //jobs/runs/delete
+{
+ "method": "POST",
+ "path": "/api/2.2/jobs/runs/delete",
+ "body": {
+ "run_id": [MY_RUN_ID]
+ }
+}

acceptance/bundle/resources/job_runs/basic/script

@@ -0,0 +1,25 @@
+cleanup() {
+ trace $CLI bundle destroy --auto-approve
+ # destroy deletes the triggered run via jobs/runs/delete (also unlinks out.requests.txt)
+ trace print_requests.py //jobs/runs/delete
+}
+trap cleanup EXIT
+
+title "deploy triggers exactly one run"
+trace $CLI bundle validate
+trace $CLI bundle plan
+trace $CLI bundle summary
+trace $CLI bundle deploy
+
+# confirm that redeploy does not trigger a second run
+trace $CLI bundle deploy
+
+trace $CLI bundle plan
+trace $CLI bundle summary
+
+title "exactly one run-now request was made"
+trace print_requests.py //jobs/run-now
+
+title "the triggered run id is stored in state"
+trace read_id.py my_run
+trace read_id.py my_job

acceptance/bundle/resources/job_runs/basic/test.toml

@@ -0,0 +1,4 @@
+# job_runs is a direct-engine-only resource; the Terraform provider has no
+# equivalent, so restrict the matrix to direct.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+RecordRequests = true

acceptance/bundle/resources/job_runs/job_parameters/databricks.yml

@@ -0,0 +1,26 @@
+bundle:
+ name: job-runs-job-parameters
+
+resources:
+ jobs:
+ my_job:
+ name: my-job
+ parameters:
+ - name: env
+ default: dev
+ - name: region
+ default: us
+ tasks:
+ - task_key: main
+ notebook_task:
+ notebook_path: /Workspace/test
+
+ job_runs:
+ my_run:
+ job_id: ${resources.jobs.my_job.id}
+ # Override only one of the job's two parameters. GetRun resolves the full
+ # parameter set (including the un-overridden `region` default), so without
+ # job_parameters in ignore_remote_changes this drift would recreate the run
+ # on every plan.
+ job_parameters:
+ env: prod

acceptance/bundle/resources/job_runs/job_parameters/output.txt

@@ -0,0 +1,49 @@
+
+=== deploy triggers the run with only the overridden parameter
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //jobs/run-now
+{
+ "method": "POST",
+ "path": "/api/2.2/jobs/run-now",
+ "body": {
+ "job_id": [NUMID],
+ "job_parameters": {
+ "env": "prod"
+ }
+ }
+}
+
+=== plan is stable: the resolved job default does not look like drift
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle summary
+Name: job-runs-job-parameters
+Target: default
+Workspace:
+ User: [USERNAME]
+ Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default
+Resources:
+ Job Runs:
+ my_run:
+ Name: 
+ URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?w=[NUMID]
+ Jobs:
+ my_job:
+ Name: my-job
+ URL: [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID]
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+ delete resources.job_runs.my_run
+ delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/job_runs/job_parameters/script

@@ -0,0 +1,13 @@
+cleanup() {
+ trace $CLI bundle destroy --auto-approve
+ rm out.requests.txt
+}
+trap cleanup EXIT
+
+title "deploy triggers the run with only the overridden parameter"
+trace $CLI bundle deploy
+trace print_requests.py //jobs/run-now
+
+title "plan is stable: the resolved job default does not look like drift"
+trace $CLI bundle plan
+trace $CLI bundle summary

acceptance/bundle/resources/job_runs/job_parameters/test.toml

@@ -0,0 +1,4 @@
+# job_runs is a direct-engine-only resource; the Terraform provider has no
+# equivalent, so restrict the matrix to direct.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+RecordRequests = true

acceptance/bundle/resources/job_runs/redeploy/databricks.yml

@@ -0,0 +1,17 @@
+bundle:
+ name: job-runs-redeploy
+
+resources:
+ jobs:
+ my_job:
+ name: my-job
+ tasks:
+ - task_key: main
+ notebook_task:
+ notebook_path: /Workspace/test
+
+ job_runs:
+ my_run:
+ job_id: ${resources.jobs.my_job.id}
+ job_parameters:
+ env: dev