Added support for lifecycle.started for clusters

acceptance/bundle/refschema/out.fields.txt

@@ -122,7 +122,7 @@ resources.apps.*.git_source.resolved_commit	string	ALL
 resources.apps.*.git_source.source_code_path	string	ALL
 resources.apps.*.git_source.tag	string	ALL
 resources.apps.*.id	string	ALL
-resources.apps.*.lifecycle	*dresources.AppStateLifecycle	REMOTE	STATE
+resources.apps.*.lifecycle	*dresources.StateLifecycle	REMOTE	STATE
 resources.apps.*.lifecycle	*resources.LifecycleWithStarted	INPUT
 resources.apps.*.lifecycle	resources.Lifecycle	INPUT
 resources.apps.*.lifecycle.prevent_destroy	bool	INPUT
@@ -389,8 +389,11 @@ resources.clusters.*.jdbc_port	int	REMOTE
 resources.clusters.*.kind	compute.Kind	ALL
 resources.clusters.*.last_restarted_time	int64	REMOTE
 resources.clusters.*.last_state_loss_time	int64	REMOTE
+resources.clusters.*.lifecycle	*dresources.StateLifecycle	REMOTE	STATE
+resources.clusters.*.lifecycle	*resources.LifecycleWithStarted	INPUT
 resources.clusters.*.lifecycle	resources.Lifecycle	INPUT
 resources.clusters.*.lifecycle.prevent_destroy	bool	INPUT
+resources.clusters.*.lifecycle.started	*bool	ALL
 resources.clusters.*.modified_status	string	INPUT
 resources.clusters.*.node_type_id	string	ALL
 resources.clusters.*.num_workers	int	ALL

acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json

@@ -46,9 +46,13 @@
  "cluster_name": "test-cluster-[UNIQUE_NAME]",
  "driver_node_type_id": "[NODE_TYPE_ID]",
  "enable_elastic_disk": false,
+ "lifecycle": {
+ "started": false
+ },
  "node_type_id": "[NODE_TYPE_ID]",
  "num_workers": 2,
- "spark_version": "13.3.x-snapshot-scala2.12"
+ "spark_version": "13.3.x-snapshot-scala2.12",
+ "state": "TERMINATED"
  },
  "changes": {
  "autoscale": {
@@ -76,6 +80,13 @@
  "reason": "empty",
  "remote": false
  },
+ "lifecycle": {
+ "action": "skip",
+ "reason": "backend_default",
+ "remote": {
+ "started": false
+ }
+ },
  "num_workers": {
  "action": "update",
  "old": 2,
@@ -152,6 +163,13 @@
  "action": "skip",
  "reason": "empty",
  "remote": false
+ },
+ "lifecycle": {
+ "action": "skip",
+ "reason": "backend_default",
+ "remote": {
+ "started": false
+ }
  }
  }
  }
@@ -191,6 +209,9 @@
  "cluster_name": "test-cluster-[UNIQUE_NAME]",
  "driver_node_type_id": "[NODE_TYPE_ID]",
  "enable_elastic_disk": false,
+ "lifecycle": {
+ "started": true
+ },
  "node_type_id": "[NODE_TYPE_ID]",
  "spark_version": "13.3.x-snapshot-scala2.12",
  "state": "RUNNING"
@@ -227,6 +248,13 @@
  "action": "skip",
  "reason": "empty",
  "remote": false
+ },
+ "lifecycle": {
+ "action": "skip",
+ "reason": "backend_default",
+ "remote": {
+ "started": true
+ }
  }
  }
  }
@@ -263,6 +291,9 @@
  "cluster_name": "test-cluster-[UNIQUE_NAME]",
  "driver_node_type_id": "[NODE_TYPE_ID]",
  "enable_elastic_disk": false,
+ "lifecycle": {
+ "started": true
+ },
  "node_type_id": "[NODE_TYPE_ID]",
  "spark_version": "13.3.x-snapshot-scala2.12",
  "state": "RUNNING"
@@ -298,6 +329,13 @@
  "reason": "empty",
  "remote": false
  },
+ "lifecycle": {
+ "action": "skip",
+ "reason": "backend_default",
+ "remote": {
+ "started": true
+ }
+ },
  "num_workers": {
  "action": "resize",
  "reason": "custom",

acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt

@@ -12,6 +12,11 @@ Deployment complete!
  "autoscale": null
 }
 
+=== Terminating the cluster
+{
+ "cluster_id": "[CLUSTER_ID]"
+}
+
 === Adding autoscale section should call update API on stopped cluster
 
 >>> [CLI] bundle deploy

acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script

@@ -15,6 +15,9 @@ CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_clust
 echo "$CLUSTER_ID:CLUSTER_ID" >> ACC_REPLS
 $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}'
 
+title "Terminating the cluster\n"
+$CLI clusters delete "${CLUSTER_ID}" | jq '{cluster_id}'
+
 title "Adding autoscale section should call update API on stopped cluster\n"
 update_file.py databricks.yml " num_workers: 2" " autoscale:
  min_workers: 2

acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json

@@ -26,7 +26,7 @@
  "serial": 1,
  "plan": {
  "resources.clusters.test_cluster": {
- "action": "update",
+ "action": "resize",
  "new_state": {
  "value": {
  "autotermination_minutes": 60,
@@ -49,12 +49,16 @@
  "cluster_name": "test-cluster-[UNIQUE_NAME]",
  "driver_node_type_id": "[NODE_TYPE_ID]",
  "enable_elastic_disk": false,
+ "lifecycle": {
+ "started": true
+ },
  "node_type_id": "[NODE_TYPE_ID]",
  "num_workers": 2,
  "spark_conf": {
  "spark.executor.memory": "2g"
  },
- "spark_version": "13.3.x-snapshot-scala2.12"
+ "spark_version": "13.3.x-snapshot-scala2.12",
+ "state": "RUNNING"
  },
  "changes": {
  "aws_attributes": {
@@ -75,8 +79,16 @@
  "reason": "empty",
  "remote": false
  },
+ "lifecycle": {
+ "action": "skip",
+ "reason": "backend_default",
+ "remote": {
+ "started": true
+ }
+ },
  "num_workers": {
- "action": "update",
+ "action": "resize",
+ "reason": "custom",
  "old": 2,
  "new": 3,
  "remote": 2
@@ -115,6 +127,9 @@
  "cluster_name": "test-cluster-[UNIQUE_NAME]",
  "driver_node_type_id": "[NODE_TYPE_ID]",
  "enable_elastic_disk": false,
+ "lifecycle": {
+ "started": true
+ },
  "node_type_id": "[NODE_TYPE_ID]",
  "num_workers": 3,
  "spark_conf": {
@@ -142,6 +157,13 @@
  "reason": "empty",
  "remote": false
  },
+ "lifecycle": {
+ "action": "skip",
+ "reason": "backend_default",
+ "remote": {
+ "started": true
+ }
+ },
  "num_workers": {
  "action": "resize",
  "reason": "custom",
@@ -183,6 +205,9 @@
  "cluster_name": "test-cluster-[UNIQUE_NAME]",
  "driver_node_type_id": "[NODE_TYPE_ID]",
  "enable_elastic_disk": false,
+ "lifecycle": {
+ "started": true
+ },
  "node_type_id": "[NODE_TYPE_ID]",
  "num_workers": 4,
  "spark_conf": {
@@ -210,6 +235,13 @@
  "reason": "empty",
  "remote": false
  },
+ "lifecycle": {
+ "action": "skip",
+ "reason": "backend_default",
+ "remote": {
+ "started": true
+ }
+ },
  "num_workers": {
  "action": "resize",
  "reason": "custom",

acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.txt

@@ -1,7 +1,7 @@
 create clusters.test_cluster
 
 Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged
-update clusters.test_cluster
+resize clusters.test_cluster
 
 Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
 resize clusters.test_cluster

acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt

@@ -20,21 +20,6 @@ Updating deployment state...
 Deployment complete!
 
 >>> jq select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk) out.requests.txt
-{
- "method": "POST",
- "path": "/api/2.1/clusters/edit",
- "body": {
- "autotermination_minutes": 60,
- "cluster_id": "[CLUSTER_ID]",
- "cluster_name": "test-cluster-[UNIQUE_NAME]",
- "node_type_id": "[NODE_TYPE_ID]",
- "num_workers": 3,
- "spark_conf": {
- "spark.executor.memory": "2g"
- },
- "spark_version": "13.3.x-snapshot-scala2.12"
- }
-}
 
 === Cluster should have new num_workers
 {

acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/databricks.yml.tmpl

@@ -0,0 +1,15 @@
+bundle:
+ name: lifecycle-started-$UNIQUE_NAME
+
+workspace:
+ root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+ clusters:
+ mycluster:
+ cluster_name: $UNIQUE_NAME
+ spark_version: "15.4.x-scala2.12"
+ node_type_id: "i3.xlarge"
+ num_workers: 1
+ lifecycle:
+ started: true

acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/output.txt

@@ -0,0 +1,16 @@
+
+=== bundle plan fails with lifecycle.started on terraform engine
+>>> errcode [CLI] bundle plan
+Error: lifecycle.started is only supported in direct deployment mode
+ in databricks.yml:15:18
+
+
+Exit code: 1
+
+=== bundle deploy fails with lifecycle.started on terraform engine
+>>> errcode [CLI] bundle deploy
+Error: lifecycle.started is only supported in direct deployment mode
+ in databricks.yml:15:18
+
+
+Exit code: 1

acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/script

@@ -0,0 +1,7 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+title "bundle plan fails with lifecycle.started on terraform engine"
+trace errcode $CLI bundle plan
+
+title "bundle deploy fails with lifecycle.started on terraform engine"
+trace errcode $CLI bundle deploy

acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/test.toml

@@ -0,0 +1,6 @@
+Local = true
+Cloud = true
+RecordRequests = false
+
+[EnvMatrix]
+ DATABRICKS_BUNDLE_ENGINE = ["terraform"]

acceptance/bundle/resources/clusters/lifecycle-started-toggle/databricks.yml.tmpl

@@ -0,0 +1,15 @@
+bundle:
+ name: lifecycle-started-toggle-$UNIQUE_NAME
+
+workspace:
+ root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+ clusters:
+ mycluster:
+ cluster_name: $UNIQUE_NAME
+ spark_version: "15.4.x-scala2.12"
+ node_type_id: "i3.xlarge"
+ num_workers: 1
+ lifecycle:
+ started: false